[fix] When the Nat-gw pod container restarts unexpectedly, trigger nat-gw statefulset restart to restore the nat-gw pod configuration (#5070)

Signed-off-by: ykl <[email protected]>
Co-authored-by: ykl <[email protected]>

Author: kldancer

pkg/controller/pod.go

@@ -559,6 +559,12 @@ func (c *Controller) handleAddOrUpdatePod(key string) (err error) {
 		}
 	}
 
+	if vpcGwName, isVpcNatGw := pod.Annotations[util.VpcNatGatewayAnnotation]; isVpcNatGw {
+		if needRestartNatGatewayPod(pod) {
+			c.addOrUpdateVpcNatGatewayQueue.Add(vpcGwName)
+		}
+	}
+
 	// check if route subnet is need.
 	return c.reconcileRouteSubnets(pod, needRouteSubnets(pod, podNets))
 }
@@ -1334,6 +1340,18 @@ func needAllocateSubnets(pod *v1.Pod, nets []*kubeovnNet) []*kubeovnNet {
 	return result
 }
 
+func needRestartNatGatewayPod(pod *v1.Pod) bool {
+	for _, psc := range pod.Status.ContainerStatuses {
+		if psc.Name != "vpc-nat-gw" {
+			continue
+		}
+		if psc.RestartCount > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 func (c *Controller) podNeedSync(pod *v1.Pod) (bool, error) {
 	// 1. check annotations
 	if pod.Annotations == nil {

pkg/controller/vpc_nat_gateway.go

@@ -256,6 +256,19 @@ func (c *Controller) handleAddOrUpdateVpcNatGw(key string) error {
 		return err
 	}
 
+	var natGwPodContainerRestartCount int32
+	oriPod, _err := c.getNatGwPod(key)
+	if _err == nil {
+		pod := oriPod.DeepCopy()
+		for _, psc := range pod.Status.ContainerStatuses {
+			if psc.Name != "vpc-nat-gw" {
+				continue
+			}
+			natGwPodContainerRestartCount = psc.RestartCount
+			break
+		}
+	}
+
 	// check or create statefulset
 	needToCreate := false
 	needToUpdate := false
@@ -269,11 +282,10 @@ func (c *Controller) handleAddOrUpdateVpcNatGw(key string) error {
 			return err
 		}
 	}
-	newSts := c.genNatGwStatefulSet(gw, oldSts.DeepCopy())
-	if !needToCreate && isVpcNatGwChanged(gw) {
+	newSts := c.genNatGwStatefulSet(gw, oldSts.DeepCopy(), natGwPodContainerRestartCount)
+	if !needToCreate && (isVpcNatGwChanged(gw) || natGwPodContainerRestartCount > 0) {
 		needToUpdate = true
 	}
-
 	switch {
 	case needToCreate:
 		// if pod create successfully, will add initVpcNatGatewayQueue
@@ -729,7 +741,7 @@ func (c *Controller) execNatGwRules(pod *corev1.Pod, operation string, rules []s
 	return nil
 }
 
-func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1.StatefulSet) (newSts *v1.StatefulSet) {
+func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1.StatefulSet, natGwPodContainerRestartCount int32) (newSts *v1.StatefulSet) {
 	replicas := int32(1)
 	name := util.GenNatGwStsName(gw.Name)
 	allowPrivilegeEscalation := true
@@ -749,6 +761,14 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 		util.LogicalSwitchAnnotation:     gw.Spec.Subnet,
 		util.IPAddressAnnotation:         gw.Spec.LanIP,
 	}
+
+	if oldSts != nil && len(oldSts.Spec.Template.Annotations) != 0 {
+		if _, ok := oldSts.Spec.Template.Annotations[util.VpcNatGatewayContainerRestartAnnotation]; !ok && natGwPodContainerRestartCount > 0 {
+			podAnnotations[util.VpcNatGatewayContainerRestartAnnotation] = ""
+		}
+	}
+	klog.V(3).Infof("%s podAnnotations:%v", gw.Name, podAnnotations)
+
 	for key, value := range podAnnotations {
 		newPodAnnotations[key] = value
 	}

pkg/util/const.go

@@ -28,20 +28,21 @@ const (
 	ExternalGatewayAnnotation    = "ovn.kubernetes.io/external_gateway"
 	ExternalGwPortNameAnnotation = "ovn.kubernetes.io/external_gw_port_name"
 
-	VpcNatGatewayAnnotation     = "ovn.kubernetes.io/vpc_nat_gw"
-	VpcNatGatewayInitAnnotation = "ovn.kubernetes.io/vpc_nat_gw_init"
-	VpcEipsAnnotation           = "ovn.kubernetes.io/vpc_eips"
-	VpcFloatingIPMd5Annotation  = "ovn.kubernetes.io/vpc_floating_ips"
-	VpcDnatMd5Annotation        = "ovn.kubernetes.io/vpc_dnat_md5"
-	VpcSnatMd5Annotation        = "ovn.kubernetes.io/vpc_snat_md5"
-	VpcCIDRsAnnotation          = "ovn.kubernetes.io/vpc_cidrs"
-	VpcLbAnnotation             = "ovn.kubernetes.io/vpc_lb"
-	VpcExternalLabel            = "ovn.kubernetes.io/vpc_external"
-	VpcEipAnnotation            = "ovn.kubernetes.io/vpc_eip"
-	VpcDnatEPortLabel           = "ovn.kubernetes.io/vpc_dnat_eport"
-	VpcNatAnnotation            = "ovn.kubernetes.io/vpc_nat"
-	OvnEipTypeLabel             = "ovn.kubernetes.io/ovn_eip_type"
-	EipV4IpLabel                = "ovn.kubernetes.io/eip_v4_ip"
+	VpcNatGatewayAnnotation                 = "ovn.kubernetes.io/vpc_nat_gw"
+	VpcNatGatewayInitAnnotation             = "ovn.kubernetes.io/vpc_nat_gw_init"
+	VpcNatGatewayContainerRestartAnnotation = "ovn.kubernetes.io/vpc_nat_gw_container_restarted"
+	VpcEipsAnnotation                       = "ovn.kubernetes.io/vpc_eips"
+	VpcFloatingIPMd5Annotation              = "ovn.kubernetes.io/vpc_floating_ips"
+	VpcDnatMd5Annotation                    = "ovn.kubernetes.io/vpc_dnat_md5"
+	VpcSnatMd5Annotation                    = "ovn.kubernetes.io/vpc_snat_md5"
+	VpcCIDRsAnnotation                      = "ovn.kubernetes.io/vpc_cidrs"
+	VpcLbAnnotation                         = "ovn.kubernetes.io/vpc_lb"
+	VpcExternalLabel                        = "ovn.kubernetes.io/vpc_external"
+	VpcEipAnnotation                        = "ovn.kubernetes.io/vpc_eip"
+	VpcDnatEPortLabel                       = "ovn.kubernetes.io/vpc_dnat_eport"
+	VpcNatAnnotation                        = "ovn.kubernetes.io/vpc_nat"
+	OvnEipTypeLabel                         = "ovn.kubernetes.io/ovn_eip_type"
+	EipV4IpLabel                            = "ovn.kubernetes.io/eip_v4_ip"
 
 	SwitchLBRuleVipsAnnotation = "ovn.kubernetes.io/switch_lb_vip"
 	SwitchLBRuleVip            = "switch_lb_vip"