[fix] When the Nat-gw pod container restarts unexpectedly, trigger nat-gw statefulset restart to restore the nat-gw pod configuration (#5072)

Signed-off-by: ykl <[email protected]>
Co-authored-by: ykl <[email protected]>

Author: kldancer

pkg/controller/pod.go

@@ -485,6 +485,12 @@ func (c *Controller) handleAddOrUpdatePod(key string) (err error) {
 		}
 	}
 
+	if vpcGwName, isVpcNatGw := pod.Annotations[util.VpcNatGatewayAnnotation]; isVpcNatGw {
+		if needRestartNatGatewayPod(pod) {
+			c.addOrUpdateVpcNatGatewayQueue.Add(vpcGwName)
+		}
+	}
+
 	// check if route subnet is need.
 	return c.reconcileRouteSubnets(pod, needRouteSubnets(pod, podNets))
 }
@@ -1349,6 +1355,18 @@ func needAllocateSubnets(pod *v1.Pod, nets []*kubeovnNet) []*kubeovnNet {
 	return result
 }
 
+func needRestartNatGatewayPod(pod *v1.Pod) bool {
+	for _, psc := range pod.Status.ContainerStatuses {
+		if psc.Name != "vpc-nat-gw" {
+			continue
+		}
+		if psc.RestartCount > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 func (c *Controller) podNeedSync(pod *v1.Pod) (bool, error) {
 	// 1. check annotations
 	if pod.Annotations == nil {

pkg/controller/vpc_nat_gateway.go

@@ -179,6 +179,18 @@ func (c *Controller) handleAddOrUpdateVpcNatGw(key string) error {
 		return err
 	}
 
+	var natGwPodContainerRestartCount int32
+	pod, _err := c.getNatGwPod(key)
+	if _err == nil {
+		for _, psc := range pod.Status.ContainerStatuses {
+			if psc.Name != "vpc-nat-gw" {
+				continue
+			}
+			natGwPodContainerRestartCount = psc.RestartCount
+			break
+		}
+	}
+
 	// check or create statefulset
 	needToCreate := false
 	needToUpdate := false
@@ -191,12 +203,12 @@ func (c *Controller) handleAddOrUpdateVpcNatGw(key string) error {
 		}
 		needToCreate, oldSts = true, nil
 	}
-	newSts, err := c.genNatGwStatefulSet(gw, oldSts)
+	newSts, err := c.genNatGwStatefulSet(gw, oldSts, natGwPodContainerRestartCount)
 	if err != nil {
 		klog.Error(err)
 		return err
 	}
-	if !needToCreate && isVpcNatGwChanged(gw) {
+	if !needToCreate && (isVpcNatGwChanged(gw) || natGwPodContainerRestartCount > 0) {
 		needToUpdate = true
 	}
 
@@ -717,7 +729,7 @@ func (c *Controller) setNatGwAPIRoute(annotations map[string]string, nadNamespac
 	return nil
 }
 
-func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1.StatefulSet) (*v1.StatefulSet, error) {
+func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1.StatefulSet, natGwPodContainerRestartCount int32) (*v1.StatefulSet, error) {
 	annotations := make(map[string]string, 7)
 	if oldSts != nil && len(oldSts.Annotations) != 0 {
 		annotations = maps.Clone(oldSts.Annotations)
@@ -731,6 +743,13 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 		util.IPAddressAnnotation:     gw.Spec.LanIP,
 	}
 
+	if oldSts != nil && len(oldSts.Spec.Template.Annotations) != 0 {
+		if _, ok := oldSts.Spec.Template.Annotations[util.VpcNatGatewayContainerRestartAnnotation]; !ok && natGwPodContainerRestartCount > 0 {
+			podAnnotations[util.VpcNatGatewayContainerRestartAnnotation] = ""
+		}
+	}
+	klog.V(3).Infof("%s podAnnotations:%v", gw.Name, podAnnotations)
+
 	// Add an interface that can reach the API server, we need access to it to probe Kube-OVN resources
 	if gw.Spec.BgpSpeaker.Enabled {
 		if err := c.setNatGwAPIAccess(podAnnotations, externalNetworkNad); err != nil {

pkg/util/const.go

@@ -24,22 +24,23 @@ const (
 	VMAnnotation                 = "ovn.kubernetes.io/virtualmachine"
 	ActivationStrategyAnnotation = "ovn.kubernetes.io/activation_strategy"
 
-	VpcNatGatewayAnnotation          = "ovn.kubernetes.io/vpc_nat_gw"
-	VpcNatGatewayInitAnnotation      = "ovn.kubernetes.io/vpc_nat_gw_init"
-	VpcNatGatewayActivatedAnnotation = "ovn.kubernetes.io/vpc_nat_gw_activated"
-	VpcEipsAnnotation                = "ovn.kubernetes.io/vpc_eips"
-	VpcFloatingIPMd5Annotation       = "ovn.kubernetes.io/vpc_floating_ips"
-	VpcDnatMd5Annotation             = "ovn.kubernetes.io/vpc_dnat_md5"
-	VpcSnatMd5Annotation             = "ovn.kubernetes.io/vpc_snat_md5"
-	VpcCIDRsAnnotation               = "ovn.kubernetes.io/vpc_cidrs"
-	VpcLbAnnotation                  = "ovn.kubernetes.io/vpc_lb"
-	VpcExternalLabel                 = "ovn.kubernetes.io/vpc_external"
-	VpcEipAnnotation                 = "ovn.kubernetes.io/vpc_eip"
-	VpcDnatEPortLabel                = "ovn.kubernetes.io/vpc_dnat_eport"
-	VpcNatAnnotation                 = "ovn.kubernetes.io/vpc_nat"
-	OvnEipTypeLabel                  = "ovn.kubernetes.io/ovn_eip_type"
-	EipV4IpLabel                     = "ovn.kubernetes.io/eip_v4_ip"
-	EipV6IpLabel                     = "ovn.kubernetes.io/eip_v6_ip"
+	VpcNatGatewayAnnotation                 = "ovn.kubernetes.io/vpc_nat_gw"
+	VpcNatGatewayInitAnnotation             = "ovn.kubernetes.io/vpc_nat_gw_init"
+	VpcNatGatewayContainerRestartAnnotation = "ovn.kubernetes.io/vpc_nat_gw_container_restarted"
+	VpcNatGatewayActivatedAnnotation        = "ovn.kubernetes.io/vpc_nat_gw_activated"
+	VpcEipsAnnotation                       = "ovn.kubernetes.io/vpc_eips"
+	VpcFloatingIPMd5Annotation              = "ovn.kubernetes.io/vpc_floating_ips"
+	VpcDnatMd5Annotation                    = "ovn.kubernetes.io/vpc_dnat_md5"
+	VpcSnatMd5Annotation                    = "ovn.kubernetes.io/vpc_snat_md5"
+	VpcCIDRsAnnotation                      = "ovn.kubernetes.io/vpc_cidrs"
+	VpcLbAnnotation                         = "ovn.kubernetes.io/vpc_lb"
+	VpcExternalLabel                        = "ovn.kubernetes.io/vpc_external"
+	VpcEipAnnotation                        = "ovn.kubernetes.io/vpc_eip"
+	VpcDnatEPortLabel                       = "ovn.kubernetes.io/vpc_dnat_eport"
+	VpcNatAnnotation                        = "ovn.kubernetes.io/vpc_nat"
+	OvnEipTypeLabel                         = "ovn.kubernetes.io/ovn_eip_type"
+	EipV4IpLabel                            = "ovn.kubernetes.io/eip_v4_ip"
+	EipV6IpLabel                            = "ovn.kubernetes.io/eip_v6_ip"
 
 	SwitchLBRuleVipsAnnotation = "ovn.kubernetes.io/switch_lb_vip"
 	SwitchLBRuleVip            = "switch_lb_vip"