Move to use newer IPsec DaemonSets when MCP is in paused state

When MCP is in paused state, network operator continues to render older
IPsec daemonsets which blocks network cluster operator not getting upgraded
to newer version. Hence this commit renders newer IPsec daemonsets for
intermediate period. When MCPs are moved to unpaused state and IPsec machine
configs are installed on it, then it goes ahead with rendering only host
flavored IPsec daemonset.

Signed-off-by: Periyasamy Palanisamy <[email protected]>

Author: pperiyasamy

bindata/network/ovn-kubernetes/common/ipsec-containerized.yaml

@@ -49,6 +49,14 @@ spec:
         - |
           #!/bin/bash
           set -exuo pipefail
+
+{{ if .IPsecCheckForLibreswan }}
+          if rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host has libreswan and therefore ipsec will be configured by ipsec host daemonset, this ovn ipsec container doesnt need to init anything"
+            exit 0
+          fi
+{{ end }}
+
 {{ if .NETWORK_NODE_IDENTITY_ENABLE }}
           # When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig
           # that will be used to talk to the API
@@ -189,6 +197,9 @@ spec:
           name: signer-ca
         - mountPath: /etc/openvswitch
           name: etc-openvswitch
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -220,6 +231,13 @@ spec:
           }
           trap cleanup SIGTERM
 
+{{ if .IPsecCheckForLibreswan }}
+          if rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host has libreswan and therefore ipsec will be configured by ipsec host daemonset, this ovn ipsec container will sleep to infinity"
+            sleep infinity
+          fi
+{{ end }}
+
           # Don't start IPsec until ovnkube-node has finished setting up the node
           counter=0
           until [ -f /etc/cni/net.d/10-ovn-kubernetes.conf ]
@@ -276,6 +294,9 @@ spec:
           name: host-var-log-ovs
         - mountPath: /etc/openvswitch
           name: etc-openvswitch
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -288,6 +309,12 @@ spec:
             - -c
             - |
               #!/bin/bash
+{{ if .IPsecCheckForLibreswan }}
+              if rpm --dbpath=/usr/share/rpm -q libreswan; then
+                echo "host has libreswan and therefore ipsec will be configured by ipsec host daemonset, this ovn ipsec container is always \"alive\""
+                exit 0
+              fi
+{{ end }}
               if [[ $(ipsec whack --trafficstatus | wc -l) -eq 0 ]]; then
                 echo "no ipsec traffic configured"
                 exit 10
@@ -321,6 +348,10 @@ spec:
       - name: host-cni-netd
         hostPath:
           path: "{{.CNIConfDir}}"
+      - name: host-usr-share-rpm
+        hostPath:
+          path: /usr/share/rpm
+          type: DirectoryOrCreate
       tolerations:
       - operator: "Exists"
 {{end}}

bindata/network/ovn-kubernetes/common/ipsec-host.yaml

@@ -50,6 +50,12 @@ spec:
         - |
           #!/bin/bash
           set -exuo pipefail
+{{ if .IPsecCheckForLibreswan }}
+          if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, this ovn ipsec container has nothing to init"
+            exit 0
+          fi
+{{ end }}
 {{ if .NETWORK_NODE_IDENTITY_ENABLE }}
           # When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig
           # that will be used to talk to the API
@@ -194,6 +200,9 @@ spec:
           name: etc-openvswitch
         - mountPath: /etc
           name: host-etc
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -210,6 +219,12 @@ spec:
           #!/bin/bash
           set -exuo pipefail
 
+{{ if .IPsecCheckForLibreswan }}
+          if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+            echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, this ovn ipsec container will sleep to infinity"
+            sleep infinity
+          fi
+{{ end }}
 
           # Don't start IPsec until ovnkube-node has finished setting up the node
           counter=0
@@ -268,6 +283,13 @@ spec:
                    # In order to maintain traffic flows during container restart, we
                    # need to ensure that xfrm state and policies are not flushed.
 
+{{ if .IPsecCheckForLibreswan }}
+                   if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+                     echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, preStop wont do anything"
+                     exit 0
+                   fi
+{{ end }}
+
                    # Don't allow ovs monitor to cleanup persistent state
                    kill "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)" 2>/dev/null || true
         env:
@@ -291,6 +313,9 @@ spec:
           name: host-var-lib
         - mountPath: /etc
           name: host-etc
+        - mountPath: /usr/share/rpm
+          name: host-usr-share-rpm
+          readOnly: true
         resources:
           requests:
             cpu: 10m
@@ -303,6 +328,12 @@ spec:
             - -c
             - |
               #!/bin/bash
+{{ if .IPsecCheckForLibreswan }}
+              if ! rpm --dbpath=/usr/share/rpm -q libreswan; then
+                echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-containerized daemonset, this ovn ipsec container is always \"alive\""
+                exit 0
+              fi
+{{ end }}
               if [[ $(ipsec whack --trafficstatus | wc -l) -eq 0 ]]; then
                 echo "no ipsec traffic configured"
                 exit 10
@@ -346,6 +377,10 @@ spec:
           path: /etc
           type: Directory
         name: host-etc
+      - name: host-usr-share-rpm
+        hostPath:
+          path: /usr/share/rpm
+          type: DirectoryOrCreate
       tolerations:
       - operator: "Exists"
 {{end}}

pkg/bootstrap/types.go

@@ -48,8 +48,7 @@ type OVNUpdateStatus struct {
 // OVNIPsecStatus contains status of current IPsec configuration
 // in the cluster.
 type OVNIPsecStatus struct {
-	LegacyIPsecUpgrade bool // true if IPsec in 4.14 or Pre-4.14 cluster is upgraded to latest version
-	OVNIPsecActive     bool // set to true unless we are sure it is not.
+	OVNIPsecActive bool // set to true unless we are sure it is not.
 }
 
 type OVNBootstrapResult struct {

pkg/network/ovn_kubernetes.go

@@ -289,6 +289,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	data.Data["IPsecMachineConfigEnable"] = IPsecMachineConfigEnable
 	data.Data["OVNIPsecDaemonsetEnable"] = OVNIPsecDaemonsetEnable
 	data.Data["OVNIPsecEnable"] = OVNIPsecEnable
+	data.Data["IPsecCheckForLibreswan"] = renderIPsecHostDaemonSet && renderIPsecContainerizedDaemonSet
 
 	// Set progressing to true until IPsec DaemonSet is rendered when EW IPsec config is enabled.
 	// TODO Do a poor man's job mapping machine config pool status to CNO progressing state for now.
@@ -499,8 +500,8 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 		objs = k8s.RemoveObjByGroupKindName(objs, "apps", "DaemonSet", util.OVN_NAMESPACE, "ovn-ipsec-containerized")
 	}
 
-	// When upgrading a legacy IPsec deployment, avoid any updates until IPsec MachineConfigs
-	// are active.
+	// When disabling IPsec deployment, avoid any updates until IPsec is completely
+	// disabled from OVN.
 	if renderIPsecDaemonSetAsCreateWaitOnly {
 		k8s.UpdateObjByGroupKindName(objs, "apps", "DaemonSet", util.OVN_NAMESPACE, "ovn-ipsec-host", func(o *uns.Unstructured) {
 			anno := o.GetAnnotations()
@@ -519,25 +520,6 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 			anno[names.CreateWaitAnnotation] = "true"
 			o.SetAnnotations(anno)
 		})
-		// The legacy ovn-ipsec deployment is only rendered during upgrades until we
-		// are ready to remove it.
-		ovnIPsecLegacyDS := &appsv1.DaemonSet{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "DaemonSet",
-				APIVersion: appsv1.SchemeGroupVersion.String(),
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "ovn-ipsec",
-				Namespace: util.OVN_NAMESPACE,
-				// We never update the legacy ovn-ipsec daemonset.
-				Annotations: map[string]string{names.CreateWaitAnnotation: "true"},
-			},
-		}
-		obj, err := k8s.ToUnstructured(ovnIPsecLegacyDS)
-		if err != nil {
-			return nil, progressing, fmt.Errorf("unable to render legacy ovn-ipsec daemonset: %w", err)
-		}
-		objs = append(objs, obj)
 	}
 
 	klog.Infof("ovnk components: ovnkube-node: isRunning=%t, update=%t; ovnkube-control-plane: isRunning=%t, update=%t",
@@ -627,7 +609,6 @@ func IsIPsecLegacyAPI(conf *operv1.OVNKubernetesConfig) bool {
 func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootstrap.BootstrapResult) (renderCNOIPsecMachineConfig, renderIPsecDaemonSet,
 	renderIPsecOVN, renderIPsecHostDaemonSet, renderIPsecContainerizedDaemonSet, renderIPsecDaemonSetAsCreateWaitOnly bool) {
 	isHypershiftHostedCluster := bootstrapResult.Infra.HostedControlPlane != nil
-	isIpsecUpgrade := bootstrapResult.OVN.IPsecUpdateStatus != nil && bootstrapResult.OVN.IPsecUpdateStatus.LegacyIPsecUpgrade
 	isOVNIPsecActive := bootstrapResult.OVN.IPsecUpdateStatus != nil && bootstrapResult.OVN.IPsecUpdateStatus.OVNIPsecActive
 
 	mode := GetIPsecMode(conf)
@@ -636,7 +617,6 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 	// change to them. So during upgrade, we must keep track if IPsec MachineConfigs are
 	// active or not for non Hybrid hosted cluster.
 	isIPsecMachineConfigActive := isIPsecMachineConfigActive(bootstrapResult.Infra)
-	isIPsecMachineConfigNotActiveOnUpgrade := isIpsecUpgrade && !isIPsecMachineConfigActive && !isHypershiftHostedCluster
 	isMachineConfigClusterOperatorReady := bootstrapResult.Infra.MachineConfigClusterOperatorReady
 	isCNOIPsecMachineConfigPresent := isCNOIPsecMachineConfigPresent(bootstrapResult.Infra)
 
@@ -645,15 +625,19 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 	renderIPsecDaemonSet = isOVNIPsecActive || mode == operv1.IPsecModeFull
 
 	// If ipsec is enabled, we render the host ipsec deployment except for
-	// hypershift hosted clusters and we need to wait for the ipsec MachineConfig
-	// extensions to be active first. We must also render host ipsec deployment
-	// at the time of upgrade though user created IPsec Machine Config is not
-	// present/active.
-	renderIPsecHostDaemonSet = (renderIPsecDaemonSet && isIPsecMachineConfigActive && !isHypershiftHostedCluster) || isIPsecMachineConfigNotActiveOnUpgrade
-
-	// The containerized ipsec deployment is only rendered during upgrades or
-	// for hypershift hosted clusters.
-	renderIPsecContainerizedDaemonSet = (renderIPsecDaemonSet && isHypershiftHostedCluster) || isIPsecMachineConfigNotActiveOnUpgrade
+	// hypershift hosted clusters. We must also render host ipsec daemonset
+	// when any of the machine config pool in paused state without ipsec machine
+	// config deployed and at the same time unpaused pools having ipsec machine
+	// config deployed. This helps host ipsec daemonset pods to be effective on
+	// unpaused pool nodes.
+	renderIPsecHostDaemonSet = renderIPsecDaemonSet && !isHypershiftHostedCluster
+
+	// The containerized ipsec deployment is only rendered for hypershift hosted clusters.
+	// We must also render containerized ipsec daemonset when any of the machine config
+	// pool in paused state without ipsec machine config deployed and at the same time
+	// unpaused pools having ipsec machine config deployed. This helps containerized daemonset
+	// pods to be effective on paused pool nodes.
+	renderIPsecContainerizedDaemonSet = (renderIPsecDaemonSet && isHypershiftHostedCluster) || !isIPsecMachineConfigActive
 
 	// MachineConfig IPsec extensions rollout is needed for the ipsec enablement and are used in both External and Full modes.
 	// except  when the containerized deployment is used in hypershift hosted clusters.
@@ -666,9 +650,8 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 	// to be active if it's not an upgrade and not a hypershift hosted cluster.
 	renderIPsecOVN = (renderIPsecHostDaemonSet || renderIPsecContainerizedDaemonSet) && mode == operv1.IPsecModeFull
 
-	// While OVN ipsec is being upgraded and IPsec MachineConfigs deployment is in progress
-	// (or) IPsec config in OVN is being disabled, then ipsec deployment is not updated.
-	renderIPsecDaemonSetAsCreateWaitOnly = isIPsecMachineConfigNotActiveOnUpgrade || (isOVNIPsecActive && !renderIPsecOVN)
+	// IPsec config in OVN is being disabled, then ipsec deployment is not updated.
+	renderIPsecDaemonSetAsCreateWaitOnly = isOVNIPsecActive && !renderIPsecOVN
 
 	return
 }
@@ -1203,66 +1186,40 @@ func bootstrapOVN(conf *operv1.Network, kubeClient cnoclient.Client, infraStatus
 		prepullerStatus.Progressing = daemonSetProgressing(prePullerDaemonSet, true)
 	}
 
-	ipsecDaemonSet := &appsv1.DaemonSet{
+	ipsecStatus := &bootstrap.OVNIPsecStatus{}
+	ipsecContainerizedDaemonSet := &appsv1.DaemonSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "DaemonSet",
 			APIVersion: appsv1.SchemeGroupVersion.String(),
 		},
 	}
-
-	ipsecStatus := &bootstrap.OVNIPsecStatus{}
-
-	// The IPsec daemonset name is ovn-ipsec if we are upgrading from <= 4.13.
-	nsn = types.NamespacedName{Namespace: util.OVN_NAMESPACE, Name: "ovn-ipsec"}
-	if err := kubeClient.ClientFor("").CRClient().Get(context.TODO(), nsn, ipsecDaemonSet); err != nil {
+	ipsecHostDaemonSet := &appsv1.DaemonSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "DaemonSet",
+			APIVersion: appsv1.SchemeGroupVersion.String(),
+		},
+	}
+	// Retrieve container based IPsec daemonset with name ovn-ipsec-containerized.
+	nsn = types.NamespacedName{Namespace: util.OVN_NAMESPACE, Name: "ovn-ipsec-containerized"}
+	if err := kubeClient.ClientFor("").CRClient().Get(context.TODO(), nsn, ipsecContainerizedDaemonSet); err != nil {
 		if !apierrors.IsNotFound(err) {
-			return nil, fmt.Errorf("Failed to retrieve existing pre-4.14 ipsec DaemonSet: %w", err)
+			return nil, fmt.Errorf("Failed to retrieve ipsec containerized DaemonSet: %w", err)
 		} else {
-			ipsecStatus = nil
+			ipsecContainerizedDaemonSet = nil
 		}
-	} else {
-		ipsecStatus.LegacyIPsecUpgrade = true
-	}
-
-	if ipsecStatus == nil {
-		ipsecStatus = &bootstrap.OVNIPsecStatus{}
-		ipsecContainerizedDaemonSet := &appsv1.DaemonSet{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "DaemonSet",
-				APIVersion: appsv1.SchemeGroupVersion.String(),
-			},
-		}
-		ipsecHostDaemonSet := &appsv1.DaemonSet{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "DaemonSet",
-				APIVersion: appsv1.SchemeGroupVersion.String(),
-			},
-		}
-		// Retrieve container based IPsec daemonset with name ovn-ipsec-containerized.
-		nsn = types.NamespacedName{Namespace: util.OVN_NAMESPACE, Name: "ovn-ipsec-containerized"}
-		if err := kubeClient.ClientFor("").CRClient().Get(context.TODO(), nsn, ipsecContainerizedDaemonSet); err != nil {
-			if !apierrors.IsNotFound(err) {
-				return nil, fmt.Errorf("Failed to retrieve existing ipsec containerized DaemonSet: %w", err)
-			} else {
-				ipsecContainerizedDaemonSet = nil
-			}
-		}
-		// Retrieve host based IPsec daemonset with name ovn-ipsec-host
-		nsn = types.NamespacedName{Namespace: util.OVN_NAMESPACE, Name: "ovn-ipsec-host"}
-		if err := kubeClient.ClientFor("").CRClient().Get(context.TODO(), nsn, ipsecHostDaemonSet); err != nil {
-			if !apierrors.IsNotFound(err) {
-				return nil, fmt.Errorf("Failed to retrieve existing ipsec host DaemonSet: %w", err)
-			} else {
-				ipsecHostDaemonSet = nil
-			}
-		}
-		if ipsecContainerizedDaemonSet != nil && ipsecHostDaemonSet != nil {
-			// Both IPsec daemonset versions exist, so this is an upgrade from 4.14.
-			ipsecStatus.LegacyIPsecUpgrade = true
-		} else if ipsecContainerizedDaemonSet == nil && ipsecHostDaemonSet == nil {
-			ipsecStatus = nil
+	}
+	// Retrieve host based IPsec daemonset with name ovn-ipsec-host
+	nsn = types.NamespacedName{Namespace: util.OVN_NAMESPACE, Name: "ovn-ipsec-host"}
+	if err := kubeClient.ClientFor("").CRClient().Get(context.TODO(), nsn, ipsecHostDaemonSet); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return nil, fmt.Errorf("Failed to retrieve ipsec host DaemonSet: %w", err)
+		} else {
+			ipsecHostDaemonSet = nil
 		}
 	}
+	if ipsecContainerizedDaemonSet == nil && ipsecHostDaemonSet == nil {
+		ipsecStatus = nil
+	}
 
 	// set OVN IPsec status into ipsecStatus only when IPsec daemonset(s) exists in the cluster.
 	if ipsecStatus != nil {