Add version check for machine config operator

The CNO started using machine configs from 4.15 for IPsec deployment, so adding
a check for machine config operator to be at least >= 4.15 to roll out IPsec
machine configs. Otherwise during OCP 4.14->4.15 upgrade, even before MCO is
upgraded to 4.15, IPsec machine configs are rolled out, it uses ipsec extension
from 4.14 version to install packages, installs libreswan 4.9 version on the
node intermeditately. So this MCO version check ensures IPsec machine configs
are rendered after MCO is upgraded to 4.15 and nodes get desired libreswan
version 4.6.

Signed-off-by: Periyasamy Palanisamy <[email protected]>

Author: pperiyasamy

pkg/network/ovn_kubernetes.go

@@ -46,6 +46,7 @@ import (
 	iputil "github.com/openshift/cluster-network-operator/pkg/util/ip"
 	"github.com/openshift/cluster-network-operator/pkg/util/k8s"
 	mcutil "github.com/openshift/cluster-network-operator/pkg/util/machineconfig"
+	"github.com/openshift/cluster-network-operator/pkg/version"
 )
 
 const CLUSTER_CONFIG_NAME = "cluster-config-v1"
@@ -598,6 +599,34 @@ func shouldRenderIPsec(conf *operv1.OVNKubernetesConfig, bootstrapResult *bootst
 	// with the the IPsec MachineConfig extensions active, the containerized
 	// daemonset is dormant and the host daemonset is active. When the upgrade
 	// finishes, the containerized daemonset is then not rendered.
+	//
+	// The upgrade from 4.14 is handled very carefully to correctly migrate
+	// from containerized ipsec deployment to the host ipsec deployment.
+	//  1. OCP 4.14 with container ipsec deployment is active using libreswan
+	//     4.6.3; and host ipsec deployment is dormant.
+	//  2. Start the 4.15 upgrade.
+	//  3. CNO upgrades to 4.15.
+	//  4. CNO renders 4.15 versions of the container ipsec deployment and
+	//     host ipsec deployment with no state change. However the host ipsec
+	//     deployment mounts to top system level directories for the host ipsec
+	//     path for this upgrade scenario. It fixes two problems.
+	//     a) version mismatch between libreswan installed on the host and
+	//        host ipsec deployment pod container.
+	//     b) host ipsec deployment pod goes into pending state if we mount the
+	//        binaries directly and libreswan has not been installed yet
+	//        installed on the host by IPsec machine configs.
+	//  5. CNO waits until MCO is upgraded to 4.15 and then deploys CNO ipsec
+	//     machine configs that will install and run libreswan 4.6.3 on the
+	//     host. Otherwise, without waiting for MCO 4.15, libreswan 4.9 may
+	//     be installed from 4.14 MCO which has all known stability problems
+	//     found from the bugs.
+	//     https://issues.redhat.com/browse/OCPBUGS-41823
+	//     https://issues.redhat.com/browse/OCPBUGS-42952
+	//  6. Host ipsec deployment becomes active using libreswan 4.6.3 from the
+	//     container which can successfully run against libreswan 4.6.3 running
+	//     on the host.
+	//  7. At the same time as step 6, containerized ipsec deployment becomes
+	//     dormant, and eventually gets removed when the upgrade is done.
 
 	isHypershiftHostedCluster := bootstrapResult.Infra.HostedControlPlane != nil
 	isOVNIPsecActiveOrRollingOut := bootstrapResult.OVN.IPsecUpdateStatus != nil && bootstrapResult.OVN.IPsecUpdateStatus.IsOVNIPsecActiveOrRollingOut
@@ -1486,10 +1515,10 @@ func shouldUpdateOVNKonUpgrade(ovn bootstrap.OVNBootstrapResult, releaseVersion
 
 	// compute version delta
 	// versionUpgrade means the existing daemonSet needs an upgrade.
-	controlPlaneDelta := compareVersions(controlPlaneVersion, releaseVersion)
-	nodeDelta := compareVersions(nodeVersion, releaseVersion)
+	controlPlaneDelta := version.CompareVersions(controlPlaneVersion, releaseVersion)
+	nodeDelta := version.CompareVersions(nodeVersion, releaseVersion)
 
-	if controlPlaneDelta == versionUnknown || nodeDelta == versionUnknown {
+	if controlPlaneDelta == version.VersionUnknown || nodeDelta == version.VersionUnknown {
 		klog.Warningf("could not determine ovn-kubernetes daemonset update directions; node: %s, control-plane: %s, release: %s",
 			nodeVersion, controlPlaneVersion, releaseVersion)
 		return true, true
@@ -1513,14 +1542,14 @@ func shouldUpdateOVNKonUpgrade(ovn bootstrap.OVNBootstrapResult, releaseVersion
 
 	// both older (than CNO)
 	// Update node only.
-	if controlPlaneDelta == versionUpgrade && nodeDelta == versionUpgrade {
+	if controlPlaneDelta == version.VersionUpgrade && nodeDelta == version.VersionUpgrade {
 		klog.V(2).Infof("Upgrading OVN-Kubernetes node before control-plane")
 		return true, false
 	}
 
 	// control plane older, node updated
 	// update control plane if node is rolled out
-	if controlPlaneDelta == versionUpgrade && nodeDelta == versionSame {
+	if controlPlaneDelta == version.VersionUpgrade && nodeDelta == version.VersionSame {
 		if ovn.NodeUpdateStatus.Progressing {
 			klog.V(2).Infof("Waiting for OVN-Kubernetes node update to roll out before updating control-plane")
 			return true, false
@@ -1531,14 +1560,14 @@ func shouldUpdateOVNKonUpgrade(ovn bootstrap.OVNBootstrapResult, releaseVersion
 
 	// both newer
 	// downgrade control plane before node
-	if controlPlaneDelta == versionDowngrade && nodeDelta == versionDowngrade {
+	if controlPlaneDelta == version.VersionDowngrade && nodeDelta == version.VersionDowngrade {
 		klog.V(2).Infof("Downgrading OVN-Kubernetes control-plane before node")
 		return false, true
 	}
 
 	// control plane same, node needs downgrade
 	// wait for control plane rollout
-	if controlPlaneDelta == versionSame && nodeDelta == versionDowngrade {
+	if controlPlaneDelta == version.VersionSame && nodeDelta == version.VersionDowngrade {
 		if ovn.ControlPlaneUpdateStatus.Progressing {
 			klog.V(2).Infof("Waiting for OVN-Kubernetes control-plane downgrade to roll out before downgrading node")
 			return false, true
@@ -1548,7 +1577,7 @@ func shouldUpdateOVNKonUpgrade(ovn bootstrap.OVNBootstrapResult, releaseVersion
 	}
 
 	// unlikely, should be caught above
-	if controlPlaneDelta == versionSame && nodeDelta == versionSame {
+	if controlPlaneDelta == version.VersionSame && nodeDelta == version.VersionSame {
 		return true, true
 	}
 
@@ -1701,7 +1730,7 @@ func isOVNIPsecNotActiveInDaemonSet(ds *appsv1.DaemonSet) bool {
 		return false
 	}
 	// If IPsec is running with older version and ipsec=true is found from nbdb container, then return false.
-	if !isVersionGreaterThanOrEqualTo(annotations["release.openshift.io/version"], 4, 15) &&
+	if !version.IsVersionGreaterThanOrEqualTo(annotations["release.openshift.io/version"], 4, 15) &&
 		isIPSecEnabledInPod(ds.Spec.Template, util.OVN_NBDB) {
 		return false
 	}

pkg/platform/platform.go

@@ -12,6 +12,7 @@ import (
 	"github.com/openshift/cluster-network-operator/pkg/hypershift"
 	"github.com/openshift/cluster-network-operator/pkg/names"
 	mcutil "github.com/openshift/cluster-network-operator/pkg/util/machineconfig"
+	"github.com/openshift/cluster-network-operator/pkg/version"
 	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -234,7 +235,16 @@ func isMachineConfigClusterOperatorReady(client cnoclient.Client) (bool, error)
 			progressing = isConditionTrue
 		}
 	}
-	machineConfigClusterOperatorReady := available && !degraded && !progressing
+	// The network operator is supporting machine configs starting with IPsec machine configs from 4.15, so
+	// we need to consider it has to be >= 4.15 as well.
+	var isDesiredOperatorVersion bool
+	for _, v := range machineConfigClusterOperator.Status.Versions {
+		if v.Name == "operator" {
+			isDesiredOperatorVersion = version.IsVersionGreaterThanOrEqualTo(v.Version, 4, 15)
+			break
+		}
+	}
+	machineConfigClusterOperatorReady := available && !degraded && !progressing && isDesiredOperatorVersion
 	return machineConfigClusterOperatorReady, nil
 }
 

pkg/network/semver.go renamed to pkg/version/semver.go

@@ -1,4 +1,4 @@
-package network
+package version
 
 import (
 	"github.com/Masterminds/semver"
@@ -8,49 +8,51 @@ import (
 type versionChange int
 
 const (
-	versionUpgrade   versionChange = -1
-	versionSame      versionChange = 0
-	versionDowngrade versionChange = 1
-	versionUnknown   versionChange = 2
+	VersionUpgrade   versionChange = -1
+	VersionSame      versionChange = 0
+	VersionDowngrade versionChange = 1
+	VersionUnknown   versionChange = 2
 )
 
 func (v versionChange) String() string {
 	switch v {
-	case versionUpgrade:
+	case VersionUpgrade:
 		return "upgrade"
-	case versionSame:
+	case VersionSame:
 		return "same"
-	case versionDowngrade:
+	case VersionDowngrade:
 		return "downgrade"
-	case versionUnknown:
+	case VersionUnknown:
 		return "unknown"
 	}
 	klog.Warningf("unhandled versionChange value %d", v)
 	return "UNHANDLED"
 }
 
-// compareVersions compares two semver versions
+// CompareVersions compares two semver versions
 // if fromVersion is older than toVersion, returns versionOlder
 // likewise, if fromVersion is newer, returns versionNewer
-func compareVersions(fromVersion, toVersion string) versionChange {
+func CompareVersions(fromVersion, toVersion string) versionChange {
 	if fromVersion == toVersion {
-		return versionSame
+		return VersionSame
 	}
 
 	v1, err := semver.NewVersion(fromVersion)
 	if err != nil {
-		return versionUnknown
+		return VersionUnknown
 	}
 
 	v2, err := semver.NewVersion(toVersion)
 	if err != nil {
-		return versionUnknown
+		return VersionUnknown
 	}
 
 	return versionChange(v1.Compare(v2))
 }
 
-func isVersionGreaterThanOrEqualTo(version string, major int, minor int) bool {
+// IsVersionGreaterThanOrEqualTo returns true if given version string
+// in greater than or equal to major.min version, otherwise returns false.
+func IsVersionGreaterThanOrEqualTo(version string, major int, minor int) bool {
 	v, err := semver.NewVersion(version)
 	if err != nil {
 		klog.Errorf("failed to parse version %s: %v", version, err)

pkg/network/semver_test.go renamed to pkg/version/semver_test.go

@@ -1,4 +1,4 @@
-package network
+package version
 
 import (
 	"strconv"
@@ -16,47 +16,47 @@ func TestDirection(t *testing.T) {
 		{
 			"1.2.3",
 			"1.2.4",
-			versionUpgrade,
+			VersionUpgrade,
 		},
 		{
 			"1.2.4",
 			"1.2.3",
-			versionDowngrade,
+			VersionDowngrade,
 		},
 		{
 			"asdf",
 			"fdsa",
-			versionUnknown,
+			VersionUnknown,
 		},
 		{
 			"1.1.1",
 			"1.1.1",
-			versionSame,
+			VersionSame,
 		},
 		{
 			"4.7.0-0.ci-2021-01-16-102811",
 			"4.7.0-0.ci-2021-01-18-121038",
-			versionUpgrade,
+			VersionUpgrade,
 		},
 		{
 			"4.7.0-0.ci-2021-01-18-121038",
 			"4.7.0-0.ci-2021-01-16-102811",
-			versionDowngrade,
+			VersionDowngrade,
 		},
 		{
 			"4.6.0-0.ci-2021-01-18-121038",
 			"4.7.0-0.ci-2021-01-16-102811",
-			versionUpgrade,
+			VersionUpgrade,
 		},
 		{
 			"4.6.5",
 			"4.7.0-0.ci-2021-01-16-102811",
-			versionUpgrade,
+			VersionUpgrade,
 		},
 	} {
 		t.Run(strconv.Itoa(idx), func(t *testing.T) {
 			g := NewGomegaWithT(t)
-			g.Expect(compareVersions(tc.from, tc.to)).To(Equal(tc.result))
+			g.Expect(CompareVersions(tc.from, tc.to)).To(Equal(tc.result))
 		})
 	}
 }
@@ -102,7 +102,7 @@ func TestVersionComparison(t *testing.T) {
 		t.Run(strconv.Itoa(idx), func(t *testing.T) {
 			g := NewGomegaWithT(t)
 
-			g.Expect(isVersionGreaterThanOrEqualTo(tc.version, tc.otherVersionMajor, tc.otherVersionMinor)).To(Equal(tc.resultGreaterThanOrEqualTo))
+			g.Expect(IsVersionGreaterThanOrEqualTo(tc.version, tc.otherVersionMajor, tc.otherVersionMinor)).To(Equal(tc.resultGreaterThanOrEqualTo))
 		})
 	}
 }