Prevent Machine rollout on provider upgrade

Signed-off-by: Andrea Mazzotti <[email protected]>

Author: anmazzotti

bootstrap/config/default/manager_pull_policy.yaml

@@ -8,4 +8,4 @@ spec:
     spec:
       containers:
       - name: manager
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent

controlplane/config/default/manager_pull_policy.yaml

@@ -8,4 +8,4 @@ spec:
     spec:
       containers:
       - name: manager
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent

test/e2e/e2e_upgrade_test.go

@@ -36,7 +36,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var _ = Describe("Workload cluster creation", func() {
+var _ = FDescribe("Workload cluster creation", func() {
 	var (
 		specName            = "create-workload-cluster"
 		namespace           *corev1.Namespace
@@ -114,6 +114,24 @@ var _ = Describe("Workload cluster creation", func() {
 				ControlPlane: client.ObjectKeyFromObject(result.ControlPlane),
 			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
 
+			WaitForClusterReady(ctx, WaitForClusterReadyInput{
+				Getter:    bootstrapClusterProxy.GetClient(),
+				Name:      result.Cluster.Name,
+				Namespace: result.Cluster.Namespace,
+			}, e2eConfig.GetIntervals(specName, "wait-cluster")...)
+
+			By("Fetching all Machines")
+			machineList := GetMachinesByCluster(ctx, GetMachinesByClusterInput{
+				Lister:      bootstrapClusterProxy.GetClient(),
+				ClusterName: result.Cluster.Name,
+				Namespace:   result.Cluster.Namespace,
+			})
+			Expect(machineList.Items).ShouldNot(BeEmpty(), "There must be at least one Machine")
+			machinesNames := []string{}
+			for _, machine := range machineList.Items {
+				machinesNames = append(machinesNames, machine.Name)
+			}
+
 			By("Upgrading to next boostrap/controlplane provider version")
 			UpgradeManagementCluster(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{
 				ClusterProxy:          bootstrapClusterProxy,
@@ -128,6 +146,26 @@ var _ = Describe("Workload cluster creation", func() {
 				ControlPlane: client.ObjectKeyFromObject(result.ControlPlane),
 			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
 
+			WaitForClusterReady(ctx, WaitForClusterReadyInput{
+				Getter:    bootstrapClusterProxy.GetClient(),
+				Name:      result.Cluster.Name,
+				Namespace: result.Cluster.Namespace,
+			}, e2eConfig.GetIntervals(specName, "wait-cluster"))
+
+			By("Verifying machine rollout did not happen")
+			updatedMachineList := GetMachinesByCluster(ctx, GetMachinesByClusterInput{
+				Lister:      bootstrapClusterProxy.GetClient(),
+				ClusterName: result.Cluster.Name,
+				Namespace:   result.Cluster.Namespace,
+			})
+			Expect(updatedMachineList.Items).ShouldNot(BeEmpty(), "There must be at least one Machine after provider upgrade")
+			updatedMachinesNames := []string{}
+			for _, machine := range updatedMachineList.Items {
+				updatedMachinesNames = append(updatedMachinesNames, machine.Name)
+			}
+			Expect(updatedMachinesNames).Should(ContainElements(machinesNames), "Machines should not have been rolled out after provider upgrade")
+			Expect(len(updatedMachinesNames)).Should(Equal(len(machinesNames)), "Number of Machines should match after provider upgrade")
+
 			By("Scaling down control plane to 2 and workers up to 2")
 			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
 				ClusterProxy: bootstrapClusterProxy,

test/e2e/helpers.go

@@ -36,6 +36,7 @@ import (
 
 	pkgerrors "github.com/pkg/errors"
 	controlplanev1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -302,6 +303,30 @@ func GetRKE2ControlPlaneByCluster(ctx context.Context, input GetRKE2ControlPlane
 	return nil
 }
 
+// GetMachinesByClusterInput is the input for GetRKE2ControlPlaneByCluster.
+type GetMachinesByClusterInput struct {
+	Lister      framework.Lister
+	ClusterName string
+	Namespace   string
+}
+
+// GetMachinesByCluster returns the Machine objects for a cluster.
+func GetMachinesByCluster(ctx context.Context, input GetMachinesByClusterInput) *clusterv1.MachineList {
+	opts := []client.ListOption{
+		client.InNamespace(input.Namespace),
+		client.MatchingLabels{
+			clusterv1.ClusterNameLabel: input.ClusterName,
+		},
+	}
+
+	machineList := &clusterv1.MachineList{}
+	Eventually(func() error {
+		return input.Lister.List(ctx, machineList, opts...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list Machine objects for Cluster %s", klog.KRef(input.Namespace, input.ClusterName))
+
+	return machineList
+}
+
 // WaitForControlPlaneAndMachinesReadyInput is the input type for WaitForControlPlaneAndMachinesReady.
 type WaitForControlPlaneAndMachinesReadyInput struct {
 	GetLister    framework.GetLister
@@ -488,6 +513,42 @@ func WaitForClusterToUpgrade(ctx context.Context, input WaitForClusterToUpgradeI
 	}, intervals...).Should(Succeed())
 }
 
+// WaitForClusterReadyInput is the input type for WaitForClusterReady.
+type WaitForClusterReadyInput struct {
+	Getter    framework.Getter
+	Name      string
+	Namespace string
+}
+
+// WaitForClusterReady will wait for a Cluster to be Ready.
+func WaitForClusterReady(ctx context.Context, input WaitForClusterReadyInput, intervals ...interface{}) {
+	By("Waiting for Cluster to be Ready")
+
+	Eventually(func() error {
+		cluster := &clusterv1.Cluster{}
+		key := types.NamespacedName{Name: input.Name, Namespace: input.Namespace}
+
+		if err := input.Getter.Get(ctx, key, cluster); err != nil {
+			return fmt.Errorf("getting Cluster %s/%s: %w", input.Namespace, input.Name, err)
+		}
+
+		readyCondition := conditions.Get(cluster, clusterv1.ReadyCondition)
+		if readyCondition == nil {
+			return fmt.Errorf("Cluster Ready condition is not found")
+		}
+
+		switch readyCondition.Status {
+		case corev1.ConditionTrue:
+			//Cluster is ready
+			return nil
+		case corev1.ConditionFalse:
+			return fmt.Errorf("Cluster is not Ready")
+		default:
+			return fmt.Errorf("Cluster Ready condition is unknown")
+		}
+	}, intervals...).Should(Succeed())
+}
+
 // setDefaults sets the default values for ApplyCustomClusterTemplateAndWaitInput if not set.
 // Currently, we set the default ControlPlaneWaiters here, which are implemented for RKE2ControlPlane.
 func setDefaults(input *ApplyCustomClusterTemplateAndWaitInput) {