Merge pull request #615 from anmazzotti/prevent_unwanted_rollout

Add e2e test to prevent Machine rollout on provider upgrade

Author: furkatgofurov7

test/e2e/e2e_upgrade_test.go

@@ -24,6 +24,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -115,6 +117,24 @@ var _ = Describe("Workload cluster creation", func() {
 				ControlPlane: client.ObjectKeyFromObject(result.ControlPlane),
 			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
 
+			WaitForClusterReady(ctx, WaitForClusterReadyInput{
+				Getter:    bootstrapClusterProxy.GetClient(),
+				Name:      result.Cluster.Name,
+				Namespace: result.Cluster.Namespace,
+			}, e2eConfig.GetIntervals(specName, "wait-cluster")...)
+
+			By("Fetching all Machines")
+			machineList := GetMachinesByCluster(ctx, GetMachinesByClusterInput{
+				Lister:      bootstrapClusterProxy.GetClient(),
+				ClusterName: result.Cluster.Name,
+				Namespace:   result.Cluster.Namespace,
+			})
+			Expect(machineList.Items).ShouldNot(BeEmpty(), "There must be at least one Machine")
+			machinesNames := []string{}
+			for _, machine := range machineList.Items {
+				machinesNames = append(machinesNames, machine.Name)
+			}
+
 			By("Upgrading to next boostrap/controlplane provider version")
 			UpgradeManagementCluster(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{
 				ClusterProxy:          bootstrapClusterProxy,
@@ -129,6 +149,41 @@ var _ = Describe("Workload cluster creation", func() {
 				ControlPlane: client.ObjectKeyFromObject(result.ControlPlane),
 			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
 
+			WaitForClusterReady(ctx, WaitForClusterReadyInput{
+				Getter:    bootstrapClusterProxy.GetClient(),
+				Name:      result.Cluster.Name,
+				Namespace: result.Cluster.Namespace,
+			}, e2eConfig.GetIntervals(specName, "wait-cluster")...)
+
+			By("Verifying machine rollout did not happen")
+			Consistently(func() error {
+				updatedMachineList := GetMachinesByCluster(ctx, GetMachinesByClusterInput{
+					Lister:      bootstrapClusterProxy.GetClient(),
+					ClusterName: result.Cluster.Name,
+					Namespace:   result.Cluster.Namespace,
+				})
+				if len(updatedMachineList.Items) == 0 {
+					return fmt.Errorf("There must be at least one Machine after provider upgrade")
+				}
+				updatedMachinesNames := []string{}
+				for _, machine := range updatedMachineList.Items {
+					updatedMachinesNames = append(updatedMachinesNames, machine.Name)
+				}
+				sameMachines, err := ContainElements(machinesNames).Match(updatedMachinesNames)
+				if err != nil {
+					return fmt.Errorf("matching machines: %w", err)
+				}
+				if !sameMachines {
+					fmt.Printf("Pre-upgrade machines: [%s]\n", strings.Join(machinesNames, ","))
+					fmt.Printf("Post-upgrade machines: [%s]\n", strings.Join(updatedMachinesNames, ","))
+					return fmt.Errorf("Machines should not have been rolled out after provider upgrade")
+				}
+				if len(updatedMachinesNames) != len(machinesNames) {
+					return fmt.Errorf("Number of Machines '%d' should match after provider upgrade '%d'", len(machinesNames), len(updatedMachinesNames))
+				}
+				return nil
+			}).WithTimeout(2 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
 			By("Scaling down control plane to 2 and workers up to 2")
 			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
 				ClusterProxy: bootstrapClusterProxy,

test/e2e/helpers.go

@@ -36,6 +36,7 @@ import (
 
 	pkgerrors "github.com/pkg/errors"
 	controlplanev1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -305,6 +306,30 @@ func GetRKE2ControlPlaneByCluster(ctx context.Context, input GetRKE2ControlPlane
 	return nil
 }
 
+// GetMachinesByClusterInput is the input for GetRKE2ControlPlaneByCluster.
+type GetMachinesByClusterInput struct {
+	Lister      framework.Lister
+	ClusterName string
+	Namespace   string
+}
+
+// GetMachinesByCluster returns the Machine objects for a cluster.
+func GetMachinesByCluster(ctx context.Context, input GetMachinesByClusterInput) *clusterv1.MachineList {
+	opts := []client.ListOption{
+		client.InNamespace(input.Namespace),
+		client.MatchingLabels{
+			clusterv1.ClusterNameLabel: input.ClusterName,
+		},
+	}
+
+	machineList := &clusterv1.MachineList{}
+	Eventually(func() error {
+		return input.Lister.List(ctx, machineList, opts...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list Machine objects for Cluster %s", klog.KRef(input.Namespace, input.ClusterName))
+
+	return machineList
+}
+
 // WaitForControlPlaneAndMachinesReadyInput is the input type for WaitForControlPlaneAndMachinesReady.
 type WaitForControlPlaneAndMachinesReadyInput struct {
 	GetLister             framework.GetLister
@@ -509,6 +534,42 @@ func WaitForClusterToUpgrade(ctx context.Context, input WaitForClusterToUpgradeI
 	}, intervals...).Should(Succeed())
 }
 
+// WaitForClusterReadyInput is the input type for WaitForClusterReady.
+type WaitForClusterReadyInput struct {
+	Getter    framework.Getter
+	Name      string
+	Namespace string
+}
+
+// WaitForClusterReady will wait for a Cluster to be Ready.
+func WaitForClusterReady(ctx context.Context, input WaitForClusterReadyInput, intervals ...interface{}) {
+	By("Waiting for Cluster to be Ready")
+
+	Eventually(func() error {
+		cluster := &clusterv1.Cluster{}
+		key := types.NamespacedName{Name: input.Name, Namespace: input.Namespace}
+
+		if err := input.Getter.Get(ctx, key, cluster); err != nil {
+			return fmt.Errorf("getting Cluster %s/%s: %w", input.Namespace, input.Name, err)
+		}
+
+		readyCondition := conditions.Get(cluster, clusterv1.ReadyCondition)
+		if readyCondition == nil {
+			return fmt.Errorf("Cluster Ready condition is not found")
+		}
+
+		switch readyCondition.Status {
+		case corev1.ConditionTrue:
+			//Cluster is ready
+			return nil
+		case corev1.ConditionFalse:
+			return fmt.Errorf("Cluster is not Ready")
+		default:
+			return fmt.Errorf("Cluster Ready condition is unknown")
+		}
+	}, intervals...).Should(Succeed())
+}
+
 // setDefaults sets the default values for ApplyCustomClusterTemplateAndWaitInput if not set.
 // Currently, we set the default ControlPlaneWaiters here, which are implemented for RKE2ControlPlane.
 func setDefaults(input *ApplyCustomClusterTemplateAndWaitInput) {