✨ Add CAPI API version upgrade test

Makefile

@@ -267,7 +267,14 @@ generate-modules: ## Run go mod tidy to ensure modules are up to date
 DOCKER_TEMPLATES := test/e2e/data/addons-helm
 
 .PHONY: generate-e2e-templates
-generate-e2e-templates: $(KUSTOMIZE) ## Generate templates for e2e tests
+generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.5 main) ## Generate cluster templates for all versions
+
+.PHONY: generate-e2e-templates-v1.5
+generate-e2e-templates-v1.5: $(KUSTOMIZE)
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.5/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.5/cluster-template.yaml
+
+.PHONY: generate-e2e-templates-main
+generate-e2e-templates-main: $(KUSTOMIZE) ## Generate templates for e2e tests on main branch.
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template.yaml
 
 .PHONY: generate-flavors

test/e2e/capi_test.go

@@ -21,6 +21,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 
 	. "github.com/onsi/ginkgo/v2"
 	"k8s.io/utils/ptr"
@@ -49,4 +50,32 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 			}
 		})
 	})
+
+	Context("API Version Upgrade", func() {
+
+		Context("upgrade from an old version of v1beta1 to current, and scale workload clusters created in the old version", func() {
+
+			capi_e2e.ClusterctlUpgradeSpec(context.TODO(), func() capi_e2e.ClusterctlUpgradeSpecInput {
+				return capi_e2e.ClusterctlUpgradeSpecInput{
+					E2EConfig:                 e2eConfig,
+					ClusterctlConfigPath:      clusterctlConfigPath,
+					BootstrapClusterProxy:     bootstrapClusterProxy,
+					ArtifactFolder:            artifactFolder,
+					SkipCleanup:               skipCleanup,
+					InitWithProvidersContract: "v1beta1",
+					ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
+						WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
+					},
+					InitWithKubernetesVersion:       e2eConfig.GetVariable(KubernetesVersionAPIUpgradeFrom),
+					InitWithBinary:                  fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/releases/download/%s/clusterctl-{OS}-{ARCH}", e2eConfig.GetVariable(OldCAPIUpgradeVersion)),
+					InitWithCoreProvider:            "cluster-api:" + e2eConfig.GetVariable(OldCAPIUpgradeVersion),
+					InitWithBootstrapProviders:      []string{"kubeadm:" + e2eConfig.GetVariable(OldCAPIUpgradeVersion)},
+					InitWithControlPlaneProviders:   []string{"kubeadm:" + e2eConfig.GetVariable(OldCAPIUpgradeVersion)},
+					InitWithInfrastructureProviders: []string{"docker:" + e2eConfig.GetVariable(OldCAPIUpgradeVersion)},
+					InitWithAddonProviders:          []string{"helm:" + e2eConfig.GetVariable(OldProviderUpgradeVersion)},
+				}
+			})
+		})
+	})
+
 })

test/e2e/common.go

@@ -32,6 +32,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const (
+	kubesystem = "kube-system"
+)
+
 // EnsureControlPlaneInitialized waits for the cluster KubeadmControlPlane object to be initialized
 // and then installs cloud-provider-azure components via Helm.
 // Fulfills the clusterctl.Waiter type so that it can be used as ApplyClusterTemplateAndWaitInput data

test/e2e/config/helm.yaml

@@ -17,6 +17,15 @@ providers:
 - name: cluster-api
   type: CoreProvider
   versions:
+  - name: v1.5.0 # latest patch of earliest minor in supported v1beta1 releases; this is used for v1beta1 old --> v1beta1 latest clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.0/core-components.yaml"
+    type: "url"
+    contract: v1beta1
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "../data/shared/v1beta1/metadata.yaml"
   - name: v1.6.0
     value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/core-components.yaml
     type: url
@@ -31,6 +40,15 @@ providers:
 - name: kubeadm
   type: BootstrapProvider
   versions:
+  - name: v1.5.0 # latest patch of earliest minor in supported v1beta1 releases; this is used for v1beta1 old --> v1beta1 latest clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.0/bootstrap-components.yaml"
+    type: "url"
+    contract: v1beta1
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "../data/shared/v1beta1/metadata.yaml"
   - name: v1.6.0
     value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/bootstrap-components.yaml
     type: url
@@ -44,6 +62,15 @@ providers:
 - name: kubeadm
   type: ControlPlaneProvider
   versions:
+  - name: v1.5.0 # latest patch of earliest minor in supported v1beta1 releases; this is used for v1beta1 old --> v1beta1 latest clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.0/control-plane-components.yaml"
+    type: "url"
+    contract: v1beta1
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "../data/shared/v1beta1/metadata.yaml"
   - name: v1.6.0
     value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/control-plane-components.yaml
     type: url
@@ -57,6 +84,17 @@ providers:
 - name: docker
   type: InfrastructureProvider
   versions:
+  - name: v1.5.0 # latest patch of earliest minor in supported v1beta1 releases; this is used for v1beta1 old --> v1beta1 latest clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.0/infrastructure-components-development.yaml"
+    type: "url"
+    contract: v1beta1
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "${PWD}/test/e2e/data/shared/v1beta1/metadata.yaml"
+    - sourcePath: "${PWD}/test/e2e/data/addons-helm/v1.5/cluster-template.yaml"
+      targetName: "cluster-template.yaml"
   - name: "v1.6.0" # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/infrastructure-components-development.yaml"
     type: "url"
@@ -71,6 +109,17 @@ providers:
 - name: helm
   type: AddonProvider
   versions:
+  - name: v0.1.1-alpha.0 # latest patch of earliest minor in supported v1beta1 releases; this is used for v1beta1 old --> v1beta1 latest clusterctl upgrades test only.
+    value: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.1.1-alpha.0/addon-components.yaml
+    type: "url"
+    contract: v1beta1
+    files:
+    - sourcePath: "${PWD}/test/e2e/data/shared/v1beta1-provider/metadata.yaml"
+    replacements:
+    - old: "imagePullPolicy: Always"
+      new: "imagePullPolicy: IfNotPresent"
+    - old: "--v=0"
+      new: "--v=2"
   - name: v0.2.99 # "vNext"; use manifests from local source files
     value: "${PWD}/config/default"
     contract: v1beta1
@@ -91,6 +140,7 @@ variables:
   KUBERNETES_VERSION: "v1.29.0"
   KUBERNETES_VERSION_UPGRADE_FROM: "v1.28.0"
   KUBERNETES_VERSION_UPGRADE_TO: "v1.29.0"
+  KUBERNETES_VERSION_API_UPGRADE_FROM: "v1.27.3"
   ETCD_VERSION_UPGRADE_TO: "3.5.10-0"
   COREDNS_VERSION_UPGRADE_TO: "v1.11.1"
   DOCKER_SERVICE_DOMAIN: "cluster.local"
@@ -112,7 +162,9 @@ variables:
   EXP_MACHINE_SET_PREFLIGHT_CHECKS: "true"
   CAPI_DIAGNOSTICS_ADDRESS: ":8080"
   CAPI_INSECURE_DIAGNOSTICS: "true"
-
+  OLD_CAPI_UPGRADE_VERSION: "v1.5.0"
+  OLD_PROVIDER_UPGRADE_VERSION: "v0.1.1-alpha.0"
+
 intervals:
   default/wait-controllers: ["3m", "10s"]
   default/wait-cluster: ["5m", "10s"]

test/e2e/data/addons-helm/v1.5/bases/calico.yaml

@@ -0,0 +1,31 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: calico
+spec:
+  clusterSelector:
+    matchLabels:
+      cni: calico
+  repoURL: https://docs.tigera.io/calico/charts
+  chartName: tigera-operator
+  # version: ${CALICO_VERSION}
+  releaseName: projectcalico
+  namespace: tigera-operator
+  valuesTemplate: |
+    installation:
+      cni:
+        type: Calico
+      calicoNetwork:
+        bgp: Disabled
+        mtu: 1350
+        ipPools:
+        ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
+        - cidr: {{ $cidr }}
+          encapsulation: VXLAN{{end}}
+      registry: mcr.microsoft.com/oss
+    # Image and registry configuration for the tigera/operator pod.
+    tigeraOperator:
+      image: tigera/operator
+      registry: mcr.microsoft.com/oss
+    calicoctl:
+      image: mcr.microsoft.com/oss/calico/ctl

test/e2e/data/addons-helm/v1.5/bases/cluster-with-kcp.yaml

@@ -0,0 +1,91 @@
+---
+# DockerCluster object referenced by the Cluster object
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: '${CLUSTER_NAME}'
+spec:
+  failureDomains:
+    fd1:
+      controlPlane: true
+    fd2:
+      controlPlane: true
+    fd3:
+      controlPlane: true
+    fd4:
+      controlPlane: false
+    fd5:
+      controlPlane: false
+    fd6:
+      controlPlane: false
+    fd7:
+      controlPlane: false
+    fd8:
+      controlPlane: false
+---
+# Cluster object with
+# - Reference to the KubeadmControlPlane object
+# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+  labels:
+    cni: "calico"
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
+    pods:
+      cidrBlocks: ['${DOCKER_POD_CIDRS}']
+    serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: '${CLUSTER_NAME}'
+  controlPlaneRef:
+    kind: KubeadmControlPlane
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    name: "${CLUSTER_NAME}-control-plane"
+---
+# DockerMachineTemplate object referenced by the KubeadmControlPlane object
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  template:
+    spec:
+      extraMounts:
+        - containerPath: "/var/run/docker.sock"
+          hostPath: "/var/run/docker.sock"
+      # The DOCKER_PRELOAD_IMAGES variable gets set in self-hosted E2E tests to the list of images of the E2E configuration.
+      preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
+---
+# KubeadmControlPlane referenced by the Cluster object with
+# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+  labels:
+    kcp-adoption.step2: ""
+spec:
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  machineTemplate:
+    infrastructureRef:
+      kind: DockerMachineTemplate
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      name: "${CLUSTER_NAME}-control-plane"
+  kubeadmConfigSpec:
+    clusterConfiguration:
+      controllerManager:
+        extraArgs: {enable-hostpath-provisioner: 'true'}
+      apiServer:
+        # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
+        certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
+    initConfiguration:
+      nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
+    joinConfiguration:
+      nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
+  version: "${KUBERNETES_VERSION}"

test/e2e/data/addons-helm/v1.5/bases/md.yaml

@@ -0,0 +1,51 @@
+---
+# DockerMachineTemplate referenced by the MachineDeployment and with
+# - extraMounts for the docker sock, thus allowing self-hosting test
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      extraMounts:
+        - containerPath: "/var/run/docker.sock"
+          hostPath: "/var/run/docker.sock"
+      # The DOCKER_PRELOAD_IMAGES variable gets set in self-hosted E2E tests to the list of images of the E2E configuration.
+      preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
+---
+# KubeadmConfigTemplate referenced by the MachineDeployment
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
+---
+# MachineDeployment object
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+  template:
+    spec:
+      clusterName: "${CLUSTER_NAME}"
+      version: "${KUBERNETES_VERSION}"
+      bootstrap:
+        configRef:
+          name: "${CLUSTER_NAME}-md-0"
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: "${CLUSTER_NAME}-md-0"
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+      failureDomain: fd4