Skip to content

Commit a066085

Browse files
Madhur97Madhur97
committed
cluster upgrade test
1 parent f691bcf commit a066085

File tree

2 files changed

+197
-0
lines changed

2 files changed

+197
-0
lines changed

‎.github/workflows/e2e.yml

+3
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,8 @@ jobs:
2525
- name: Set netfilter conntrack max
2626
run: sudo sysctl -w net.netfilter.nf_conntrack_max=131072
2727

28+
- name: Run K8s upgrade e2e tests
29+
run: yes | GINKGO_FOCUS="\[K8s-upgrade\]" make test-e2e
30+
2831
- name: Run PR-Blocking e2e tests
2932
run: yes | GINKGO_FOCUS="\[PR-Blocking\]" make test-e2e

‎test/e2e/cluster_upgrade_test.go

+194
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
// Copyright 2022 VMware, Inc. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// nolint: testpackage
5+
package e2e
6+
7+
import (
8+
"context"
9+
"fmt"
10+
"github.com/docker/docker/api/types"
11+
"github.com/docker/docker/client"
12+
. "github.com/onsi/ginkgo"
13+
. "github.com/onsi/gomega"
14+
corev1 "k8s.io/api/core/v1"
15+
"k8s.io/utils/pointer"
16+
"os"
17+
"path/filepath"
18+
"sigs.k8s.io/cluster-api/test/framework"
19+
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
20+
"sigs.k8s.io/cluster-api/util"
21+
)
22+
23+
var _ = Describe("Cluster upgrade test [K8s-upgrade]", func() {
24+
25+
var (
26+
ctx context.Context
27+
specName = "upgrade"
28+
namespace *corev1.Namespace
29+
cancelWatches context.CancelFunc
30+
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
31+
byoHostCapacityPool = 4
32+
byoHostName string
33+
dockerClient *client.Client
34+
allbyohostContainerIDs []string
35+
allAgentLogFiles []string
36+
kubernetesVersionUpgradeFrom = "v1.22.3"
37+
kubernetesVersionUpgradeTo = "v1.23.5"
38+
etcdUpgradeVersion = "3.5.1-0"
39+
coreDNSUpgradeVersion = "v1.8.6"
40+
)
41+
42+
BeforeEach(func() {
43+
ctx = context.TODO()
44+
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
45+
46+
Expect(e2eConfig).NotTo(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
47+
Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName)
48+
Expect(bootstrapClusterProxy).NotTo(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName)
49+
Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName)
50+
Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
51+
52+
// set up a Namespace where to host objects for this spec and create a watcher for the namespace events.
53+
namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
54+
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
55+
})
56+
57+
It("Should successfully upgrade cluster", func() {
58+
clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))
59+
var err error
60+
dockerClient, err = client.NewClientWithOpts(client.FromEnv)
61+
Expect(err).NotTo(HaveOccurred())
62+
63+
By("Creating byohost capacity pool containing 4 docker hosts")
64+
for i := 0; i < byoHostCapacityPool; i++ {
65+
66+
byoHostName = fmt.Sprintf("byohost-%s", util.RandomString(6))
67+
68+
runner := ByoHostRunner{
69+
Context: ctx,
70+
clusterConName: clusterConName,
71+
ByoHostName: byoHostName,
72+
Namespace: namespace.Name,
73+
PathToHostAgentBinary: pathToHostAgentBinary,
74+
DockerClient: dockerClient,
75+
NetworkInterface: "kind",
76+
bootstrapClusterProxy: bootstrapClusterProxy,
77+
CommandArgs: map[string]string{
78+
"--bootstrap-kubeconfig": "/bootstrap.conf",
79+
"--namespace": namespace.Name,
80+
"--v": "1",
81+
},
82+
}
83+
runner.BootstrapKubeconfigData = generateBootstrapKubeconfig(runner.Context, bootstrapClusterProxy, clusterConName)
84+
byohost, err := runner.SetupByoDockerHost()
85+
Expect(err).NotTo(HaveOccurred())
86+
output, byohostContainerID, err := runner.ExecByoDockerHost(byohost)
87+
allbyohostContainerIDs = append(allbyohostContainerIDs, byohostContainerID)
88+
Expect(err).NotTo(HaveOccurred())
89+
90+
// read the log of host agent container in backend, and write it
91+
agentLogFile := fmt.Sprintf("/tmp/host-agent-%d.log", i)
92+
93+
f := WriteDockerLog(output, agentLogFile)
94+
defer func() {
95+
deferredErr := f.Close()
96+
if deferredErr != nil {
97+
Showf("error closing file %s:, %v", agentLogFile, deferredErr)
98+
}
99+
}()
100+
allAgentLogFiles = append(allAgentLogFiles, agentLogFile)
101+
}
102+
103+
By("creating a workload cluster with one control plane node and one worker node")
104+
105+
setControlPlaneIP(context.Background(), dockerClient)
106+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
107+
ClusterProxy: bootstrapClusterProxy,
108+
ConfigCluster: clusterctl.ConfigClusterInput{
109+
LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()),
110+
ClusterctlConfigPath: clusterctlConfigPath,
111+
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
112+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
113+
Flavor: clusterctl.DefaultFlavor,
114+
Namespace: namespace.Name,
115+
ClusterName: clusterName,
116+
KubernetesVersion: kubernetesVersionUpgradeFrom,
117+
ControlPlaneMachineCount: pointer.Int64Ptr(1),
118+
WorkerMachineCount: pointer.Int64Ptr(1),
119+
},
120+
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
121+
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
122+
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
123+
}, clusterResources)
124+
125+
By("Upgrading the control plane")
126+
framework.UpgradeControlPlaneAndWaitForUpgrade(ctx, framework.UpgradeControlPlaneAndWaitForUpgradeInput{
127+
ClusterProxy: bootstrapClusterProxy,
128+
Cluster: clusterResources.Cluster,
129+
ControlPlane: clusterResources.ControlPlane,
130+
EtcdImageTag: etcdUpgradeVersion,
131+
DNSImageTag: coreDNSUpgradeVersion,
132+
KubernetesUpgradeVersion: kubernetesVersionUpgradeTo,
133+
WaitForMachinesToBeUpgraded: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"),
134+
WaitForKubeProxyUpgrade: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"),
135+
WaitForDNSUpgrade: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"),
136+
WaitForEtcdUpgrade: e2eConfig.GetIntervals(specName, "wait-machine-upgrade"),
137+
})
138+
139+
By("Upgrading the machine deployment")
140+
framework.UpgradeMachineDeploymentsAndWait(ctx, framework.UpgradeMachineDeploymentsAndWaitInput{
141+
ClusterProxy: bootstrapClusterProxy,
142+
Cluster: clusterResources.Cluster,
143+
UpgradeVersion: kubernetesVersionUpgradeTo,
144+
MachineDeployments: clusterResources.MachineDeployments,
145+
WaitForMachinesToBeUpgraded: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
146+
})
147+
148+
By("Waiting until nodes are ready")
149+
workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterResources.Cluster.Name)
150+
workloadClient := workloadProxy.GetClient()
151+
framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{
152+
Lister: workloadClient,
153+
KubernetesVersion: kubernetesVersionUpgradeTo,
154+
Count: int(clusterResources.ExpectedTotalNodes()),
155+
WaitForNodesReady: e2eConfig.GetIntervals(specName, "wait-nodes-ready"),
156+
})
157+
})
158+
JustAfterEach(func() {
159+
if CurrentGinkgoTestDescription().Failed {
160+
ShowInfo(allAgentLogFiles)
161+
}
162+
})
163+
164+
AfterEach(func() {
165+
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
166+
dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, clusterResources.Cluster, e2eConfig.GetIntervals, skipCleanup)
167+
168+
if dockerClient != nil {
169+
for _, byohostContainerID := range allbyohostContainerIDs {
170+
err := dockerClient.ContainerStop(ctx, byohostContainerID, nil)
171+
Expect(err).NotTo(HaveOccurred())
172+
173+
err = dockerClient.ContainerRemove(ctx, byohostContainerID, types.ContainerRemoveOptions{})
174+
Expect(err).NotTo(HaveOccurred())
175+
}
176+
177+
}
178+
179+
for _, agentLogFile := range allAgentLogFiles {
180+
err := os.Remove(agentLogFile)
181+
if err != nil {
182+
Showf("error removing file %s: %v", agentLogFile, err)
183+
}
184+
}
185+
err := os.Remove(ReadByohControllerManagerLogShellFile)
186+
if err != nil {
187+
Showf("error removing file %s: %v", ReadByohControllerManagerLogShellFile, err)
188+
}
189+
err = os.Remove(ReadAllPodsShellFile)
190+
if err != nil {
191+
Showf("error removing file %s: %v", ReadAllPodsShellFile, err)
192+
}
193+
})
194+
})

0 commit comments

Comments
 (0)