Skip to content

Commit d6d6c17

Browse files
Danil-GrigorevDanil-Grigorev
committed
Use kube-vip for e2e tests
Signed-off-by: Danil Grigorev <[email protected]>
1 parent 07d15d4 commit d6d6c17

File tree

10 files changed

+259
-68
lines changed

10 files changed

+259
-68
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ SKIP_CREATE_MGMT_CLUSTER ?= false
384384

385385
.PHONY: test-e2e-run
386386
test-e2e-run: $(GINKGO) $(KUSTOMIZE) e2e-image inotify-check ## Run the end-to-end tests
387-
CAPI_KUSTOMIZE_PATH="$(KUSTOMIZE)" time $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) \
387+
CAPI_KUSTOMIZE_PATH="$(KUSTOMIZE)" time $(GINKGO) -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) \
388388
--tags=e2e --focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) \
389389
--timeout=$(GINKGO_TIMEOUT) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
390390
-e2e.artifacts-folder="$(ARTIFACTS)" \

‎pkg/rke2/workload_cluster.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ import (
3737
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3838
"sigs.k8s.io/cluster-api/util"
3939
"sigs.k8s.io/cluster-api/util/annotations"
40-
"sigs.k8s.io/cluster-api/util/certs"
4140
"sigs.k8s.io/cluster-api/util/collections"
4241
"sigs.k8s.io/cluster-api/util/conditions"
4342
"sigs.k8s.io/cluster-api/util/patch"
@@ -123,7 +122,7 @@ func (m *Management) NewWorkload(
123122
return nil, err
124123
}
125124

126-
if _, err := certs.DecodePrivateKeyPEM(etcdKeyPair.Key); err == nil {
125+
if !strings.Contains(string(etcdKeyPair.Key), "EC PRIVATE KEY") {
127126
clientKey, err := m.Tracker.GetEtcdClientCertificateKey(ctx, clusterKey)
128127
if err != nil {
129128
return nil, err

‎samples/docker/kube-vip/rke2-sample.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ spec:
1717
cidrBlocks:
1818
- 10.46.0.0/16
1919
serviceDomain: cluster.local
20+
controlPlaneEndpoint:
21+
host: "${REGISTRATION_VIP}"
22+
port: 6443
2023
controlPlaneRef:
2124
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
2225
kind: RKE2ControlPlane

‎test/e2e/common.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,17 @@ func setupSpecNamespace(ctx context.Context, specName string, clusterProxy frame
6363
return namespace, cancelWatches
6464
}
6565

66+
func cleanupInstallation(ctx context.Context, clusterctlLogFolder, clusterctlConfigPath, kubeconfigPath string) func() {
67+
return func() {
68+
By("Removing existing installations")
69+
clusterctl.Delete(ctx, clusterctl.DeleteInput{
70+
LogFolder: clusterctlLogFolder,
71+
ClusterctlConfigPath: clusterctlConfigPath,
72+
KubeconfigPath: kubeconfigPath,
73+
})
74+
}
75+
}
76+
6677
type cleanupInput struct {
6778
SpecName string
6879
ClusterProxy framework.ClusterProxy

‎test/e2e/data/infrastructure/cluster-template-docker-legacy.yaml

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
cidrBlocks:
7676
- 10.46.0.0/16
7777
serviceDomain: cluster.local
78+
controlPlaneEndpoint:
79+
host: "${REGISTRATION_VIP}"
80+
port: 6443
7881
controlPlaneRef:
7982
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
8083
kind: RKE2ControlPlane
@@ -111,7 +114,51 @@ spec:
111114
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
112115
kind: DockerMachineTemplate
113116
name: "${CLUSTER_NAME}-control-plane"
114-
nodeDrainTimeout: 2m
117+
nodeDrainTimeout: 30s
118+
registrationMethod: "address"
119+
registrationAddress: "${REGISTRATION_VIP}"
120+
rolloutStrategy:
121+
type: "RollingUpdate"
122+
rollingUpdate:
123+
maxSurge: 3
124+
preRKE2Commands:
125+
- mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
126+
files:
127+
- path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
128+
content: |
129+
apiVersion: v1
130+
kind: ServiceAccount
131+
metadata:
132+
name: kube-vip
133+
namespace: kube-system
134+
---
135+
apiVersion: rbac.authorization.k8s.io/v1
136+
kind: ClusterRole
137+
metadata:
138+
annotations:
139+
rbac.authorization.kubernetes.io/autoupdate: "true"
140+
name: system:kube-vip-role
141+
rules:
142+
- apiGroups: [""]
143+
resources: ["services", "services/status", "nodes", "endpoints"]
144+
verbs: ["list","get","watch", "update"]
145+
- apiGroups: ["coordination.k8s.io"]
146+
resources: ["leases"]
147+
verbs: ["list", "get", "watch", "update", "create"]
148+
---
149+
kind: ClusterRoleBinding
150+
apiVersion: rbac.authorization.k8s.io/v1
151+
metadata:
152+
name: system:kube-vip-binding
153+
roleRef:
154+
apiGroup: rbac.authorization.k8s.io
155+
kind: ClusterRole
156+
name: system:kube-vip-role
157+
subjects:
158+
- kind: ServiceAccount
159+
name: kube-vip
160+
namespace: kube-system
161+
owner: root:root
115162
---
116163
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
117164
kind: DockerMachineTemplate
@@ -121,6 +168,7 @@ spec:
121168
template:
122169
spec:
123170
customImage: kindest/node:${KIND_IMAGE_VERSION}
171+
bootstrapTimeout: 15m
124172
---
125173
apiVersion: cluster.x-k8s.io/v1beta1
126174
kind: MachineDeployment
@@ -154,6 +202,7 @@ spec:
154202
template:
155203
spec:
156204
customImage: kindest/node:${KIND_IMAGE_VERSION}
205+
bootstrapTimeout: 15m
157206
---
158207
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
159208
kind: RKE2ConfigTemplate

‎test/e2e/data/infrastructure/cluster-template-docker.yaml

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
cidrBlocks:
7676
- 10.46.0.0/16
7777
serviceDomain: cluster.local
78+
controlPlaneEndpoint:
79+
host: "${REGISTRATION_VIP}"
80+
port: 6443
7881
controlPlaneRef:
7982
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
8083
kind: RKE2ControlPlane
@@ -111,7 +114,51 @@ spec:
111114
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
112115
kind: DockerMachineTemplate
113116
name: "${CLUSTER_NAME}-control-plane"
114-
nodeDrainTimeout: 2m
117+
nodeDrainTimeout: 30s
118+
registrationMethod: "address"
119+
registrationAddress: "${REGISTRATION_VIP}"
120+
rolloutStrategy:
121+
type: "RollingUpdate"
122+
rollingUpdate:
123+
maxSurge: 3
124+
preRKE2Commands:
125+
- mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
126+
files:
127+
- path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
128+
content: |
129+
apiVersion: v1
130+
kind: ServiceAccount
131+
metadata:
132+
name: kube-vip
133+
namespace: kube-system
134+
---
135+
apiVersion: rbac.authorization.k8s.io/v1
136+
kind: ClusterRole
137+
metadata:
138+
annotations:
139+
rbac.authorization.kubernetes.io/autoupdate: "true"
140+
name: system:kube-vip-role
141+
rules:
142+
- apiGroups: [""]
143+
resources: ["services", "services/status", "nodes", "endpoints"]
144+
verbs: ["list","get","watch", "update"]
145+
- apiGroups: ["coordination.k8s.io"]
146+
resources: ["leases"]
147+
verbs: ["list", "get", "watch", "update", "create"]
148+
---
149+
kind: ClusterRoleBinding
150+
apiVersion: rbac.authorization.k8s.io/v1
151+
metadata:
152+
name: system:kube-vip-binding
153+
roleRef:
154+
apiGroup: rbac.authorization.k8s.io
155+
kind: ClusterRole
156+
name: system:kube-vip-role
157+
subjects:
158+
- kind: ServiceAccount
159+
name: kube-vip
160+
namespace: kube-system
161+
owner: root:root
115162
---
116163
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
117164
kind: DockerMachineTemplate
@@ -121,6 +168,7 @@ spec:
121168
template:
122169
spec:
123170
customImage: kindest/node:${KIND_IMAGE_VERSION}
171+
bootstrapTimeout: 15m
124172
---
125173
apiVersion: cluster.x-k8s.io/v1beta1
126174
kind: MachineDeployment
@@ -154,6 +202,7 @@ spec:
154202
template:
155203
spec:
156204
customImage: kindest/node:${KIND_IMAGE_VERSION}
205+
bootstrapTimeout: 15m
157206
---
158207
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
159208
kind: RKE2ConfigTemplate

‎test/e2e/e2e_test.go

Lines changed: 43 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import (
2929
. "github.com/onsi/gomega"
3030

3131
corev1 "k8s.io/api/core/v1"
32-
"k8s.io/utils/pointer"
32+
"k8s.io/utils/ptr"
3333

3434
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
3535
"sigs.k8s.io/cluster-api/util"
@@ -44,6 +44,7 @@ var _ = Describe("Workload cluster creation", func() {
4444
result *ApplyClusterTemplateAndWaitResult
4545
clusterName string
4646
clusterctlLogFolder string
47+
registrationIP string
4748
)
4849

4950
BeforeEach(func() {
@@ -54,6 +55,8 @@ var _ = Describe("Workload cluster creation", func() {
5455

5556
Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
5657

58+
registrationIP = randomIp()
59+
5760
By("Initializing the bootstrap cluster")
5861
initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder)
5962

@@ -70,14 +73,15 @@ var _ = Describe("Workload cluster creation", func() {
7073

7174
AfterEach(func() {
7275
cleanInput := cleanupInput{
73-
SpecName: specName,
74-
Cluster: result.Cluster,
75-
ClusterProxy: bootstrapClusterProxy,
76-
Namespace: namespace,
77-
CancelWatches: cancelWatches,
78-
IntervalsGetter: e2eConfig.GetIntervals,
79-
SkipCleanup: skipCleanup,
80-
ArtifactFolder: artifactFolder,
76+
SpecName: specName,
77+
Cluster: result.Cluster,
78+
ClusterProxy: bootstrapClusterProxy,
79+
Namespace: namespace,
80+
CancelWatches: cancelWatches,
81+
IntervalsGetter: e2eConfig.GetIntervals,
82+
SkipCleanup: skipCleanup,
83+
ArtifactFolder: artifactFolder,
84+
AdditionalCleanup: cleanupInstallation(ctx, clusterctlLogFolder, clusterctlConfigPath, bootstrapClusterProxy.GetKubeconfigPath()),
8185
}
8286

8387
dumpSpecResourcesAndCleanup(ctx, cleanInput)
@@ -97,8 +101,11 @@ var _ = Describe("Workload cluster creation", func() {
97101
Namespace: namespace.Name,
98102
ClusterName: clusterName,
99103
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
100-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
101-
WorkerMachineCount: pointer.Int64Ptr(1),
104+
ControlPlaneMachineCount: ptr.To(int64(1)),
105+
WorkerMachineCount: ptr.To(int64(1)),
106+
ClusterctlVariables: map[string]string{
107+
"REGISTRATION_VIP": registrationIP,
108+
},
102109
},
103110
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
104111
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -118,8 +125,11 @@ var _ = Describe("Workload cluster creation", func() {
118125
Namespace: namespace.Name,
119126
ClusterName: clusterName,
120127
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
121-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
122-
WorkerMachineCount: pointer.Int64Ptr(3),
128+
ControlPlaneMachineCount: ptr.To(int64(1)),
129+
WorkerMachineCount: ptr.To(int64(3)),
130+
ClusterctlVariables: map[string]string{
131+
"REGISTRATION_VIP": registrationIP,
132+
},
123133
},
124134
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
125135
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -138,8 +148,11 @@ var _ = Describe("Workload cluster creation", func() {
138148
Namespace: namespace.Name,
139149
ClusterName: clusterName,
140150
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
141-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
142-
WorkerMachineCount: pointer.Int64Ptr(3),
151+
ControlPlaneMachineCount: ptr.To(int64(1)),
152+
WorkerMachineCount: ptr.To(int64(3)),
153+
ClusterctlVariables: map[string]string{
154+
"REGISTRATION_VIP": registrationIP,
155+
},
143156
},
144157
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
145158
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -171,8 +184,11 @@ var _ = Describe("Workload cluster creation", func() {
171184
Namespace: namespace.Name,
172185
ClusterName: clusterName,
173186
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
174-
ControlPlaneMachineCount: pointer.Int64Ptr(3),
175-
WorkerMachineCount: pointer.Int64Ptr(3),
187+
ControlPlaneMachineCount: ptr.To(int64(3)),
188+
WorkerMachineCount: ptr.To(int64(3)),
189+
ClusterctlVariables: map[string]string{
190+
"REGISTRATION_VIP": registrationIP,
191+
},
176192
},
177193
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
178194
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -197,8 +213,11 @@ var _ = Describe("Workload cluster creation", func() {
197213
Namespace: namespace.Name,
198214
ClusterName: clusterName,
199215
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
200-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
201-
WorkerMachineCount: pointer.Int64Ptr(3),
216+
ControlPlaneMachineCount: ptr.To(int64(1)),
217+
WorkerMachineCount: ptr.To(int64(3)),
218+
ClusterctlVariables: map[string]string{
219+
"REGISTRATION_VIP": registrationIP,
220+
},
202221
},
203222
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
204223
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -223,8 +242,11 @@ var _ = Describe("Workload cluster creation", func() {
223242
Namespace: namespace.Name,
224243
ClusterName: clusterName,
225244
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
226-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
227-
WorkerMachineCount: pointer.Int64Ptr(1),
245+
ControlPlaneMachineCount: ptr.To(int64(1)),
246+
WorkerMachineCount: ptr.To(int64(1)),
247+
ClusterctlVariables: map[string]string{
248+
"REGISTRATION_VIP": registrationIP,
249+
},
228250
},
229251
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
230252
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),

0 commit comments

Comments
 (0)