Skip to content

Commit 0084303

Browse files
Danil-GrigorevDanil-Grigorev
committed
Use kube-vip for e2e tests
Signed-off-by: Danil Grigorev <[email protected]>
1 parent 07d15d4 commit 0084303

File tree

8 files changed

+208
-39
lines changed

8 files changed

+208
-39
lines changed

‎pkg/rke2/workload_cluster.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ import (
3737
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3838
"sigs.k8s.io/cluster-api/util"
3939
"sigs.k8s.io/cluster-api/util/annotations"
40-
"sigs.k8s.io/cluster-api/util/certs"
4140
"sigs.k8s.io/cluster-api/util/collections"
4241
"sigs.k8s.io/cluster-api/util/conditions"
4342
"sigs.k8s.io/cluster-api/util/patch"
@@ -123,7 +122,7 @@ func (m *Management) NewWorkload(
123122
return nil, err
124123
}
125124

126-
if _, err := certs.DecodePrivateKeyPEM(etcdKeyPair.Key); err == nil {
125+
if !strings.Contains(string(etcdKeyPair.Key), "EC PRIVATE KEY") {
127126
clientKey, err := m.Tracker.GetEtcdClientCertificateKey(ctx, clusterKey)
128127
if err != nil {
129128
return nil, err

‎samples/docker/kube-vip/rke2-sample.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ spec:
1717
cidrBlocks:
1818
- 10.46.0.0/16
1919
serviceDomain: cluster.local
20+
controlPlaneEndpoint:
21+
host: "${REGISTRATION_VIP}"
22+
port: 6443
2023
controlPlaneRef:
2124
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
2225
kind: RKE2ControlPlane

‎test/e2e/data/infrastructure/cluster-template-docker-legacy.yaml

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
cidrBlocks:
7676
- 10.46.0.0/16
7777
serviceDomain: cluster.local
78+
controlPlaneEndpoint:
79+
host: "${REGISTRATION_VIP}"
80+
port: 6443
7881
controlPlaneRef:
7982
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
8083
kind: RKE2ControlPlane
@@ -111,7 +114,47 @@ spec:
111114
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
112115
kind: DockerMachineTemplate
113116
name: "${CLUSTER_NAME}-control-plane"
114-
nodeDrainTimeout: 2m
117+
nodeDrainTimeout: 30s
118+
registrationMethod: "address"
119+
registrationAddress: "${REGISTRATION_VIP}"
120+
preRKE2Commands:
121+
- mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
122+
files:
123+
- path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
124+
content: |
125+
apiVersion: v1
126+
kind: ServiceAccount
127+
metadata:
128+
name: kube-vip
129+
namespace: kube-system
130+
---
131+
apiVersion: rbac.authorization.k8s.io/v1
132+
kind: ClusterRole
133+
metadata:
134+
annotations:
135+
rbac.authorization.kubernetes.io/autoupdate: "true"
136+
name: system:kube-vip-role
137+
rules:
138+
- apiGroups: [""]
139+
resources: ["services", "services/status", "nodes", "endpoints"]
140+
verbs: ["list","get","watch", "update"]
141+
- apiGroups: ["coordination.k8s.io"]
142+
resources: ["leases"]
143+
verbs: ["list", "get", "watch", "update", "create"]
144+
---
145+
kind: ClusterRoleBinding
146+
apiVersion: rbac.authorization.k8s.io/v1
147+
metadata:
148+
name: system:kube-vip-binding
149+
roleRef:
150+
apiGroup: rbac.authorization.k8s.io
151+
kind: ClusterRole
152+
name: system:kube-vip-role
153+
subjects:
154+
- kind: ServiceAccount
155+
name: kube-vip
156+
namespace: kube-system
157+
owner: root:root
115158
---
116159
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
117160
kind: DockerMachineTemplate
@@ -121,6 +164,7 @@ spec:
121164
template:
122165
spec:
123166
customImage: kindest/node:${KIND_IMAGE_VERSION}
167+
bootstrapTimeout: 15m
124168
---
125169
apiVersion: cluster.x-k8s.io/v1beta1
126170
kind: MachineDeployment
@@ -154,6 +198,7 @@ spec:
154198
template:
155199
spec:
156200
customImage: kindest/node:${KIND_IMAGE_VERSION}
201+
bootstrapTimeout: 15m
157202
---
158203
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
159204
kind: RKE2ConfigTemplate

‎test/e2e/data/infrastructure/cluster-template-docker.yaml

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
cidrBlocks:
7676
- 10.46.0.0/16
7777
serviceDomain: cluster.local
78+
controlPlaneEndpoint:
79+
host: "${REGISTRATION_VIP}"
80+
port: 6443
7881
controlPlaneRef:
7982
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
8083
kind: RKE2ControlPlane
@@ -111,7 +114,47 @@ spec:
111114
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
112115
kind: DockerMachineTemplate
113116
name: "${CLUSTER_NAME}-control-plane"
114-
nodeDrainTimeout: 2m
117+
nodeDrainTimeout: 30s
118+
registrationMethod: "address"
119+
registrationAddress: "${REGISTRATION_VIP}"
120+
preRKE2Commands:
121+
- mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
122+
files:
123+
- path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
124+
content: |
125+
apiVersion: v1
126+
kind: ServiceAccount
127+
metadata:
128+
name: kube-vip
129+
namespace: kube-system
130+
---
131+
apiVersion: rbac.authorization.k8s.io/v1
132+
kind: ClusterRole
133+
metadata:
134+
annotations:
135+
rbac.authorization.kubernetes.io/autoupdate: "true"
136+
name: system:kube-vip-role
137+
rules:
138+
- apiGroups: [""]
139+
resources: ["services", "services/status", "nodes", "endpoints"]
140+
verbs: ["list","get","watch", "update"]
141+
- apiGroups: ["coordination.k8s.io"]
142+
resources: ["leases"]
143+
verbs: ["list", "get", "watch", "update", "create"]
144+
---
145+
kind: ClusterRoleBinding
146+
apiVersion: rbac.authorization.k8s.io/v1
147+
metadata:
148+
name: system:kube-vip-binding
149+
roleRef:
150+
apiGroup: rbac.authorization.k8s.io
151+
kind: ClusterRole
152+
name: system:kube-vip-role
153+
subjects:
154+
- kind: ServiceAccount
155+
name: kube-vip
156+
namespace: kube-system
157+
owner: root:root
115158
---
116159
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
117160
kind: DockerMachineTemplate
@@ -121,6 +164,7 @@ spec:
121164
template:
122165
spec:
123166
customImage: kindest/node:${KIND_IMAGE_VERSION}
167+
bootstrapTimeout: 15m
124168
---
125169
apiVersion: cluster.x-k8s.io/v1beta1
126170
kind: MachineDeployment
@@ -154,6 +198,7 @@ spec:
154198
template:
155199
spec:
156200
customImage: kindest/node:${KIND_IMAGE_VERSION}
201+
bootstrapTimeout: 15m
157202
---
158203
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
159204
kind: RKE2ConfigTemplate

‎test/e2e/e2e_test.go

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import (
2929
. "github.com/onsi/gomega"
3030

3131
corev1 "k8s.io/api/core/v1"
32-
"k8s.io/utils/pointer"
32+
"k8s.io/utils/ptr"
3333

3434
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
3535
"sigs.k8s.io/cluster-api/util"
@@ -44,6 +44,7 @@ var _ = Describe("Workload cluster creation", func() {
4444
result *ApplyClusterTemplateAndWaitResult
4545
clusterName string
4646
clusterctlLogFolder string
47+
registrationIP string
4748
)
4849

4950
BeforeEach(func() {
@@ -54,6 +55,8 @@ var _ = Describe("Workload cluster creation", func() {
5455

5556
Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
5657

58+
registrationIP = randomIp()
59+
5760
By("Initializing the bootstrap cluster")
5861
initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder)
5962

@@ -97,8 +100,11 @@ var _ = Describe("Workload cluster creation", func() {
97100
Namespace: namespace.Name,
98101
ClusterName: clusterName,
99102
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
100-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
101-
WorkerMachineCount: pointer.Int64Ptr(1),
103+
ControlPlaneMachineCount: ptr.To(int64(1)),
104+
WorkerMachineCount: ptr.To(int64(1)),
105+
ClusterctlVariables: map[string]string{
106+
"REGISTRATION_VIP": registrationIP,
107+
},
102108
},
103109
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
104110
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -118,8 +124,11 @@ var _ = Describe("Workload cluster creation", func() {
118124
Namespace: namespace.Name,
119125
ClusterName: clusterName,
120126
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
121-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
122-
WorkerMachineCount: pointer.Int64Ptr(3),
127+
ControlPlaneMachineCount: ptr.To(int64(1)),
128+
WorkerMachineCount: ptr.To(int64(3)),
129+
ClusterctlVariables: map[string]string{
130+
"REGISTRATION_VIP": registrationIP,
131+
},
123132
},
124133
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
125134
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -138,8 +147,11 @@ var _ = Describe("Workload cluster creation", func() {
138147
Namespace: namespace.Name,
139148
ClusterName: clusterName,
140149
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
141-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
142-
WorkerMachineCount: pointer.Int64Ptr(3),
150+
ControlPlaneMachineCount: ptr.To(int64(1)),
151+
WorkerMachineCount: ptr.To(int64(3)),
152+
ClusterctlVariables: map[string]string{
153+
"REGISTRATION_VIP": registrationIP,
154+
},
143155
},
144156
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
145157
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -171,8 +183,11 @@ var _ = Describe("Workload cluster creation", func() {
171183
Namespace: namespace.Name,
172184
ClusterName: clusterName,
173185
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
174-
ControlPlaneMachineCount: pointer.Int64Ptr(3),
175-
WorkerMachineCount: pointer.Int64Ptr(3),
186+
ControlPlaneMachineCount: ptr.To(int64(3)),
187+
WorkerMachineCount: ptr.To(int64(3)),
188+
ClusterctlVariables: map[string]string{
189+
"REGISTRATION_VIP": registrationIP,
190+
},
176191
},
177192
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
178193
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -197,8 +212,11 @@ var _ = Describe("Workload cluster creation", func() {
197212
Namespace: namespace.Name,
198213
ClusterName: clusterName,
199214
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
200-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
201-
WorkerMachineCount: pointer.Int64Ptr(3),
215+
ControlPlaneMachineCount: ptr.To(int64(1)),
216+
WorkerMachineCount: ptr.To(int64(3)),
217+
ClusterctlVariables: map[string]string{
218+
"REGISTRATION_VIP": registrationIP,
219+
},
202220
},
203221
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
204222
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -223,8 +241,11 @@ var _ = Describe("Workload cluster creation", func() {
223241
Namespace: namespace.Name,
224242
ClusterName: clusterName,
225243
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
226-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
227-
WorkerMachineCount: pointer.Int64Ptr(1),
244+
ControlPlaneMachineCount: ptr.To(int64(1)),
245+
WorkerMachineCount: ptr.To(int64(1)),
246+
ClusterctlVariables: map[string]string{
247+
"REGISTRATION_VIP": registrationIP,
248+
},
228249
},
229250
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
230251
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),

0 commit comments

Comments
 (0)