Skip to content

Commit e1a0dc7

Browse files
Danil-GrigorevDanil-Grigorev
committed
Use kube-vip for e2e tests
Signed-off-by: Danil Grigorev <[email protected]>
1 parent 07d15d4 commit e1a0dc7

File tree

7 files changed

+129
-4
lines changed

7 files changed

+129
-4
lines changed

‎pkg/rke2/workload_cluster.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ import (
3737
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3838
"sigs.k8s.io/cluster-api/util"
3939
"sigs.k8s.io/cluster-api/util/annotations"
40-
"sigs.k8s.io/cluster-api/util/certs"
4140
"sigs.k8s.io/cluster-api/util/collections"
4241
"sigs.k8s.io/cluster-api/util/conditions"
4342
"sigs.k8s.io/cluster-api/util/patch"
@@ -123,7 +122,7 @@ func (m *Management) NewWorkload(
123122
return nil, err
124123
}
125124

126-
if _, err := certs.DecodePrivateKeyPEM(etcdKeyPair.Key); err == nil {
125+
if !strings.Contains(string(etcdKeyPair.Key), "EC PRIVATE KEY") {
127126
clientKey, err := m.Tracker.GetEtcdClientCertificateKey(ctx, clusterKey)
128127
if err != nil {
129128
return nil, err

‎samples/docker/kube-vip/rke2-sample.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ spec:
1717
cidrBlocks:
1818
- 10.46.0.0/16
1919
serviceDomain: cluster.local
20+
controlPlaneEndpoint:
21+
host: "${REGISTRATION_VIP}"
22+
port: 6443
2023
controlPlaneRef:
2124
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
2225
kind: RKE2ControlPlane

‎test/e2e/data/infrastructure/cluster-template-docker-legacy.yaml

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
cidrBlocks:
7676
- 10.46.0.0/16
7777
serviceDomain: cluster.local
78+
controlPlaneEndpoint:
79+
host: "${REGISTRATION_VIP}"
80+
port: 6443
7881
controlPlaneRef:
7982
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
8083
kind: RKE2ControlPlane
@@ -111,7 +114,47 @@ spec:
111114
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
112115
kind: DockerMachineTemplate
113116
name: "${CLUSTER_NAME}-control-plane"
114-
nodeDrainTimeout: 2m
117+
nodeDrainTimeout: 30s
118+
registrationMethod: "address"
119+
registrationAddress: "${REGISTRATION_VIP}"
120+
preRKE2Commands:
121+
- mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
122+
files:
123+
- path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
124+
content: |
125+
apiVersion: v1
126+
kind: ServiceAccount
127+
metadata:
128+
name: kube-vip
129+
namespace: kube-system
130+
---
131+
apiVersion: rbac.authorization.k8s.io/v1
132+
kind: ClusterRole
133+
metadata:
134+
annotations:
135+
rbac.authorization.kubernetes.io/autoupdate: "true"
136+
name: system:kube-vip-role
137+
rules:
138+
- apiGroups: [""]
139+
resources: ["services", "services/status", "nodes", "endpoints"]
140+
verbs: ["list","get","watch", "update"]
141+
- apiGroups: ["coordination.k8s.io"]
142+
resources: ["leases"]
143+
verbs: ["list", "get", "watch", "update", "create"]
144+
---
145+
kind: ClusterRoleBinding
146+
apiVersion: rbac.authorization.k8s.io/v1
147+
metadata:
148+
name: system:kube-vip-binding
149+
roleRef:
150+
apiGroup: rbac.authorization.k8s.io
151+
kind: ClusterRole
152+
name: system:kube-vip-role
153+
subjects:
154+
- kind: ServiceAccount
155+
name: kube-vip
156+
namespace: kube-system
157+
owner: root:root
115158
---
116159
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
117160
kind: DockerMachineTemplate
@@ -121,6 +164,7 @@ spec:
121164
template:
122165
spec:
123166
customImage: kindest/node:${KIND_IMAGE_VERSION}
167+
bootstrapTimeout: 15m
124168
---
125169
apiVersion: cluster.x-k8s.io/v1beta1
126170
kind: MachineDeployment
@@ -154,6 +198,7 @@ spec:
154198
template:
155199
spec:
156200
customImage: kindest/node:${KIND_IMAGE_VERSION}
201+
bootstrapTimeout: 15m
157202
---
158203
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
159204
kind: RKE2ConfigTemplate

‎test/e2e/data/infrastructure/cluster-template-docker.yaml

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
cidrBlocks:
7676
- 10.46.0.0/16
7777
serviceDomain: cluster.local
78+
controlPlaneEndpoint:
79+
host: "${REGISTRATION_VIP}"
80+
port: 6443
7881
controlPlaneRef:
7982
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
8083
kind: RKE2ControlPlane
@@ -111,7 +114,47 @@ spec:
111114
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
112115
kind: DockerMachineTemplate
113116
name: "${CLUSTER_NAME}-control-plane"
114-
nodeDrainTimeout: 2m
117+
nodeDrainTimeout: 30s
118+
registrationMethod: "address"
119+
registrationAddress: "${REGISTRATION_VIP}"
120+
preRKE2Commands:
121+
- mkdir -p /var/lib/rancher/rke2/server/manifests/ && ctr images pull ghcr.io/kube-vip/kube-vip:v0.6.0 && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v0.6.0 vip /kube-vip manifest daemonset --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address ${REGISTRATION_VIP} --controlplane --leaderElection --taint --services --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
122+
files:
123+
- path: /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml
124+
content: |
125+
apiVersion: v1
126+
kind: ServiceAccount
127+
metadata:
128+
name: kube-vip
129+
namespace: kube-system
130+
---
131+
apiVersion: rbac.authorization.k8s.io/v1
132+
kind: ClusterRole
133+
metadata:
134+
annotations:
135+
rbac.authorization.kubernetes.io/autoupdate: "true"
136+
name: system:kube-vip-role
137+
rules:
138+
- apiGroups: [""]
139+
resources: ["services", "services/status", "nodes", "endpoints"]
140+
verbs: ["list","get","watch", "update"]
141+
- apiGroups: ["coordination.k8s.io"]
142+
resources: ["leases"]
143+
verbs: ["list", "get", "watch", "update", "create"]
144+
---
145+
kind: ClusterRoleBinding
146+
apiVersion: rbac.authorization.k8s.io/v1
147+
metadata:
148+
name: system:kube-vip-binding
149+
roleRef:
150+
apiGroup: rbac.authorization.k8s.io
151+
kind: ClusterRole
152+
name: system:kube-vip-role
153+
subjects:
154+
- kind: ServiceAccount
155+
name: kube-vip
156+
namespace: kube-system
157+
owner: root:root
115158
---
116159
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
117160
kind: DockerMachineTemplate
@@ -121,6 +164,7 @@ spec:
121164
template:
122165
spec:
123166
customImage: kindest/node:${KIND_IMAGE_VERSION}
167+
bootstrapTimeout: 15m
124168
---
125169
apiVersion: cluster.x-k8s.io/v1beta1
126170
kind: MachineDeployment
@@ -154,6 +198,7 @@ spec:
154198
template:
155199
spec:
156200
customImage: kindest/node:${KIND_IMAGE_VERSION}
201+
bootstrapTimeout: 15m
157202
---
158203
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
159204
kind: RKE2ConfigTemplate

‎test/e2e/e2e_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,9 @@ var _ = Describe("Workload cluster creation", func() {
9999
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
100100
ControlPlaneMachineCount: pointer.Int64Ptr(1),
101101
WorkerMachineCount: pointer.Int64Ptr(1),
102+
ClusterctlVariables: map[string]string{
103+
"REGISTRATION_VIP": "192.168.32.168",
104+
},
102105
},
103106
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
104107
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -120,6 +123,9 @@ var _ = Describe("Workload cluster creation", func() {
120123
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
121124
ControlPlaneMachineCount: pointer.Int64Ptr(1),
122125
WorkerMachineCount: pointer.Int64Ptr(3),
126+
ClusterctlVariables: map[string]string{
127+
"REGISTRATION_VIP": "192.168.32.168",
128+
},
123129
},
124130
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
125131
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -140,6 +146,9 @@ var _ = Describe("Workload cluster creation", func() {
140146
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
141147
ControlPlaneMachineCount: pointer.Int64Ptr(1),
142148
WorkerMachineCount: pointer.Int64Ptr(3),
149+
ClusterctlVariables: map[string]string{
150+
"REGISTRATION_VIP": "192.168.32.168",
151+
},
143152
},
144153
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
145154
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -173,6 +182,9 @@ var _ = Describe("Workload cluster creation", func() {
173182
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
174183
ControlPlaneMachineCount: pointer.Int64Ptr(3),
175184
WorkerMachineCount: pointer.Int64Ptr(3),
185+
ClusterctlVariables: map[string]string{
186+
"REGISTRATION_VIP": "192.168.32.168",
187+
},
176188
},
177189
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
178190
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -199,6 +211,9 @@ var _ = Describe("Workload cluster creation", func() {
199211
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
200212
ControlPlaneMachineCount: pointer.Int64Ptr(1),
201213
WorkerMachineCount: pointer.Int64Ptr(3),
214+
ClusterctlVariables: map[string]string{
215+
"REGISTRATION_VIP": "192.168.32.168",
216+
},
202217
},
203218
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
204219
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -225,6 +240,9 @@ var _ = Describe("Workload cluster creation", func() {
225240
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
226241
ControlPlaneMachineCount: pointer.Int64Ptr(1),
227242
WorkerMachineCount: pointer.Int64Ptr(1),
243+
ClusterctlVariables: map[string]string{
244+
"REGISTRATION_VIP": "192.168.32.168",
245+
},
228246
},
229247
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
230248
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),

‎test/e2e/e2e_upgrade_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ var _ = Describe("Workload cluster creation", func() {
100100
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
101101
ControlPlaneMachineCount: pointer.Int64Ptr(3),
102102
WorkerMachineCount: pointer.Int64Ptr(1),
103+
ClusterctlVariables: map[string]string{
104+
"REGISTRATION_VIP": "192.168.32.168",
105+
},
103106
},
104107
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
105108
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -137,6 +140,9 @@ var _ = Describe("Workload cluster creation", func() {
137140
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
138141
ControlPlaneMachineCount: pointer.Int64Ptr(2),
139142
WorkerMachineCount: pointer.Int64Ptr(1),
143+
ClusterctlVariables: map[string]string{
144+
"REGISTRATION_VIP": "192.168.32.168",
145+
},
140146
},
141147
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
142148
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -166,6 +172,9 @@ var _ = Describe("Workload cluster creation", func() {
166172
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
167173
ControlPlaneMachineCount: pointer.Int64Ptr(2),
168174
WorkerMachineCount: pointer.Int64Ptr(1),
175+
ClusterctlVariables: map[string]string{
176+
"REGISTRATION_VIP": "192.168.32.168",
177+
},
169178
},
170179
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
171180
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
@@ -199,6 +208,9 @@ var _ = Describe("Workload cluster creation", func() {
199208
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
200209
ControlPlaneMachineCount: pointer.Int64Ptr(1),
201210
WorkerMachineCount: pointer.Int64Ptr(1),
211+
ClusterctlVariables: map[string]string{
212+
"REGISTRATION_VIP": "192.168.32.168",
213+
},
202214
},
203215
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
204216
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),

‎test/e2e/metadata_e2e_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ var _ = Describe("Workload cluster creation", func() {
103103
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
104104
ControlPlaneMachineCount: pointer.Int64Ptr(1),
105105
WorkerMachineCount: pointer.Int64Ptr(1),
106+
ClusterctlVariables: map[string]string{
107+
"REGISTRATION_VIP": "192.168.32.168",
108+
},
106109
},
107110
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
108111
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),

0 commit comments

Comments
 (0)