Skip to content

Commit 9cb3710

Browse files
anmazzottianmazzotti
committed
Test kubeconfig cert rotation
Signed-off-by: Andrea Mazzotti <[email protected]>
1 parent a68cc5a commit 9cb3710

File tree

3 files changed

+115
-39
lines changed

3 files changed

+115
-39
lines changed

‎controlplane/internal/controllers/rke2controlplane_controller.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -845,6 +845,7 @@ func (r *RKE2ControlPlaneReconciler) reconcileKubeconfig(
845845

846846
switch {
847847
case apierrors.IsNotFound(err):
848+
logger.Info("Kubeconfig Secret not found, creating a new one")
848849
createErr := kubeconfig.CreateSecretWithOwner(
849850
ctx,
850851
r.Client,
@@ -853,6 +854,7 @@ func (r *RKE2ControlPlaneReconciler) reconcileKubeconfig(
853854
controllerOwnerRef,
854855
)
855856
if errors.Is(createErr, kubeconfig.ErrDependentCertificateNotFound) {
857+
logger.Info("Could not find Secret CA to create Kubeconfig Secret, requeuing...")
856858
return ctrl.Result{RequeueAfter: dependentCertRequeueAfter}, nil
857859
}
858860
// always return if we have just created in order to skip rotation checks
@@ -864,6 +866,7 @@ func (r *RKE2ControlPlaneReconciler) reconcileKubeconfig(
864866

865867
// only do rotation on owned secrets
866868
if !util.IsControlledBy(configSecret, rcp) {
869+
logger.Info("Kubeconfig Secret not controlled by RKE2ControlPlane, nothing to do")
867870
return ctrl.Result{}, nil
868871
}
869872

@@ -874,8 +877,7 @@ func (r *RKE2ControlPlaneReconciler) reconcileKubeconfig(
874877

875878
if needsRotation {
876879
logger.Info("Rotating kubeconfig secret")
877-
878-
if err := kubeconfig.CreateSecretWithOwner(ctx, r.Client, clusterName, endpoint.String(), controllerOwnerRef); err != nil {
880+
if err := kubeconfig.UpdateSecret(ctx, r.Client, clusterName, endpoint.String(), configSecret); err != nil {
879881
return ctrl.Result{}, errors.Wrap(err, "failed to regenerate kubeconfig")
880882
}
881883
}

‎controlplane/internal/controllers/rke2controlplane_controller_test.go

+96-35
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@ import (
1616
bootstrapv1 "github.com/rancher/cluster-api-provider-rke2/bootstrap/api/v1beta1"
1717
controlplanev1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1beta1"
1818
"github.com/rancher/cluster-api-provider-rke2/pkg/rke2"
19+
"github.com/rancher/cluster-api-provider-rke2/pkg/secret"
1920
corev1 "k8s.io/api/core/v1"
2021
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
22+
"k8s.io/apimachinery/pkg/types"
2123
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2224
"sigs.k8s.io/cluster-api/util/certs"
2325
"sigs.k8s.io/cluster-api/util/collections"
@@ -212,6 +214,15 @@ var _ = Describe("Reconcile control plane conditions", func() {
212214
Expect(testEnv.Client.Create(ctx, cluster)).To(Succeed())
213215

214216
rcp = &controlplanev1.RKE2ControlPlane{
217+
ObjectMeta: metav1.ObjectMeta{
218+
Name: "test",
219+
Namespace: ns.Name,
220+
UID: "foobar",
221+
},
222+
TypeMeta: metav1.TypeMeta{
223+
APIVersion: controlplanev1.GroupVersion.String(),
224+
Kind: rke2ControlPlaneKind,
225+
},
215226
Status: controlplanev1.RKE2ControlPlaneStatus{
216227
Initialized: true,
217228
},
@@ -220,16 +231,34 @@ var _ = Describe("Reconcile control plane conditions", func() {
220231
cp, err = rke2.NewControlPlane(ctx, testEnv.GetClient(), cluster, rcp, collections.FromMachineList(&ml))
221232
Expect(err).ToNot(HaveOccurred())
222233

223-
ref := metav1.OwnerReference{
224-
APIVersion: clusterv1.GroupVersion.String(),
225-
Kind: clusterv1.ClusterKind,
226-
UID: cp.Cluster.GetUID(),
227-
Name: cp.Cluster.GetName(),
234+
// Generate new Secret Cluster CA
235+
certPEM, _, err := generateCertAndKey(time.Now().Add(3650 * 24 * time.Hour)) // 10 years from now
236+
Expect(err).ShouldNot(HaveOccurred())
237+
caSecret := &corev1.Secret{
238+
ObjectMeta: metav1.ObjectMeta{
239+
Name: secret.Name(cluster.Name, secret.ClusterCA),
240+
Namespace: ns.Name,
241+
},
242+
StringData: map[string]string{
243+
secret.TLSCrtDataName: string(certPEM),
244+
},
228245
}
229-
Expect(testEnv.Client.Create(ctx, kubeconfig.GenerateSecretWithOwner(
230-
client.ObjectKeyFromObject(cp.Cluster),
231-
kubeconfig.FromEnvTestConfig(testEnv.Config, cp.Cluster),
232-
ref))).To(Succeed())
246+
Expect(testEnv.Client.Create(ctx, caSecret)).Should(Succeed())
247+
248+
// Generate new Secret Client Cluster CA
249+
certPEM, keyPEM, err := generateCertAndKey(time.Now().Add(3650 * 24 * time.Hour)) // 10 years from now
250+
Expect(err).ShouldNot(HaveOccurred())
251+
ccaSecret := &corev1.Secret{
252+
ObjectMeta: metav1.ObjectMeta{
253+
Name: secret.Name(cluster.Name, secret.ClientClusterCA),
254+
Namespace: ns.Name,
255+
},
256+
StringData: map[string]string{
257+
secret.TLSCrtDataName: string(certPEM),
258+
secret.TLSKeyDataName: string(keyPEM),
259+
},
260+
}
261+
Expect(testEnv.Client.Create(ctx, ccaSecret)).Should(Succeed())
233262
})
234263

235264
AfterEach(func() {
@@ -244,6 +273,18 @@ var _ = Describe("Reconcile control plane conditions", func() {
244273
managementCluster: &rke2.Management{Client: testEnv.GetClient(), SecretCachingClient: testEnv.GetClient()},
245274
managementClusterUncached: &rke2.Management{Client: testEnv.GetClient()},
246275
}
276+
// Create a new Kubeconfig Secret
277+
ref := metav1.OwnerReference{
278+
APIVersion: clusterv1.GroupVersion.String(),
279+
Kind: clusterv1.ClusterKind,
280+
UID: cp.Cluster.GetUID(),
281+
Name: cp.Cluster.GetName(),
282+
}
283+
Expect(testEnv.Client.Create(ctx, kubeconfig.GenerateSecretWithOwner(
284+
client.ObjectKeyFromObject(cp.Cluster),
285+
kubeconfig.FromEnvTestConfig(testEnv.Config, cp.Cluster),
286+
ref))).To(Succeed())
287+
247288
_, err := r.reconcileControlPlaneConditions(ctx, cp)
248289
Expect(err).ToNot(HaveOccurred())
249290
Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(machine), machine)).To(Succeed())
@@ -269,22 +310,34 @@ var _ = Describe("Reconcile control plane conditions", func() {
269310
clusterName := client.ObjectKey{Namespace: ns.Name, Name: "test"}
270311
endpoint := clusterv1.APIEndpoint{Host: "1.2.3.4", Port: 6443}
271312

272-
// Create kubeconfig secret with short expiry
273-
shortExpiryDate := time.Now().Add(24 * time.Hour) // 1 day from now
274-
secret, err := createKubeconfigSecret(ns.Name, shortExpiryDate)
313+
// Trigger first reconcile to generate a new Kubeconfig Secret
314+
_, err = r.reconcileKubeconfig(ctx, clusterName, endpoint, rcp)
275315
Expect(err).ToNot(HaveOccurred())
276-
Expect(testEnv.Create(ctx, secret)).To(Succeed())
316+
317+
// Fetch the original Kubeconfig Secret
318+
kubeconfigSecret := &corev1.Secret{}
319+
Expect(testEnv.Get(ctx, types.NamespacedName{
320+
Namespace: ns.Name,
321+
Name: secret.Name(clusterName.Name, secret.Kubeconfig),
322+
}, kubeconfigSecret)).Should(Succeed())
323+
originalSecret := kubeconfigSecret.DeepCopy()
324+
325+
// Override the kubeconfig secret with short expiry
326+
shortExpiryDate := time.Now().Add(24 * time.Hour) // 1 day from now
327+
Expect(updateKubeconfigSecret(kubeconfigSecret, shortExpiryDate)).Should(Succeed())
328+
Expect(testEnv.Update(ctx, kubeconfigSecret)).To(Succeed())
277329

278330
// Check that rotation is needed
279-
needsRotation, err := kubeconfig.NeedsClientCertRotation(secret, certs.ClientCertificateRenewalDuration)
331+
needsRotation, err := kubeconfig.NeedsClientCertRotation(kubeconfigSecret, certs.ClientCertificateRenewalDuration)
280332
Expect(err).ToNot(HaveOccurred())
281333
Expect(needsRotation).To(BeTrue())
282334

283335
// Rotate kubeconfig secret
284336
_, err = r.reconcileKubeconfig(ctx, clusterName, endpoint, rcp)
285337
Expect(err).ToNot(HaveOccurred())
286338

287-
Expect(testEnv.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: secret.Name}, secret)).To(Succeed())
339+
Expect(testEnv.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: kubeconfigSecret.Name}, kubeconfigSecret)).To(Succeed())
340+
Expect(kubeconfigSecret.StringData[secret.KubeconfigDataName]).Should(Equal(originalSecret.StringData[secret.KubeconfigDataName]), "Kubeconfig data must have been updated")
288341
})
289342

290343
It("should not rotate kubeconfig secret if not needed", func() {
@@ -297,22 +350,38 @@ var _ = Describe("Reconcile control plane conditions", func() {
297350
clusterName := client.ObjectKey{Namespace: ns.Name, Name: "test"}
298351
endpoint := clusterv1.APIEndpoint{Host: "1.2.3.4", Port: 6443}
299352

300-
// Create kubeconfig secret with long expiry
301-
longExpiryDate := time.Now().Add(365 * 24 * time.Hour) // 1 year from now
302-
secret, err := createKubeconfigSecret(ns.Name, longExpiryDate)
353+
// Trigger first reconcile to generate a new Kubeconfig Secret
354+
_, err = r.reconcileKubeconfig(ctx, clusterName, endpoint, rcp)
303355
Expect(err).ToNot(HaveOccurred())
304-
Expect(testEnv.Create(ctx, secret)).To(Succeed())
356+
kubeconfigSecret := &corev1.Secret{}
357+
Expect(testEnv.Get(ctx, types.NamespacedName{
358+
Namespace: ns.Name,
359+
Name: secret.Name(clusterName.Name, secret.Kubeconfig),
360+
}, kubeconfigSecret)).Should(Succeed())
361+
362+
// Override the kubeconfig secret with a long expiry
363+
longExpiryDate := time.Now().Add(365 * 24 * time.Hour) // 1 year from now
364+
Expect(updateKubeconfigSecret(kubeconfigSecret, longExpiryDate)).Should(Succeed())
365+
Expect(testEnv.Update(ctx, kubeconfigSecret)).To(Succeed())
305366

306367
// Check that no rotation is needed
307-
needsRotation, err := kubeconfig.NeedsClientCertRotation(secret, certs.ClientCertificateRenewalDuration)
368+
needsRotation, err := kubeconfig.NeedsClientCertRotation(kubeconfigSecret, certs.ClientCertificateRenewalDuration)
308369
Expect(err).ToNot(HaveOccurred())
309370
Expect(needsRotation).To(BeFalse())
310371

372+
// Fetch the overridden kubeconfigSecret
373+
Expect(testEnv.Get(ctx, types.NamespacedName{
374+
Namespace: ns.Name,
375+
Name: secret.Name(clusterName.Name, secret.Kubeconfig),
376+
}, kubeconfigSecret)).Should(Succeed())
377+
updatedSecret := kubeconfigSecret.DeepCopy()
378+
311379
// Ensure no rotation occurs
312380
_, err = r.reconcileKubeconfig(ctx, clusterName, endpoint, rcp)
313381
Expect(err).ToNot(HaveOccurred())
314382

315-
Expect(testEnv.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: secret.Name}, secret)).To(Succeed())
383+
Expect(testEnv.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: kubeconfigSecret.Name}, kubeconfigSecret)).To(Succeed())
384+
Expect(kubeconfigSecret.StringData[secret.KubeconfigDataName]).Should(Equal(updatedSecret.StringData[secret.KubeconfigDataName]), "Kubeconfig data must stay the same")
316385
})
317386
})
318387

@@ -347,20 +416,14 @@ func generateCertAndKey(expiryDate time.Time) ([]byte, []byte, error) {
347416
return certPEM, keyPEM, nil
348417
}
349418

350-
// createKubeconfigSecret creates a Kubernetes secret with a kubeconfig containing a client certificate and key.
351-
func createKubeconfigSecret(namespace string, expiryDate time.Time) (*corev1.Secret, error) {
419+
// updateKubeconfigSecret updates a Kubernetes secret with a kubeconfig containing a client certificate and key.
420+
func updateKubeconfigSecret(configSecret *corev1.Secret, expiryDate time.Time) error {
352421
certPEM, keyPEM, err := generateCertAndKey(expiryDate)
353422
if err != nil {
354-
return nil, err
423+
return fmt.Errorf("Generating Cert and Key: %w", err)
355424
}
356425

357-
secret := &corev1.Secret{
358-
ObjectMeta: metav1.ObjectMeta{
359-
Name: "test-kubeconfig-secret",
360-
Namespace: namespace,
361-
},
362-
Data: map[string][]byte{
363-
"value": []byte(fmt.Sprintf(`
426+
configSecret.Data[secret.KubeconfigDataName] = []byte(fmt.Sprintf(`
364427
apiVersion: v1
365428
kind: Config
366429
clusters:
@@ -378,9 +441,7 @@ users:
378441
user:
379442
client-certificate-data: %s
380443
client-key-data: %s
381-
`, base64.StdEncoding.EncodeToString(certPEM), base64.StdEncoding.EncodeToString(keyPEM))),
382-
},
383-
}
444+
`, base64.StdEncoding.EncodeToString(certPEM), base64.StdEncoding.EncodeToString(keyPEM)))
384445

385-
return secret, nil
446+
return nil
386447
}

‎pkg/kubeconfig/kubeconfig.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func generateKubeconfig(ctx context.Context, c client.Client, clusterName client
4545
clusterCA, err := secret.GetFromNamespacedName(ctx, c, clusterName, secret.ClusterCA)
4646
if err != nil {
4747
if apierrors.IsNotFound(errors.Cause(err)) {
48-
return nil, ErrDependentCertificateNotFound
48+
return nil, fmt.Errorf("getting Cluster CA: %w", ErrDependentCertificateNotFound)
4949
}
5050

5151
return nil, err
@@ -54,7 +54,7 @@ func generateKubeconfig(ctx context.Context, c client.Client, clusterName client
5454
clientClusterCA, err := secret.GetFromNamespacedName(ctx, c, clusterName, secret.ClientClusterCA)
5555
if err != nil {
5656
if apierrors.IsNotFound(errors.Cause(err)) {
57-
return nil, ErrDependentCertificateNotFound
57+
return nil, fmt.Errorf("getting Client Cluster CA: %w", ErrDependentCertificateNotFound)
5858
}
5959

6060
return nil, err
@@ -166,6 +166,19 @@ func CreateSecretWithOwner(ctx context.Context, c client.Client, clusterName cli
166166
return c.Create(ctx, GenerateSecretWithOwner(clusterName, out, owner))
167167
}
168168

169+
// UpdateSecretWithOwner updates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference.
170+
func UpdateSecret(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, configSecret *corev1.Secret) error {
171+
server := "https://" + endpoint
172+
173+
out, err := generateKubeconfig(ctx, c, clusterName, server)
174+
if err != nil {
175+
return err
176+
}
177+
configSecret.Data[secret.KubeconfigDataName] = out
178+
179+
return c.Update(ctx, configSecret)
180+
}
181+
169182
// GenerateSecret returns a Kubernetes secret for the given Cluster and kubeconfig data.
170183
func GenerateSecret(cluster *clusterv1.Cluster, data []byte) *corev1.Secret {
171184
name := util.ObjectKey(cluster)

0 commit comments

Comments
 (0)