test: wait for ca injection in case we use an external clusterctl binary for upgrading

Author: chrischdi

test/framework/clusterctl/ca_injection.go

@@ -0,0 +1,219 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clusterctl
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+)
+
+const certManagerCAAnnotation = "cert-manager.io/inject-ca-from"
+
+func verifyCAInjection(ctx context.Context, c client.Client) error {
+	v := newCAInjectionVerifier(c)
+
+	errs := []error{}
+	errs = append(errs, v.verifyCustomResourceDefinitions(ctx)...)
+	errs = append(errs, v.verifyMutatingWebhookConfigurations(ctx)...)
+	errs = append(errs, v.verifyValidatingWebhookConfigurations(ctx)...)
+
+	return kerrors.NewAggregate(errs)
+}
+
+// certificateInjectionVerifier waits for cert-managers ca-injector to inject the
+// referred CA certificate to all CRDs, MutatingWebhookConfigurations and
+// ValidatingWebhookConfigurations.
+// As long as the correct CA certificates are not injected the kube-apiserver will
+// reject the requests due to certificate verification errors.
+type certificateInjectionVerifier struct {
+	Client client.Client
+}
+
+// newCAInjectionVerifier creates a new CRD migrator.
+func newCAInjectionVerifier(client client.Client) *certificateInjectionVerifier {
+	return &certificateInjectionVerifier{
+		Client: client,
+	}
+}
+
+func (c *certificateInjectionVerifier) verifyCustomResourceDefinitions(ctx context.Context) []error {
+	crds := &apiextensionsv1.CustomResourceDefinitionList{}
+	if err := c.Client.List(ctx, crds, client.HasLabels{clusterv1.ProviderNameLabel}); err != nil {
+		return []error{err}
+	}
+
+	errs := []error{}
+	for i := range crds.Items {
+		crd := crds.Items[i]
+		ca, err := c.getCACertificateFor(ctx, &crd)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+
+		if crd.Spec.Conversion.Webhook == nil || crd.Spec.Conversion.Webhook.ClientConfig == nil {
+			continue
+		}
+
+		if string(crd.Spec.Conversion.Webhook.ClientConfig.CABundle) != ca {
+			errs = append(errs, fmt.Errorf("injected CA for CustomResourceDefinition %s does not match", crd.Name))
+		}
+	}
+
+	return errs
+}
+
+func (c *certificateInjectionVerifier) verifyMutatingWebhookConfigurations(ctx context.Context) []error {
+	mutateHooks := &admissionregistrationv1.MutatingWebhookConfigurationList{}
+	if err := c.Client.List(ctx, mutateHooks, client.HasLabels{clusterv1.ProviderNameLabel}); err != nil {
+		return []error{err}
+	}
+
+	errs := []error{}
+	for i := range mutateHooks.Items {
+		mutateHook := mutateHooks.Items[i]
+		ca, err := c.getCACertificateFor(ctx, &mutateHook)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		var changed bool
+		for _, webhook := range mutateHook.Webhooks {
+			if string(webhook.ClientConfig.CABundle) != ca {
+				changed = true
+				errs = append(errs, fmt.Errorf("injected CA for MutatingWebhookConfiguration %s does not match", webhook.Name))
+			}
+		}
+		if changed {
+			annotatedHook := mutateHook.DeepCopy()
+			annotatedHook.Annotations["cluster-api-force-ca-injection"] = rand.String(10)
+			if err := c.Client.Patch(ctx, annotatedHook, client.StrategicMergeFrom(&mutateHook)); err != nil {
+				errs = append(errs, err)
+			}
+		}
+	}
+
+	return errs
+}
+
+func (c *certificateInjectionVerifier) verifyValidatingWebhookConfigurations(ctx context.Context) []error {
+	validateHooks := &admissionregistrationv1.ValidatingWebhookConfigurationList{}
+	if err := c.Client.List(ctx, validateHooks, client.HasLabels{clusterv1.ProviderNameLabel}); err != nil {
+		return []error{err}
+	}
+
+	errs := []error{}
+	for i := range validateHooks.Items {
+		validateHook := validateHooks.Items[i]
+		ca, err := c.getCACertificateFor(ctx, &validateHook)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		var changed bool
+		for _, webhook := range validateHook.Webhooks {
+			if string(webhook.ClientConfig.CABundle) != ca {
+				changed = true
+				errs = append(errs, fmt.Errorf("injected CA for ValidatingWebhookConfiguration %s does not match", webhook.Name))
+			}
+		}
+		if changed {
+			annotatedHook := validateHook.DeepCopy()
+			annotatedHook.Annotations["cluster-api-force-ca-injection"] = rand.String(10)
+			if err := c.Client.Patch(ctx, annotatedHook, client.StrategicMergeFrom(&validateHook)); err != nil {
+				errs = append(errs, err)
+			}
+		}
+	}
+
+	return errs
+}
+
+// getCACertificateFor returns the ca certificate from the secret referred by the
+// Certificate object. It reads the namespaced name of the Certificate from the
+// injection annotation of the passed object.
+func (c *certificateInjectionVerifier) getCACertificateFor(ctx context.Context, obj client.Object) (string, error) {
+	annotationValue, ok := obj.GetAnnotations()[certManagerCAAnnotation]
+	if !ok || annotationValue == "" {
+		return "", fmt.Errorf("getting value for injection annotation")
+	}
+
+	certificateObjKey := splitObjectKey(annotationValue)
+
+	certificate := &unstructured.Unstructured{}
+	certificate.SetKind("Certificate")
+	certificate.SetAPIVersion("cert-manager.io/v1")
+
+	if err := c.Client.Get(ctx, certificateObjKey, certificate); err != nil {
+		return "", errors.Wrapf(err, "getting certificate %s", certificateObjKey)
+	}
+
+	secretName, _, err := unstructured.NestedString(certificate.Object, "spec", "secretName")
+	if err != nil || secretName == "" {
+		return "", errors.Wrapf(err, "reading .spec.secretName name from certificate %s", certificateObjKey)
+	}
+
+	secretObjKey := client.ObjectKey{Namespace: certificate.GetNamespace(), Name: secretName}
+	certificateSecret := &corev1.Secret{}
+	if err := c.Client.Get(ctx, secretObjKey, certificateSecret); err != nil {
+		return "", errors.Wrapf(err, "getting secret %s for certificate %s", secretObjKey, certificateObjKey)
+	}
+
+	ca, ok := certificateSecret.Data["ca.crt"]
+	if !ok {
+		return "", errors.Errorf("data for \"ca.crt\" not found in secret %s", secretObjKey)
+	}
+
+	return string(ca), nil
+}
+
+// splitObjectKey splits the string by the name separator and returns it as client.ObjectKey.
+func splitObjectKey(nameStr string) client.ObjectKey {
+	splitPoint := strings.IndexRune(nameStr, types.Separator)
+	if splitPoint == -1 {
+		return client.ObjectKey{Name: nameStr}
+	}
+	return client.ObjectKey{Namespace: nameStr[:splitPoint], Name: nameStr[splitPoint+1:]}
+}
+
+// newVerifyBackoff creates a new API Machinery backoff parameter set suitable for use with clusterctl verify operations.
+func newVerifyBackoff() wait.Backoff {
+	// Return a exponential backoff configuration which returns durations for a total time of ~5m.
+	// Example: 0, .5s, 1.2s, 2s, 3.1s, 4.5s, 6.4s, 9s, 12s, 16s, 21s, 28s, ...
+	// Jitter is added as a random fraction of the duration multiplied by the jitter factor.
+	return wait.Backoff{
+		Duration: 500 * time.Millisecond,
+		Factor:   1.2,
+		Steps:    30,
+		Jitter:   0.4,
+	}
+}

test/framework/clusterctl/clusterctl_helpers.go

@@ -95,6 +95,13 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init
 
 		if input.ClusterctlBinaryPath != "" {
 			InitWithBinary(ctx, input.ClusterctlBinaryPath, initInput)
+			// Old versions of clusterctl may deploy CRDs, Mutating- and/or ValidatingWebhookConfigurations
+			// before creating the new Certificate objects. This check ensures the CA's are up to date before
+			// continuing.
+			Eventually(func() error {
+				return verifyCAInjection(ctx, client)
+			}, time.Minute*5, time.Second*10).Should(Succeed(), "Failed to verify CA injection")
+
 		} else {
 			Init(ctx, initInput)
 		}
@@ -183,14 +190,20 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen
 		LogFolder:                 input.LogFolder,
 	}
 
+	client := input.ClusterProxy.GetClient()
+
 	if input.ClusterctlBinaryPath != "" {
 		UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput)
+		// Old versions of clusterctl may deploy CRDs, Mutating- and/or ValidatingWebhookConfigurations
+		// before creating the new Certificate objects. This check ensures the CA's are up to date before
+		// continuing.
+		Eventually(func() error {
+			return verifyCAInjection(ctx, client)
+		}, time.Minute*5, time.Second*10).Should(Succeed(), "Failed to verify CA injection")
 	} else {
 		Upgrade(ctx, upgradeInput)
 	}
 
-	client := input.ClusterProxy.GetClient()
-
 	log.Logf("Waiting for provider controllers to be running")
 	controllersDeployments := framework.GetControllerDeployments(ctx, framework.GetControllerDeploymentsInput{
 		Lister: client,