Add etcd certificate generation

Signed-off-by: Danil Grigorev <[email protected]>

Author: Danil-Grigorev

Makefile

@@ -99,7 +99,7 @@ GOLANGCI_LINT_VER := v1.55.1
 GOLANGCI_LINT_BIN := golangci-lint
 GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN))
 
-GINKGO_VER := v2.14.0
+GINKGO_VER := v2.16.0
 GINKGO_BIN := ginkgo
 GINKGO := $(abspath $(TOOLS_BIN_DIR)/$(GINKGO_BIN)-$(GINKGO_VER))
 GINKGO_PKG := github.com/onsi/ginkgo/v2/ginkgo

controlplane/internal/controllers/rke2controlplane_controller.go

@@ -40,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/collections"
@@ -236,10 +237,26 @@ func (r *RKE2ControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr c
 	r.controller = c
 	r.recorder = mgr.GetEventRecorderFor("rke2-control-plane-controller")
 
+	// Set up a ClusterCacheTracker and ClusterCacheReconciler to provide to controllers
+	// requiring a connection to a remote cluster
+	tracker, err := remote.NewClusterCacheTracker(
+		mgr,
+		remote.ClusterCacheTrackerOptions{
+			SecretCachingClient: r.SecretCachingClient,
+			ControllerName:      "rke2-control-plane-controller",
+			Log:                 &ctrl.Log,
+			Indexes:             []remote.Index{},
+		},
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to create cluster cache tracker")
+	}
+
 	if r.managementCluster == nil {
 		r.managementCluster = &rke2.Management{
 			Client:              r.Client,
 			SecretCachingClient: r.SecretCachingClient,
+			Tracker:             tracker,
 		}
 	}
 

pkg/rke2/management_cluster.go

@@ -18,6 +18,13 @@ package rke2
 
 import (
 	"context"
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
 	"time"
 
 	"github.com/pkg/errors"
@@ -123,18 +130,18 @@ func (m *Management) GetWorkloadCluster(ctx context.Context, clusterKey ctrlclie
 }
 
 func (m *Management) getEtcdCAKeyPair(ctx context.Context, clusterKey ctrlclient.ObjectKey) (*certs.KeyPair, error) {
-	etcd := &secret.ManagedCertificate{
-		Purpose: secret.EtcdCA,
-	}
+	certificates := secret.Certificates([]secret.Certificate{&secret.ManagedCertificate{
+		Purpose: secret.EtcdServerCA,
+	}})
 
 	// Try to get the certificate via the cached ctrlclient.
-	s, err := etcd.Lookup(ctx, m.SecretCachingClient, clusterKey)
-	if err != nil || s == nil {
+	err := certificates.Lookup(ctx, m.SecretCachingClient, clusterKey)
+	if err != nil {
 		// Return error if we got an errors which is not a NotFound error.
 		return nil, errors.Wrapf(err, "failed to get secret; etcd CA bundle %s/%s", clusterKey.Namespace, secret.Name(clusterKey.Name, secret.EtcdCA))
 	}
 
-	return etcd.KeyPair, nil
+	return certificates[0].GetKeyPair(), nil
 }
 
 func (m *Management) getRemoteKeyPair(ctx context.Context, remoteClient ctrlclient.Client, clusterKey ctrlclient.ObjectKey) (*certs.KeyPair, error) {
@@ -152,3 +159,51 @@ func (m *Management) getRemoteKeyPair(ctx context.Context, remoteClient ctrlclie
 
 	return etcdCertificate.KeyPair, nil
 }
+
+func generateClientCert(caCertEncoded, caKeyEncoded []byte, clientKey *rsa.PrivateKey) (tls.Certificate, error) {
+	caCert, err := certs.DecodeCertPEM(caCertEncoded)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+
+	caKey, err := certs.DecodePrivateKeyPEM(caKeyEncoded)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+
+	x509Cert, err := newClientCert(caCert, clientKey, caKey)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+
+	return tls.X509KeyPair(certs.EncodeCertPEM(x509Cert), certs.EncodePrivateKeyPEM(clientKey))
+}
+
+func newClientCert(caCert *x509.Certificate, key *rsa.PrivateKey, caKey crypto.Signer) (*x509.Certificate, error) {
+	cfg := certs.Config{
+		CommonName: "cluster-api.x-k8s.io",
+	}
+
+	now := time.Now().UTC()
+
+	tmpl := x509.Certificate{
+		SerialNumber: new(big.Int).SetInt64(0),
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		NotBefore:   now.Add(time.Minute * -5),
+		NotAfter:    now.Add(secret.TenYears), // 10 years
+		KeyUsage:    x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+
+	b, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, key.Public(), caKey)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create signed client certificate: %+v", tmpl)
+	}
+
+	c, err := x509.ParseCertificate(b)
+
+	return c, errors.WithStack(err)
+}

pkg/rke2/workload_cluster.go

@@ -37,6 +37,7 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
+	"sigs.k8s.io/cluster-api/util/certs"
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/patch"
@@ -122,6 +123,18 @@ func (m *Management) NewWorkload(
 		return nil, err
 	}
 
+	if _, err := certs.DecodePrivateKeyPEM(keyPair.Key); err == nil {
+		clientKey, err := m.Tracker.GetEtcdClientCertificateKey(ctx, clusterKey)
+		if err != nil {
+			return nil, err
+		}
+
+		clientCert, err = generateClientCert(keyPair.Cert, keyPair.Key, clientKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	caPool := x509.NewCertPool()
 	caPool.AppendCertsFromPEM(keyPair.Cert)
 	tlsConfig := &tls.Config{

pkg/secret/certificates.go

@@ -46,14 +46,21 @@ const (
 	// be automatically used by RKE2 to use the pre-defined certificates instead of generating them.
 	DefaultCertificatesDir = "/var/lib/rancher/rke2/server/tls"
 
+	// DefaultETCDCertificatesDir is the default location (file path) where the provider will put the etcd certificates, this location will then
+	// be automatically used by RKE2 to use the pre-defined certificates instead of generating them.
+	DefaultETCDCertificatesDir = DefaultCertificatesDir + "/etcd"
+
 	// Kubeconfig is the secret name suffix storing the Cluster Kubeconfig.
 	Kubeconfig = Purpose("kubeconfig")
 
 	// KubeconfigDataName is the data entry name for the Kubeconfig file content.
 	KubeconfigDataName string = "value"
 
 	// EtcdCA is the secret name suffix for the Etcd CA.
-	EtcdCA Purpose = "etcd"
+	EtcdCA Purpose = "peer-etcd"
+
+	// EtcdServerCA is the secret name suffix for the Etcd CA.
+	EtcdServerCA Purpose = "etcd"
 
 	// ClusterCA is the secret name suffix for APIServer CA.
 	ClusterCA = Purpose("ca")
@@ -165,6 +172,16 @@ func NewCertificatesForInitialControlPlane() Certificates {
 			CertFile: filepath.Join(certificatesDir, "client-ca.crt"),
 			KeyFile:  filepath.Join(certificatesDir, "client-ca.key"),
 		},
+		&ManagedCertificate{
+			Purpose:  EtcdCA,
+			CertFile: filepath.Join(DefaultETCDCertificatesDir, "peer-ca.crt"),
+			KeyFile:  filepath.Join(DefaultETCDCertificatesDir, "peer-ca.key"),
+		},
+		&ManagedCertificate{
+			Purpose:  EtcdServerCA,
+			CertFile: filepath.Join(DefaultETCDCertificatesDir, "server-ca.crt"),
+			KeyFile:  filepath.Join(DefaultETCDCertificatesDir, "server-ca.key"),
+		},
 	}
 
 	return certificates
@@ -419,6 +436,7 @@ func (c Certificates) AsFiles() []bootstrapv1.File {
 	clientClusterCA := c.GetByPurpose(ClientClusterCA)
 
 	etcdCA := c.GetByPurpose(EtcdCA)
+	etcdServerCA := c.GetByPurpose(EtcdServerCA)
 
 	certFiles := make([]bootstrapv1.File, 0)
 	if clusterCA != nil {
@@ -433,6 +451,10 @@ func (c Certificates) AsFiles() []bootstrapv1.File {
 		certFiles = append(certFiles, etcdCA.AsFiles()...)
 	}
 
+	if etcdServerCA != nil {
+		certFiles = append(certFiles, etcdServerCA.AsFiles()...)
+	}
+
 	// these will only exist if external etcd was defined and supplied by the user
 	apiserverEtcdClientCert := c.GetByPurpose(APIServerEtcdClient)
 	if apiserverEtcdClientCert != nil {

test/e2e/e2e_test.go

@@ -218,8 +218,8 @@ var _ = Describe("Workload cluster creation", func() {
 					Flavor:                   "docker",
 					Namespace:                namespace.Name,
 					ClusterName:              clusterName,
-					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
-					ControlPlaneMachineCount: pointer.Int64Ptr(3), //TODO: change this back to 1 when scaling is supported
+					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
+					ControlPlaneMachineCount: pointer.Int64Ptr(1),
 					WorkerMachineCount:       pointer.Int64Ptr(1),
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),