Merge pull request #302 from anmazzotti/set_control_plane_initialized

Check rke2-serving secret to determine controlPlane.Status.Initialized

Author: mbologna

controlplane/internal/controllers/rke2controlplane_controller.go

@@ -369,7 +369,16 @@ func (r *RKE2ControlPlaneReconciler) updateStatus(ctx context.Context, rcp *cont
 	rcp.Status.ReadyReplicas = int32(len(readyMachines))
 	rcp.Status.UnavailableReplicas = replicas - rcp.Status.ReadyReplicas
 
-	if rcp.Status.ReadyReplicas > 0 {
+	workloadCluster, err := r.getWorkloadCluster(ctx, util.ObjectKey(cluster))
+	if err != nil {
+		logger.Error(err, "Failed to get remote client for workload cluster", "cluster key", util.ObjectKey(cluster))
+
+		return fmt.Errorf("getting workload cluster: %w", err)
+	}
+
+	status := workloadCluster.ClusterStatus(ctx)
+
+	if status.HasRKE2ServingSecret {
 		rcp.Status.Initialized = true
 	}
 
@@ -790,11 +799,11 @@ func (r *RKE2ControlPlaneReconciler) reconcileControlPlaneConditions(
 		return ctrl.Result{}, nil
 	}
 
-	workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(controlPlane.Cluster))
+	workloadCluster, err := r.getWorkloadCluster(ctx, util.ObjectKey(controlPlane.Cluster))
 	if err != nil {
-		logger.Error(err, "Unable to get Workload cluster")
+		logger.Error(err, "Failed to get remote client for workload cluster", "cluster key", util.ObjectKey(controlPlane.Cluster))
 
-		return ctrl.Result{}, errors.Wrap(err, "cannot get remote client to workload cluster")
+		return ctrl.Result{}, fmt.Errorf("getting workload cluster: %w", err)
 	}
 
 	defer func() {
@@ -842,11 +851,11 @@ func (r *RKE2ControlPlaneReconciler) upgradeControlPlane(
 		return ctrl.Result{}, nil
 	}
 
-	workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(cluster))
+	workloadCluster, err := r.getWorkloadCluster(ctx, util.ObjectKey(cluster))
 	if err != nil {
-		logger.Error(err, "failed to get remote client for workload cluster", "cluster key", util.ObjectKey(cluster))
+		logger.Error(err, "Failed to get remote client for workload cluster", "cluster key", util.ObjectKey(cluster))
 
-		return ctrl.Result{}, err
+		return ctrl.Result{}, fmt.Errorf("getting workload cluster: %w", err)
 	}
 
 	if err := workloadCluster.InitWorkload(ctx, controlPlane); err != nil {
@@ -892,3 +901,14 @@ func (r *RKE2ControlPlaneReconciler) ClusterToRKE2ControlPlane(ctx context.Conte
 		return nil
 	}
 }
+
+// getWorkloadCluster gets a cluster object.
+// The cluster comes with an etcd client generator to connect to any etcd pod living on a managed machine.
+func (r *RKE2ControlPlaneReconciler) getWorkloadCluster(ctx context.Context, clusterKey types.NamespacedName) (rke2.WorkloadCluster, error) {
+	workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, clusterKey)
+	if err != nil {
+		return nil, fmt.Errorf("getting remote client for workload cluster: %w", err)
+	}
+
+	return workloadCluster, nil
+}

pkg/rke2/workload_cluster.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/rest"
@@ -51,6 +52,7 @@ const (
 	etcdDialTimeout           = 10 * time.Second
 	etcdCallTimeout           = 15 * time.Second
 	minimalNodeCount          = 2
+	rke2ServingSecretKey      = "rke2-serving" //nolint: gosec
 )
 
 // ErrControlPlaneMinNodes is returned when the control plane has fewer than 2 nodes.
@@ -62,7 +64,7 @@ type WorkloadCluster interface {
 	InitWorkload(ctx context.Context, controlPlane *ControlPlane) error
 	UpdateNodeMetadata(ctx context.Context, controlPlane *ControlPlane) error
 
-	ClusterStatus() ClusterStatus
+	ClusterStatus(ctx context.Context) ClusterStatus
 	UpdateAgentConditions(controlPlane *ControlPlane)
 	UpdateEtcdConditions(controlPlane *ControlPlane)
 	// Upgrade related tasks.
@@ -196,6 +198,8 @@ type ClusterStatus struct {
 	Nodes int32
 	// ReadyNodes are the count of nodes that are reporting ready
 	ReadyNodes int32
+	// HasRKE2ServingSecret will be true if the rke2-serving secret has been uploaded, false otherwise.
+	HasRKE2ServingSecret bool
 }
 
 func (w *Workload) getControlPlaneNodes(ctx context.Context) (*corev1.NodeList, error) {
@@ -253,7 +257,7 @@ func (w *Workload) PatchNodes(ctx context.Context, cp *ControlPlane) error {
 }
 
 // ClusterStatus returns the status of the cluster.
-func (w *Workload) ClusterStatus() ClusterStatus {
+func (w *Workload) ClusterStatus(ctx context.Context) ClusterStatus {
 	status := ClusterStatus{}
 
 	// count the control plane nodes
@@ -266,6 +270,20 @@ func (w *Workload) ClusterStatus() ClusterStatus {
 		}
 	}
 
+	// find the rke2-serving secret
+	key := ctrlclient.ObjectKey{
+		Name:      rke2ServingSecretKey,
+		Namespace: metav1.NamespaceSystem,
+	}
+	err := w.Client.Get(ctx, key, &corev1.Secret{})
+	// In case of error we do assume the control plane is not initialized yet.
+	if err != nil {
+		logger := log.FromContext(ctx)
+		logger.Info("Control Plane does not seem to be initialized yet.", "reason", err.Error())
+	}
+
+	status.HasRKE2ServingSecret = err == nil
+
 	return status
 }
 

pkg/rke2/workload_cluster_test.go

@@ -1,8 +1,11 @@
 package rke2
 
 import (
+	"context"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
 	bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1beta1"
 	controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1beta1"
 	corev1 "k8s.io/api/core/v1"
@@ -12,6 +15,8 @@ import (
 	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
 )
 
 var _ = Describe("Node metadata propagation", func() {
@@ -394,3 +399,113 @@ var _ = Describe("Cloud-init fields validation", func() {
 		}})).ToNot(Succeed())
 	})
 })
+
+var _ = Describe("ClusterStatus validation", func() {
+	var (
+		err           error
+		ns            *corev1.Namespace
+		node1         *corev1.Node
+		node2         *corev1.Node
+		servingSecret *corev1.Secret
+		connectionErr interceptor.Funcs
+	)
+
+	BeforeEach(func() {
+		ns, err = testEnv.CreateNamespace(ctx, "ns")
+		Expect(err).ToNot(HaveOccurred())
+
+		node1 = &corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "node1",
+				Labels: map[string]string{
+					"node-role.kubernetes.io/master": "true",
+				},
+				Annotations: map[string]string{
+					clusterv1.MachineAnnotation: "node1",
+				},
+			},
+			Status: corev1.NodeStatus{
+				Conditions: []corev1.NodeCondition{{
+					Type:   corev1.NodeReady,
+					Status: corev1.ConditionTrue,
+				}},
+			}}
+
+		node2 = &corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "node2",
+				Labels: map[string]string{
+					"node-role.kubernetes.io/master": "true",
+				},
+				Annotations: map[string]string{
+					clusterv1.MachineAnnotation: "node2",
+				},
+			},
+			Status: corev1.NodeStatus{
+				Conditions: []corev1.NodeCondition{{
+					Type:   corev1.NodeReady,
+					Status: corev1.ConditionFalse,
+				}},
+			}}
+
+		servingSecret = &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      rke2ServingSecretKey,
+				Namespace: metav1.NamespaceSystem,
+			},
+		}
+
+		connectionErr = interceptor.Funcs{
+			Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, _ ...client.GetOption) error {
+				return errors.New("test connection error")
+			},
+		}
+	})
+
+	AfterEach(func() {
+		testEnv.Cleanup(ctx, ns)
+	})
+
+	It("should set HasRKE2ServingSecret if servingSecret found", func() {
+		fakeClient := fake.NewClientBuilder().WithObjects([]client.Object{servingSecret}...).Build()
+		w := &Workload{
+			Client: fakeClient,
+			Nodes: map[string]*corev1.Node{
+				node1.Name: node1,
+				node2.Name: node2,
+			},
+		}
+		status := w.ClusterStatus(ctx)
+		Expect(status.Nodes).To(BeEquivalentTo(2), "There are 2 nodes in this cluster")
+		Expect(status.ReadyNodes).To(BeEquivalentTo(1), "Only 1 node has the NodeReady condition")
+		Expect(status.HasRKE2ServingSecret).To(BeTrue(), "rke2-serving Secret exists in kube-system namespace")
+	})
+	It("should not set HasRKE2ServingSecret if servingSecret not existing", func() {
+		fakeClient := fake.NewClientBuilder().Build()
+		w := &Workload{
+			Client: fakeClient,
+			Nodes: map[string]*corev1.Node{
+				node1.Name: node1,
+				node2.Name: node2,
+			},
+		}
+		status := w.ClusterStatus(ctx)
+		Expect(status.Nodes).To(BeEquivalentTo(2), "There are 2 nodes in this cluster")
+		Expect(status.ReadyNodes).To(BeEquivalentTo(1), "Only 1 node has the NodeReady condition")
+		Expect(status.HasRKE2ServingSecret).To(BeFalse(), "rke2-serving Secret does not exists in kube-system namespace")
+	})
+	It("should not set HasRKE2ServingSecret if client returns error", func() {
+		fakeClient := fake.NewClientBuilder().WithInterceptorFuncs(connectionErr).Build()
+		w := &Workload{
+			Client: fakeClient,
+			Nodes: map[string]*corev1.Node{
+				node1.Name: node1,
+				node2.Name: node2,
+			},
+		}
+		status := w.ClusterStatus(ctx)
+		Expect(status.Nodes).To(BeEquivalentTo(2), "There are 2 nodes in this cluster")
+		Expect(status.ReadyNodes).To(BeEquivalentTo(1), "Only 1 node has the NodeReady condition")
+		Expect(status.HasRKE2ServingSecret).To(BeFalse(), "On connection error assume control plane not initialized")
+	})
+})