Merge pull request #1122 from gianlucam76/mgmt-cluster

(feat) drift-detection in management cluster

Author: gianlucam76

Makefile

@@ -337,6 +337,7 @@ deploy-projectsveltos: $(KUSTOMIZE)
 
 	@echo 'Install libsveltos CRDs'
 	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_debuggingconfigurations.lib.projectsveltos.io.yaml
+	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_resourcesummaries.lib.projectsveltos.io.yaml
 	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sveltosclusters.lib.projectsveltos.io.yaml
 	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_clustersets.lib.projectsveltos.io.yaml
 	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/manifests/apiextensions.k8s.io_v1_customresourcedefinition_sets.lib.projectsveltos.io.yaml

cmd/main.go

@@ -181,6 +181,8 @@ func main() {
 	controllers.SetLuaConfigMap(luaConfigMap)
 	controllers.SetCAPIOnboardAnnotation(capiOnboardAnnotation)
 	controllers.SetDriftDetectionRegistry(registry)
+	controllers.SetAgentInMgmtCluster(agentInMgmtCluster)
+
 	// Start dependency manager
 	dependencymanager.InitializeManagerInstance(ctx, mgr.GetClient(), autoDeployDependencies, ctrl.Log.WithName("dependency_manager"))
 
@@ -495,7 +497,6 @@ func getClusterSummaryReconciler(ctx context.Context, mgr manager.Manager) *cont
 		ShardKey:             shardKey,
 		Version:              version,
 		ReportMode:           reportMode,
-		AgentInMgmtCluster:   agentInMgmtCluster,
 		Deployer:             d,
 		ClusterMap:           make(map[corev1.ObjectReference]*libsveltosset.Set),
 		ReferenceMap:         make(map[corev1.ObjectReference]*libsveltosset.Set),

config/rbac/role.yaml

@@ -145,6 +145,7 @@ rules:
   - lib.projectsveltos.io
   resources:
   - clustersets
+  - resourcesummaries
   - sets
   verbs:
   - create
@@ -180,6 +181,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - lib.projectsveltos.io
+  resources:
+  - resourcesummaries/status
+  verbs:
+  - get
+  - list
+  - update
 - apiGroups:
   - source.toolkit.fluxcd.io
   resources:

controllers/clustersummary_controller.go

@@ -90,7 +90,6 @@ type ClusterSummaryReconciler struct {
 	Scheme               *runtime.Scheme
 	Logger               logr.Logger
 	ReportMode           ReportMode
-	AgentInMgmtCluster   bool   // if true, indicates drift-detection-manager needs to be started in the management cluster
 	ShardKey             string // when set, only clusters matching the ShardKey will be reconciled
 	Version              string
 	Deployer             deployer.DeployerInterface
@@ -103,13 +102,18 @@ type ClusterSummaryReconciler struct {
 	ctrl              controller.Controller
 }
 
+// If the drift-detection component is deployed in the management cluster, the addon-controller will deploy ResourceSummaries within the same cluster,
+// thus requiring the necessary permissions.
+
 //+kubebuilder:rbac:groups=config.projectsveltos.io,resources=clustersummaries,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=config.projectsveltos.io,resources=clustersummaries/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=config.projectsveltos.io,resources=clustersummaries/finalizers,verbs=update;patch
 //+kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterconfigurations,verbs=get;list;watch
 //+kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterconfigurations/status,verbs=get;list;update
 //+kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterreports,verbs=get;list;watch
 //+kubebuilder:rbac:groups=config.projectsveltos.io,resources=clusterreports/status,verbs=get;list;update
+//+kubebuilder:rbac:groups=lib.projectsveltos.io,resources=resourcesummaries,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=lib.projectsveltos.io,resources=resourcesummaries/status,verbs=get;list;update
 //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
 //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
 //+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=kubeadmcontrolplanes,verbs=get;watch;list
@@ -431,10 +435,11 @@ func (r *ClusterSummaryReconciler) SetupWithManager(ctx context.Context, mgr ctr
 	// Later on, in main, we detect that and if CAPI is present WatchForCAPI will be invoked.
 
 	if r.ReportMode == CollectFromManagementCluster {
-		go collectAndProcessResourceSummaries(ctx, mgr.GetClient(), r.ShardKey, r.Version, mgr.GetLogger())
+		go collectAndProcessResourceSummaries(ctx, mgr.GetClient(), getAgentInMgmtCluster(), r.ShardKey,
+			r.Version, mgr.GetLogger())
 	}
 
-	if r.AgentInMgmtCluster {
+	if getAgentInMgmtCluster() {
 		go removeStaleDriftDetectionManager(ctx, r.Logger)
 	}
 
@@ -1135,14 +1140,8 @@ func (r *ClusterSummaryReconciler) removeResourceSummary(ctx context.Context,
 	// ResourceSummary is a Sveltos resource deployed in managed clusters.
 	// Such resources are always created, removed using cluster-admin roles.
 	cs := clusterSummaryScope.ClusterSummary
-	remoteClient, err := clusterproxy.GetKubernetesClient(ctx, r.Client, cs.Spec.ClusterNamespace,
-		cs.Spec.ClusterName, "", "", cs.Spec.ClusterType, logger)
-	if err != nil {
-		return err
-	}
-
-	err = unDeployResourceSummaryInstance(ctx, remoteClient, cs.Spec.ClusterNamespace,
-		cs.Name, logger)
+	err := unDeployResourceSummaryInstance(ctx, cs.Spec.ClusterNamespace, cs.Spec.ClusterName,
+		cs.Name, cs.Spec.ClusterType, logger)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			return nil

controllers/clustersummary_deployer.go

@@ -164,7 +164,7 @@ func (r *ClusterSummaryReconciler) proceedDeployingFeature(ctx context.Context,
 	// Getting here means either feature failed to be deployed or configuration has changed.
 	// Feature must be (re)deployed.
 	options := deployer.Options{HandlerOptions: map[string]string{}}
-	if r.AgentInMgmtCluster {
+	if getAgentInMgmtCluster() {
 		options.HandlerOptions[driftDetectionInMgtmCluster] = "management"
 	}
 

controllers/export_test.go

@@ -153,6 +153,7 @@ var (
 var (
 	DeployDebuggingConfigurationCRD                  = deployDebuggingConfigurationCRD
 	DeployResourceSummaryCRD                         = deployResourceSummaryCRD
+	DeployDriftDetectionManagerInCluster             = deployDriftDetectionManagerInCluster
 	DeployResourceSummaryInCluster                   = deployResourceSummaryInCluster
 	DeployResourceSummaryInstance                    = deployResourceSummaryInstance
 	UpdateDeployedGroupVersionKind                   = updateDeployedGroupVersionKind
@@ -162,8 +163,8 @@ var (
 	GetDriftDetectionNamespaceInMgmtCluster          = getDriftDetectionNamespaceInMgmtCluster
 	TransformDriftExclusionsToPatches                = transformDriftExclusionsToPatches
 
-	GetResourceSummaryNamespace = getResourceSummaryNamespace
-	GetResourceSummaryName      = getResourceSummaryName
+	GetResourceSummaryNamespaceInManagedCluster = getResourceSummaryNamespaceInManagedCluster
+	GetResourceSummaryNameInManagedCluster      = getResourceSummaryNameInManagedCluster
 )
 
 var (

controllers/handlers_utils.go

@@ -1977,6 +1977,12 @@ func getClusterProfileSpecHash(ctx context.Context, clusterSummary *configv1beta
 	// or viceversa) reconcile.
 	config += fmt.Sprintf("%v", clusterProfileSpec.SyncMode)
 
+	// When using ContinuousWithDriftDetection in agentless mode, ResourceSummary instances are now managed in the management cluster.
+	// This addition ensures the ClusterSummary is redeployed due to the change in deployment location.
+	if clusterProfileSpec.SyncMode == configv1beta1.SyncModeContinuousWithDriftDetection && getAgentInMgmtCluster() {
+		config += ("agentless")
+	}
+
 	// If Reloader changes, Reloader needs to be deployed or undeployed
 	// So consider it in the hash
 	config += fmt.Sprintf("%v", clusterProfileSpec.Reloader)

controllers/management_cluster.go

@@ -32,6 +32,7 @@ var (
 	luaConfigMap            string
 	capiOnboardAnnotation   string
 	driftDetectionRegistry  string
+	agentInMgmtCluster      bool
 )
 
 func SetManagementClusterAccess(c client.Client, config *rest.Config) {
@@ -51,6 +52,14 @@ func SetCAPIOnboardAnnotation(key string) {
 	capiOnboardAnnotation = key
 }
 
+func SetDriftDetectionRegistry(reg string) {
+	driftDetectionRegistry = reg
+}
+
+func SetAgentInMgmtCluster(isInMgmtCluster bool) {
+	agentInMgmtCluster = isInMgmtCluster
+}
+
 func getManagementClusterConfig() *rest.Config {
 	return managementClusterConfig
 }
@@ -71,8 +80,12 @@ func getCAPIOnboardAnnotation() string {
 	return capiOnboardAnnotation
 }
 
-func SetDriftDetectionRegistry(reg string) {
-	driftDetectionRegistry = reg
+func getDriftDetectionRegistry() string {
+	return driftDetectionRegistry
+}
+
+func getAgentInMgmtCluster() bool {
+	return agentInMgmtCluster
 }
 
 func collectDriftDetectionConfigMap(ctx context.Context) (*corev1.ConfigMap, error) {
@@ -100,7 +113,3 @@ func collectLuaConfigMap(ctx context.Context) (*corev1.ConfigMap, error) {
 
 	return configMap, nil
 }
-
-func getDriftDetectionRegistry() string {
-	return driftDetectionRegistry
-}