Add hub_template_active_watches metric

Makefile

@@ -221,7 +221,7 @@ kind-bootstrap-cluster-dev: kind-create-cluster install-crds install-resources
 .PHONY: kind-deploy-controller
 kind-deploy-controller: manifests
 	@echo installing $(IMG)
-	kubectl create ns $(KIND_NAMESPACE)
+	-kubectl create ns $(KIND_NAMESPACE)
 	kubectl apply -f deploy/operator.yaml -n $(KIND_NAMESPACE)
 
 .PHONY: kind-deploy-controller-dev

controllers/propagator/metric.go

@@ -8,11 +8,20 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
-var roothandlerMeasure = prometheus.NewHistogram(prometheus.HistogramOpts{
-	Name: "ocm_handle_root_policy_duration_seconds",
-	Help: "Time the handleRootPolicy function takes to complete.",
-})
+var (
+	hubTemplateActiveWatchesMetric = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "hub_templates_active_watches",
+			Help: "The number of active watch API requests for Hub policy templates",
+		},
+	)
+	roothandlerMeasure = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name: "ocm_handle_root_policy_duration_seconds_bucket",
+		Help: "Time the handleRootPolicy function takes to complete.",
+	})
+)
 
 func init() {
 	metrics.Registry.MustRegister(roothandlerMeasure)
+	metrics.Registry.MustRegister(hubTemplateActiveWatchesMetric)
 }

controllers/propagator/policy_controller.go

@@ -98,6 +98,14 @@ type PolicyReconciler struct {
 func (r *PolicyReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
 	log := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
 
+	// Set the hub template watch metric after reconcile
+	defer func() {
+		hubTempWatches := r.DynamicWatcher.GetWatchCount()
+		log.V(3).Info("Setting hub template watch metric", "value", hubTempWatches)
+
+		hubTemplateActiveWatchesMetric.Set(float64(hubTempWatches))
+	}()
+
 	log.Info("Reconciling the policy")
 
 	// Fetch the Policy instance

controllers/propagator/propagation.go

@@ -187,7 +187,8 @@ func (r *PolicyReconciler) cleanUpPolicy(instance *policiesv1.Policy) error {
 		return nil
 	}
 
-	log.V(2).Info("Deleting %d replicated policies because root policy was deleted", len(replicatedPlcList.Items))
+	log.V(2).Info(
+		"Deleting replicated policies because root policy was deleted", "count", len(replicatedPlcList.Items))
 
 	policiesChan := make(chan policiesv1.Policy, len(replicatedPlcList.Items))
 	deletionResultsChan := make(chan deletionResult, len(replicatedPlcList.Items))
@@ -532,6 +533,7 @@ func (r *PolicyReconciler) recordWarning(instance *policiesv1.Policy, msgPrefix
 // method because it makes the retries more targeted and prevents race conditions, such as a
 // placement binding getting updated, from causing inconsistencies.
 func (r *PolicyReconciler) handleRootPolicy(instance *policiesv1.Policy) error {
+	// Generate a metric for elapsed handling time for each policy
 	entryTS := time.Now()
 	defer func() {
 		now := time.Now()

test/e2e/case12_encryptionkeys_controller_test.go

@@ -92,7 +92,7 @@ var _ = Describe("Test policy encryption key rotation", func() {
 		Expect(err).Should(BeNil())
 	})
 
-	It("should have roated the key in the "+EncryptionKeySecret+" secret", func() {
+	It("should have rotated the key in the "+EncryptionKeySecret+" secret", func() {
 		var secret *unstructured.Unstructured
 		Eventually(func() interface{} {
 			secret = utils.GetWithTimeout(

test/e2e/case14_templates_metrics_test.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2021 Red Hat, Inc.
+// Copyright Contributors to the Open Cluster Management project
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"open-cluster-management.io/governance-policy-propagator/test/utils"
+)
+
+var _ = Describe("Test policy template metrics", Ordered, func() {
+	const (
+		policyName = "case9-test-policy"
+		policyYaml = "../resources/case9_templates/case9-test-policy.yaml"
+	)
+
+	Describe("Create policy, placement and referenced resource in ns:"+testNamespace, func() {
+		It("should be created in user ns", func() {
+			By("Creating " + policyYaml)
+			utils.Kubectl("apply",
+				"-f", policyYaml,
+				"-n", testNamespace)
+			plc := utils.GetWithTimeout(
+				clientHubDynamic, gvrPolicy, policyName, testNamespace, true, defaultTimeoutSeconds,
+			)
+			Expect(plc).NotTo(BeNil())
+		})
+
+		It("should resolve templates and propagate to cluster ns managed1", func() {
+			By("Patching test-policy-plr with decision of cluster managed1")
+			plr := utils.GetWithTimeout(
+				clientHubDynamic, gvrPlacementRule, policyName+"-plr", testNamespace,
+				true, defaultTimeoutSeconds,
+			)
+			plr.Object["status"] = utils.GeneratePlrStatus("managed1")
+			_, err := clientHubDynamic.Resource(gvrPlacementRule).Namespace(testNamespace).UpdateStatus(
+				context.TODO(), plr, metav1.UpdateOptions{},
+			)
+			Expect(err).To(BeNil())
+			plc := utils.GetWithTimeout(
+				clientHubDynamic, gvrPolicy, testNamespace+"."+policyName, "managed1",
+				true, defaultTimeoutSeconds,
+			)
+			Expect(plc).ToNot(BeNil())
+
+			yamlPlc := utils.ParseYaml(case9ReplicatedPolicyYamlM1)
+			Eventually(func() interface{} {
+				replicatedPlc := utils.GetWithTimeout(
+					clientHubDynamic,
+					gvrPolicy,
+					testNamespace+"."+policyName,
+					"managed1",
+					true,
+					defaultTimeoutSeconds,
+				)
+
+				return replicatedPlc.Object["spec"]
+			}, defaultTimeoutSeconds, 1).Should(utils.SemanticEqual(yamlPlc.Object["spec"]))
+		})
+
+		It("should correctly report root policy hub template watches when propagated", func() {
+			By("Checking metric endpoint for root policy hub template watches")
+			Eventually(func() interface{} {
+				return utils.GetMetrics("hub_templates_active_watches", "\"[0-9]\"")
+			}, defaultTimeoutSeconds, 1).Should(Equal([]string{"2"}))
+		})
+
+		cleanup := func() {
+			utils.Kubectl("delete",
+				"-f", policyYaml,
+				"-n", testNamespace)
+			opt := metav1.ListOptions{}
+			utils.ListWithTimeout(clientHubDynamic, gvrPolicy, opt, 0, false, defaultTimeoutSeconds)
+		}
+
+		It("should clean up", cleanup)
+
+		It("should report root policy 0 hub template watches after clean up", func() {
+			By("Checking metric endpoint for root policy hub template watches")
+			Eventually(func() interface{} {
+				return utils.GetMetrics("hub_templates_active_watches", "\"[0-9]\"")
+			}, defaultTimeoutSeconds, 1).Should(Equal([]string{"0"}))
+		})
+
+		AfterAll(cleanup)
+	})
+})