Add hub_template_active_watches metric

Gauge that records the current number of active watches.

ref: stolostron/backlog#25866
Signed-off-by: Dale Haiducek <[email protected]>

Author: dhaiducek

controllers/propagator/metric.go

@@ -8,11 +8,20 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
-var roothandlerMeasure = prometheus.NewHistogram(prometheus.HistogramOpts{
-	Name: "ocm_handle_root_policy_duration_seconds",
-	Help: "Time the handleRootPolicy function takes to complete.",
-})
+var (
+	hubTemplateActiveWatchesMetric = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "hub_templates_active_watches",
+			Help: "The number of active watch API requests for Hub policy templates",
+		},
+	)
+	roothandlerMeasure = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name: "ocm_handle_root_policy_duration_seconds_bucket",
+		Help: "Time the handleRootPolicy function takes to complete.",
+	})
+)
 
 func init() {
 	metrics.Registry.MustRegister(roothandlerMeasure)
+	metrics.Registry.MustRegister(hubTemplateActiveWatchesMetric)
 }

controllers/propagator/policy_controller.go

@@ -98,6 +98,14 @@ type PolicyReconciler struct {
 func (r *PolicyReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
 	log := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
 
+	// Set the hub template watch metric after reconcile
+	defer func() {
+		hubTempWatches := r.DynamicWatcher.GetWatchCount()
+		log.V(3).Info("Setting hub template watch metric", "value", hubTempWatches)
+
+		hubTemplateActiveWatchesMetric.Set(float64(hubTempWatches))
+	}()
+
 	log.Info("Reconciling the policy")
 
 	// Fetch the Policy instance

test/e2e/case14_templates_metrics_test.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Red Hat, Inc.
+// Copyright Contributors to the Open Cluster Management project
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"open-cluster-management.io/governance-policy-propagator/test/utils"
+)
+
+var _ = Describe("Test policy template metrics", Ordered, func() {
+	const (
+		policyName = "case9-test-policy"
+		policyYaml = "../resources/case9_templates/case9-test-policy.yaml"
+	)
+
+	Describe("Create policy, placement and referenced resource in ns:"+testNamespace, func() {
+		It("should be created in user ns", func() {
+			By("Creating " + policyYaml)
+			utils.Kubectl("apply",
+				"-f", policyYaml,
+				"-n", testNamespace)
+			plc := utils.GetWithTimeout(
+				clientHubDynamic, gvrPolicy, policyName, testNamespace, true, defaultTimeoutSeconds,
+			)
+			Expect(plc).NotTo(BeNil())
+		})
+
+		It("should resolve templates and propagate to cluster ns managed1", func() {
+			By("Patching test-policy-plr with decision of cluster managed1")
+			plr := utils.GetWithTimeout(
+				clientHubDynamic, gvrPlacementRule, policyName+"-plr", testNamespace,
+				true, defaultTimeoutSeconds,
+			)
+			plr.Object["status"] = utils.GeneratePlrStatus("managed1")
+			_, err := clientHubDynamic.Resource(gvrPlacementRule).Namespace(testNamespace).UpdateStatus(
+				context.TODO(), plr, metav1.UpdateOptions{},
+			)
+			Expect(err).To(BeNil())
+			plc := utils.GetWithTimeout(
+				clientHubDynamic, gvrPolicy, testNamespace+"."+policyName, "managed1",
+				true, defaultTimeoutSeconds,
+			)
+			Expect(plc).ToNot(BeNil())
+
+			yamlPlc := utils.ParseYaml(case9ReplicatedPolicyYamlM1)
+			Eventually(func() interface{} {
+				replicatedPlc := utils.GetWithTimeout(
+					clientHubDynamic,
+					gvrPolicy,
+					testNamespace+"."+policyName,
+					"managed1",
+					true,
+					defaultTimeoutSeconds,
+				)
+
+				return replicatedPlc.Object["spec"]
+			}, defaultTimeoutSeconds, 1).Should(utils.SemanticEqual(yamlPlc.Object["spec"]))
+		})
+
+		It("should correctly report root policy hub template watches when propagated", func() {
+			By("Checking metric endpoint for root policy hub template watches")
+			Eventually(func() interface{} {
+				return utils.GetMetrics("hub_templates_active_watches", "\"[0-9]\"")
+			}, defaultTimeoutSeconds, 1).Should(Equal([]string{"2"}))
+		})
+
+		It("should clean up", func() {
+			utils.Kubectl("delete",
+				"-f", policyYaml,
+				"-n", testNamespace)
+			opt := metav1.ListOptions{}
+			utils.ListWithTimeout(clientHubDynamic, gvrPolicy, opt, 0, false, defaultTimeoutSeconds)
+		})
+
+		It("should report root policy 0 hub template watches after clean up", func() {
+			By("Checking metric endpoint for root policy hub template watches")
+			Eventually(func() interface{} {
+				return utils.GetMetrics("hub_templates_active_watches", "\"[0-9]\"")
+			}, defaultTimeoutSeconds, 1).Should(Equal([]string{"0"}))
+		})
+	})
+})