🚀feat(cyclops-ctrl): adds prometheus metrics to reconciler

cyclops-ctrl/cmd/main/main.go

@@ -126,6 +126,7 @@ func main() {
 		k8sClient,
 		renderer,
 		telemetryClient,
+		monitor,
 	)).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Module")
 		os.Exit(1)

cyclops-ctrl/internal/modulecontroller/module_controller.go

@@ -38,6 +38,7 @@ import (
 
 	cyclopsv1alpha1 "github.com/cyclops-ui/cyclops/cyclops-ctrl/api/v1alpha1"
 	"github.com/cyclops-ui/cyclops/cyclops-ctrl/internal/models"
+	"github.com/cyclops-ui/cyclops/cyclops-ctrl/internal/prometheus"
 	"github.com/cyclops-ui/cyclops/cyclops-ctrl/internal/telemetry"
 	templaterepo "github.com/cyclops-ui/cyclops/cyclops-ctrl/internal/template"
 	"github.com/cyclops-ui/cyclops/cyclops-ctrl/internal/template/render"
@@ -54,6 +55,7 @@ type ModuleReconciler struct {
 	renderer         *render.Renderer
 
 	telemetryClient telemetry.Client
+	monitor         prometheus.Monitor
 	logger          logr.Logger
 }
 
@@ -64,6 +66,7 @@ func NewModuleReconciler(
 	kubernetesClient *k8sclient.KubernetesClient,
 	renderer *render.Renderer,
 	telemetryClient telemetry.Client,
+	monitor prometheus.Monitor,
 ) *ModuleReconciler {
 	return &ModuleReconciler{
 		Client:           client,
@@ -72,6 +75,7 @@ func NewModuleReconciler(
 		kubernetesClient: kubernetesClient,
 		renderer:         renderer,
 		telemetryClient:  telemetryClient,
+		monitor:          monitor,
 		logger:           ctrl.Log.WithName("reconciler"),
 	}
 }
@@ -92,6 +96,13 @@ func NewModuleReconciler(
 func (r *ModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	_ = log.FromContext(ctx)
 	r.telemetryClient.ModuleReconciliation()
+	r.monitor.OnReconciliation()
+
+	startTime := time.Now()
+
+	defer func(startTime time.Time) {
+		r.monitor.ObserveReconciliationDuration(time.Since(startTime).Seconds())
+	}(startTime)
 
 	var module cyclopsv1alpha1.Module
 	err := r.Get(ctx, req.NamespacedName, &module)
@@ -100,6 +111,7 @@ func (r *ModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		resources, err := r.kubernetesClient.GetResourcesForModule(req.Name)
 		if err != nil {
 			r.logger.Error(err, "error on get module resources", "namespaced name", req.NamespacedName)
+			r.monitor.OnFailedReconciliation()
 			return ctrl.Result{}, err
 		}
 
@@ -119,6 +131,7 @@ func (r *ModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		return ctrl.Result{}, nil
 	}
 	if err != nil {
+		r.monitor.OnFailedReconciliation()
 		return ctrl.Result{}, err
 	}
 
@@ -142,6 +155,8 @@ func (r *ModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			return ctrl.Result{}, err
 		}
 
+		r.monitor.OnFailedReconciliation()
+
 		return ctrl.Result{}, err
 	}
 
@@ -153,10 +168,13 @@ func (r *ModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			return ctrl.Result{}, err
 		}
 
+		r.monitor.OnFailedReconciliation()
+
 		return ctrl.Result{}, err
 	}
 
 	if len(installErrors) != 0 {
+		r.monitor.OnFailedReconciliation()
 		return ctrl.Result{}, r.setStatus(
 			ctx,
 			module,

cyclops-ctrl/internal/prometheus/handler.go

@@ -17,6 +17,11 @@ type Monitor struct {
 	CacheCostAdded   prometheus.Gauge
 	CacheKeysEvicted prometheus.Gauge
 	CacheCostEvicted prometheus.Gauge
+
+	// Reconciler Metrics
+	ReconciliationDuration      prometheus.Histogram
+	ReconciliationCounter       prometheus.Counter
+	FailedReconciliationCounter prometheus.Counter
 }
 
 func NewMonitor(logger logr.Logger) (Monitor, error) {
@@ -57,6 +62,22 @@ func NewMonitor(logger logr.Logger) (Monitor, error) {
 			Help:      "No of cache cost evicted",
 			Namespace: "cyclops",
 		}),
+		ReconciliationDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Name:      "reconciliation_duration_seconds",
+			Help:      "Duration of reconciler",
+			Namespace: "cyclops",
+			Buckets:   prometheus.DefBuckets,
+		}),
+		ReconciliationCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Name:      "no_of_reconciliations",
+			Help:      "No of reconciliations",
+			Namespace: "cyclops",
+		}),
+		FailedReconciliationCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Name:      "failed_reconciliations",
+			Help:      "No of failed reconciliations",
+			Namespace: "cyclops",
+		}),
 	}
 
 	metricsList :=
@@ -68,6 +89,9 @@ func NewMonitor(logger logr.Logger) (Monitor, error) {
 			m.CacheCostAdded,
 			m.CacheKeysEvicted,
 			m.CacheCostEvicted,
+			m.ReconciliationDuration,
+			m.ReconciliationCounter,
+			m.FailedReconciliationCounter,
 		}
 
 	for _, metric := range metricsList {
@@ -88,6 +112,18 @@ func (m *Monitor) DecModule() {
 	m.ModulesDeployed.Dec()
 }
 
+func (m *Monitor) OnReconciliation() {
+	m.ReconciliationCounter.Inc()
+}
+
+func (m *Monitor) OnFailedReconciliation() {
+	m.FailedReconciliationCounter.Inc()
+}
+
+func (m *Monitor) ObserveReconciliationDuration(duration float64) {
+	m.ReconciliationDuration.Observe(duration)
+}
+
 func (m *Monitor) UpdateCacheMetrics(cache *ristretto.Cache) {
 	cacheMetrics := cache.Metrics