metrics: register times for the reconcile loops

Author: Hector Fernandez

docs/keps/20200302-kubefed-metrics.md

@@ -44,7 +44,7 @@ and consumed from Kubefed to create dashboards and better understand this engine
 ## Motivation
 
 We aim to define a generic strategy on how to identify, consume and expose
-metrics from our core components in Kommander.
+custom Kubefed metrics.
 
 
 ### Goals
@@ -115,7 +115,7 @@ of the different propagated resources. The label `action` will hold the `create`
 
 Regarding cluster join/unjoin operations, these metrics are also convenient to register:
 
-* `joinedcluster_total`: a gauge metric that holds the number joined clusters.
+* `joined_cluster_total`: a gauge metric that holds the number joined clusters.
 
 * `join_cluster_duration_seconds`: this `histogram` metric holds the duration in seconds of the join cluster action.
 

pkg/controller/federatedtypeconfig/controller.go

@@ -19,6 +19,7 @@ package federatedtypeconfig
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -36,6 +37,7 @@ import (
 	statuscontroller "sigs.k8s.io/kubefed/pkg/controller/status"
 	synccontroller "sigs.k8s.io/kubefed/pkg/controller/sync"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 const finalizer string = "core.kubefed.io/federated-type-config"
@@ -128,6 +130,7 @@ func (c *Controller) Run(stopChan <-chan struct{}) {
 
 func (c *Controller) reconcile(qualifiedName util.QualifiedName) util.ReconciliationStatus {
 	key := qualifiedName.String()
+	defer metrics.UpdateControllerReconcileDurationFromStart("federatedtypeconfigcontroller", time.Now())
 
 	klog.V(3).Infof("Running reconcile FederatedTypeConfig for %q", key)
 

pkg/controller/ingressdns/controller.go

@@ -38,6 +38,7 @@ import (
 	dnsv1a1 "sigs.k8s.io/kubefed/pkg/apis/multiclusterdns/v1alpha1"
 	genericclient "sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 const (
@@ -201,6 +202,8 @@ func (c *Controller) reconcileOnClusterChange() {
 }
 
 func (c *Controller) reconcile(qualifiedName util.QualifiedName) util.ReconciliationStatus {
+	defer metrics.UpdateControllerReconcileDurationFromStart("ingressdnscontroller", time.Now())
+
 	if !c.isSynced() {
 		return util.StatusNotSynced
 	}

pkg/controller/kubefedcluster/controller.go

@@ -241,9 +241,10 @@ func (cc *ClusterController) updateClusterStatus() error {
 
 func (cc *ClusterController) updateIndividualClusterStatus(cluster *fedv1b1.KubeFedCluster,
 	storedData *ClusterData, wg *sync.WaitGroup) {
+	defer metrics.ClusterHealthStatusDurationFromStart(time.Now())
+
 	clusterClient := storedData.clusterKubeClient
 
-	clusterHealthStatusStart := time.Now()
 	currentClusterStatus, err := clusterClient.GetClusterHealthStatus()
 	if err != nil {
 		cc.RecordError(cluster, "RetrievingClusterHealthFailed", errors.Wrap(err, "Failed to retrieve health of the cluster"))
@@ -261,8 +262,6 @@ func (cc *ClusterController) updateIndividualClusterStatus(cluster *fedv1b1.Kube
 		klog.Warningf("Failed to update the status of cluster %q: %v", cluster.Name, err)
 	}
 
-	metrics.ClusterHealthStatusDurationFromStart(clusterHealthStatusStart)
-
 	wg.Done()
 }
 

pkg/controller/schedulingmanager/controller.go

@@ -17,6 +17,8 @@ limitations under the License.
 package schedulingmanager
 
 import (
+	"time"
+
 	"github.com/pkg/errors"
 
 	"k8s.io/apimachinery/pkg/util/runtime"
@@ -27,6 +29,7 @@ import (
 	corev1b1 "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1"
 	"sigs.k8s.io/kubefed/pkg/controller/schedulingpreference"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 	"sigs.k8s.io/kubefed/pkg/schedulingtypes"
 )
 
@@ -141,6 +144,8 @@ func (c *SchedulingManager) shutdown() {
 }
 
 func (c *SchedulingManager) reconcile(qualifiedName util.QualifiedName) util.ReconciliationStatus {
+	defer metrics.UpdateControllerReconcileDurationFromStart("schedulingmanagercontroller", time.Now())
+
 	key := qualifiedName.String()
 
 	klog.V(3).Infof("Running reconcile FederatedTypeConfig %q in scheduling manager", key)

pkg/controller/schedulingpreference/controller.go

@@ -35,6 +35,7 @@ import (
 
 	fedv1b1 "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 	"sigs.k8s.io/kubefed/pkg/schedulingtypes"
 )
 
@@ -191,6 +192,8 @@ func (s *SchedulingPreferenceController) reconcileOnClusterChange() {
 }
 
 func (s *SchedulingPreferenceController) reconcile(qualifiedName util.QualifiedName) util.ReconciliationStatus {
+	defer metrics.UpdateControllerReconcileDurationFromStart("schedulingpreferencecontroller", time.Now())
+
 	if !s.isSynced() {
 		return util.StatusNotSynced
 	}

pkg/controller/servicedns/controller.go

@@ -37,6 +37,7 @@ import (
 	dnsv1a1 "sigs.k8s.io/kubefed/pkg/apis/multiclusterdns/v1alpha1"
 	genericclient "sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 const (
@@ -257,6 +258,8 @@ func (c *Controller) reconcileOnClusterChange() {
 }
 
 func (c *Controller) reconcile(qualifiedName util.QualifiedName) util.ReconciliationStatus {
+	defer metrics.UpdateControllerReconcileDurationFromStart("servicednscontroller", time.Now())
+
 	if !c.isSynced() {
 		return util.StatusNotSynced
 	}

pkg/controller/status/controller.go

@@ -37,6 +37,7 @@ import (
 	fedv1b1 "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1"
 	genericclient "sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 const (
@@ -231,6 +232,8 @@ func (s *KubeFedStatusController) reconcileOnClusterChange() {
 }
 
 func (s *KubeFedStatusController) reconcile(qualifiedName util.QualifiedName) util.ReconciliationStatus {
+	defer metrics.UpdateControllerReconcileDurationFromStart("statuscontroller", time.Now())
+
 	if !s.isSynced() {
 		return util.StatusNotSynced
 	}

pkg/controller/util/federated_informer.go

@@ -322,11 +322,10 @@ func (f *federatedInformerImpl) Start() {
 
 // GetClientForCluster returns a client for the cluster, if present.
 func (f *federatedInformerImpl) GetClientForCluster(clusterName string) (generic.Client, error) {
+	defer metrics.ClusterClientConnectionDurationFromStart(time.Now())
 	f.Lock()
 	defer f.Unlock()
 
-	clientConnectionStart := time.Now()
-
 	// return cached client if one exists (to prevent frequent secret retrieval and rest discovery)
 	if client, ok := f.clusterClients[clusterName]; ok {
 		return client, nil
@@ -341,7 +340,6 @@ func (f *federatedInformerImpl) GetClientForCluster(clusterName string) (generic
 	}
 	f.clusterClients[clusterName] = client
 
-	metrics.ClusterClientConnectionDurationFromStart(clientConnectionStart)
 	return client, nil
 }
 

pkg/metrics/metrics.go

@@ -34,7 +34,7 @@ var (
 
 	joinedClusterTotal = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Name: "joinedcluster_total",
+			Name: "joined_cluster_total",
 			Help: "Number of total joined clusters.",
 		},
 	)