From b4588f34ed866ed49eb74bb27cbc1e91035ecb88 Mon Sep 17 00:00:00 2001 From: Hidetake Iwata Date: Fri, 19 Mar 2021 14:01:30 +0900 Subject: [PATCH 1/3] Add metrics of RunnerDeployment and HRA --- .../horizontalrunnerautoscaler_controller.go | 5 ++ .../metrics/horizontalrunnerautoscaler.go | 67 +++++++++++++++++++ controllers/metrics/metrics.go | 14 ++++ controllers/metrics/runnerdeployment.go | 37 ++++++++++ controllers/runnerdeployment_controller.go | 3 + 5 files changed, 126 insertions(+) create mode 100644 controllers/metrics/horizontalrunnerautoscaler.go create mode 100644 controllers/metrics/metrics.go create mode 100644 controllers/metrics/runnerdeployment.go diff --git a/controllers/horizontalrunnerautoscaler_controller.go b/controllers/horizontalrunnerautoscaler_controller.go index a64b83b13f..e2c2f54835 100644 --- a/controllers/horizontalrunnerautoscaler_controller.go +++ b/controllers/horizontalrunnerautoscaler_controller.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/summerwind/actions-runner-controller/api/v1alpha1" + "github.com/summerwind/actions-runner-controller/controllers/metrics" ) const ( @@ -73,6 +74,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl return ctrl.Result{}, nil } + metrics.SetHorizontalRunnerAutoscalerSpec(hra.ObjectMeta, hra.Spec) + var rd v1alpha1.RunnerDeployment if err := r.Get(ctx, types.NamespacedName{ Namespace: req.Namespace, @@ -145,6 +148,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl } if updated != nil { + metrics.SetHorizontalRunnerAutoscalerStatus(updated.ObjectMeta, updated.Status) + if err := r.Status().Patch(ctx, updated, client.MergeFrom(&hra)); err != nil { return ctrl.Result{}, fmt.Errorf("patching horizontalrunnerautoscaler status to add cache entry: %w", err) } diff --git a/controllers/metrics/horizontalrunnerautoscaler.go b/controllers/metrics/horizontalrunnerautoscaler.go new file mode 100644 index 0000000000..c7c85f2be8 --- /dev/null +++ b/controllers/metrics/horizontalrunnerautoscaler.go @@ -0,0 +1,67 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/summerwind/actions-runner-controller/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + hraName = "hra_name" + hraNamespace = "hra_namespace" +) + +var ( + horizontalRunnerAutoscalerMetrics = []prometheus.Collector{ + horizontalRunnerAutoscalerMinReplicas, + horizontalRunnerAutoscalerMaxReplicas, + horizontalRunnerAutoscalerDesiredReplicas, + } +) + +var ( + horizontalRunnerAutoscalerMinReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontal_runner_autoscaler_min_replicas", + Help: "minReplicas of HorizontalRunnerAutoscaler", + }, + []string{hraName, hraNamespace}, + ) + horizontalRunnerAutoscalerMaxReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontal_runner_autoscaler_max_replicas", + Help: "maxReplicas of HorizontalRunnerAutoscaler", + }, + []string{hraName, hraNamespace}, + ) + horizontalRunnerAutoscalerDesiredReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "horizontal_runner_autoscaler_desired_replicas", + Help: "desiredReplicas of HorizontalRunnerAutoscaler", + }, + []string{hraName, hraNamespace}, + ) +) + +func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) { + labels := prometheus.Labels{ + hraName: o.Name, + hraNamespace: o.Namespace, + } + if spec.MaxReplicas != nil { + horizontalRunnerAutoscalerMaxReplicas.With(labels).Set(float64(*spec.MaxReplicas)) + } + if spec.MinReplicas != nil { + horizontalRunnerAutoscalerMinReplicas.With(labels).Set(float64(*spec.MinReplicas)) + } +} + +func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.HorizontalRunnerAutoscalerStatus) { + labels := prometheus.Labels{ + hraName: o.Name, + hraNamespace: o.Namespace, + } + if status.DesiredReplicas != nil { + horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas)) + } +} diff --git a/controllers/metrics/metrics.go b/controllers/metrics/metrics.go new file mode 100644 index 0000000000..113df6e09c --- /dev/null +++ b/controllers/metrics/metrics.go @@ -0,0 +1,14 @@ +// Package metrics provides the metrics of custom resources such as HRA. +// +// This depends on the metrics exporter of kubebuilder. +// See https://book.kubebuilder.io/reference/metrics.html for details. +package metrics + +import ( + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +func init() { + metrics.Registry.MustRegister(runnerDeploymentMetrics...) + metrics.Registry.MustRegister(horizontalRunnerAutoscalerMetrics...) +} diff --git a/controllers/metrics/runnerdeployment.go b/controllers/metrics/runnerdeployment.go new file mode 100644 index 0000000000..614d6e6dc6 --- /dev/null +++ b/controllers/metrics/runnerdeployment.go @@ -0,0 +1,37 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/summerwind/actions-runner-controller/api/v1alpha1" +) + +const ( + rdName = "rd_name" + rdNamespace = "rd_namespace" +) + +var ( + runnerDeploymentMetrics = []prometheus.Collector{ + runnerDeploymentReplicas, + } +) + +var ( + runnerDeploymentReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "runner_deployment_replicas", + Help: "replicas of RunnerDeployment", + }, + []string{rdName, rdNamespace}, + ) +) + +func SetRunnerDeployment(rd v1alpha1.RunnerDeployment) { + labels := prometheus.Labels{ + rdName: rd.Name, + rdNamespace: rd.Namespace, + } + if rd.Spec.Replicas != nil { + runnerDeploymentReplicas.With(labels).Set(float64(*rd.Spec.Replicas)) + } +} diff --git a/controllers/runnerdeployment_controller.go b/controllers/runnerdeployment_controller.go index ed5d84e01f..04a1afb76d 100644 --- a/controllers/runnerdeployment_controller.go +++ b/controllers/runnerdeployment_controller.go @@ -38,6 +38,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/summerwind/actions-runner-controller/api/v1alpha1" + "github.com/summerwind/actions-runner-controller/controllers/metrics" ) const ( @@ -77,6 +78,8 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e return ctrl.Result{}, nil } + metrics.SetRunnerDeployment(rd) + var myRunnerReplicaSetList v1alpha1.RunnerReplicaSetList if err := r.List(ctx, &myRunnerReplicaSetList, client.InNamespace(req.Namespace), client.MatchingFields{runnerSetOwnerKey: req.Name}); err != nil { return ctrl.Result{}, err From 88877faab01b7164208bc23794a835cd92a6cb65 Mon Sep 17 00:00:00 2001 From: Hidetake Iwata Date: Fri, 19 Mar 2021 15:17:04 +0900 Subject: [PATCH 2/3] Change label names to kube-state-metrics style --- controllers/metrics/horizontalrunnerautoscaler.go | 4 ++-- controllers/metrics/runnerdeployment.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/metrics/horizontalrunnerautoscaler.go b/controllers/metrics/horizontalrunnerautoscaler.go index c7c85f2be8..f1bf75a74b 100644 --- a/controllers/metrics/horizontalrunnerautoscaler.go +++ b/controllers/metrics/horizontalrunnerautoscaler.go @@ -7,8 +7,8 @@ import ( ) const ( - hraName = "hra_name" - hraNamespace = "hra_namespace" + hraName = "horizontalrunnerautoscaler" + hraNamespace = "namespace" ) var ( diff --git a/controllers/metrics/runnerdeployment.go b/controllers/metrics/runnerdeployment.go index 614d6e6dc6..42c389e3cd 100644 --- a/controllers/metrics/runnerdeployment.go +++ b/controllers/metrics/runnerdeployment.go @@ -6,8 +6,8 @@ import ( ) const ( - rdName = "rd_name" - rdNamespace = "rd_namespace" + rdName = "runnerdeployment" + rdNamespace = "namespace" ) var ( From 44dc6b05807c1cbb25354659b930075d3e46e026 Mon Sep 17 00:00:00 2001 From: Hidetake Iwata Date: Fri, 19 Mar 2021 16:06:46 +0900 Subject: [PATCH 3/3] Change metric names to kube-state-metrics style --- controllers/metrics/horizontalrunnerautoscaler.go | 6 +++--- controllers/metrics/runnerdeployment.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/metrics/horizontalrunnerautoscaler.go b/controllers/metrics/horizontalrunnerautoscaler.go index f1bf75a74b..338de194d8 100644 --- a/controllers/metrics/horizontalrunnerautoscaler.go +++ b/controllers/metrics/horizontalrunnerautoscaler.go @@ -22,21 +22,21 @@ var ( var ( horizontalRunnerAutoscalerMinReplicas = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "horizontal_runner_autoscaler_min_replicas", + Name: "horizontalrunnerautoscaler_spec_min_replicas", Help: "minReplicas of HorizontalRunnerAutoscaler", }, []string{hraName, hraNamespace}, ) horizontalRunnerAutoscalerMaxReplicas = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "horizontal_runner_autoscaler_max_replicas", + Name: "horizontalrunnerautoscaler_spec_max_replicas", Help: "maxReplicas of HorizontalRunnerAutoscaler", }, []string{hraName, hraNamespace}, ) horizontalRunnerAutoscalerDesiredReplicas = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "horizontal_runner_autoscaler_desired_replicas", + Name: "horizontalrunnerautoscaler_status_desired_replicas", Help: "desiredReplicas of HorizontalRunnerAutoscaler", }, []string{hraName, hraNamespace}, diff --git a/controllers/metrics/runnerdeployment.go b/controllers/metrics/runnerdeployment.go index 42c389e3cd..c062a1cc24 100644 --- a/controllers/metrics/runnerdeployment.go +++ b/controllers/metrics/runnerdeployment.go @@ -19,7 +19,7 @@ var ( var ( runnerDeploymentReplicas = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "runner_deployment_replicas", + Name: "runnerdeployment_spec_replicas", Help: "replicas of RunnerDeployment", }, []string{rdName, rdNamespace},