Skip to content

Commit 140c6be

Browse files
jayanthvnjayanthvn
committed
Prometheus metrics scraping from CNI metrics helper
1 parent 3a799bc commit 140c6be

File tree

14 files changed

+482
-401
lines changed

14 files changed

+482
-401
lines changed

‎charts/cni-metrics-helper/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ The following table lists the configurable parameters for this chart and their d
5151
| image.account | ECR repository account number | 602401143452 |
5252
| image.domain | ECR repository domain | amazonaws.com |
5353
| env.USE_CLOUDWATCH | Whether to export CNI metrics to CloudWatch | true |
54+
| env.USE_PROMETHEUS | Whether to export CNI metrics to Prometheus | false |
5455
| env.AWS_CLUSTER_ID | ID of the cluster to use when exporting metrics to CloudWatch | default |
5556
| env.AWS_VPC_K8S_CNI_LOGLEVEL | Log verbosity level (ie. FATAL, ERROR, WARN, INFO, DEBUG) | INFO |
5657
| env.METRIC_UPDATE_INTERVAL | Interval at which to update CloudWatch metrics, in seconds. | |

‎charts/cni-metrics-helper/values.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ image:
1212

1313
env:
1414
USE_CLOUDWATCH: "true"
15+
USE_PROMETHEUS: "false"
1516
AWS_CLUSTER_ID: ""
1617
AWS_VPC_K8S_CNI_LOGLEVEL: "INFO"
1718

‎cmd/aws-k8s-agent/main.go

+11-2
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,17 @@ import (
2222
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/eventrecorder"
2323
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
2424
"github.com/aws/amazon-vpc-cni-k8s/pkg/version"
25+
"github.com/aws/amazon-vpc-cni-k8s/utils"
26+
metrics "github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics"
2527
)
2628

2729
const (
2830
appName = "aws-node"
31+
// metricsPort is the port for prometheus metrics
32+
metricsPort = 61678
33+
34+
// Environment variable to disable the metrics endpoint on 61678
35+
envDisableMetrics = "DISABLE_METRICS"
2936
)
3037

3138
func main() {
@@ -67,8 +74,10 @@ func _main() int {
6774
// Pool manager
6875
go ipamContext.StartNodeIPPoolManager()
6976

70-
// Prometheus metrics
71-
go ipamContext.ServeMetrics()
77+
if utils.GetBoolAsStringEnvVar(envDisableMetrics, false) {
78+
// Prometheus metrics
79+
go metrics.ServeMetrics(metricsPort)
80+
}
7281

7382
// CNI introspection endpoints
7483
go ipamContext.ServeIntrospection()

‎cmd/cni-metrics-helper/main.go

+40-4
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,34 @@ import (
2929
"github.com/aws/amazon-vpc-cni-k8s/cmd/cni-metrics-helper/metrics"
3030
"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
3131
"github.com/aws/amazon-vpc-cni-k8s/pkg/publisher"
32+
"github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics"
3233
)
3334

3435
const (
3536
appName = "cni-metrics-helper"
37+
38+
// metricsPort is the port for prometheus metrics
39+
metricsPort = 61681
40+
41+
// Environment variable to enable the metrics endpoint on 61681
42+
envEnablePrometheusMetrics = "USE_PROMETHEUS"
43+
)
44+
45+
var (
46+
prometheusRegistered = false
3647
)
3748

3849
type options struct {
39-
submitCW bool
40-
help bool
50+
submitCW bool
51+
help bool
52+
submitPrometheus bool
53+
}
54+
55+
func prometheusRegister() {
56+
if !prometheusRegistered {
57+
prometheusmetrics.PrometheusRegister()
58+
prometheusRegistered = true
59+
}
4160
}
4261

4362
func main() {
@@ -52,6 +71,7 @@ func main() {
5271
flags := pflag.NewFlagSet("", pflag.ExitOnError)
5372
flags.AddGoFlagSet(flag.CommandLine)
5473
flags.BoolVar(&options.submitCW, "cloudwatch", true, "a bool")
74+
flags.BoolVar(&options.submitPrometheus, "prometheus metrics", false, "a bool")
5575

5676
flags.Usage = func() {
5777
_, _ = fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
@@ -85,6 +105,17 @@ func main() {
85105
}
86106
}
87107

108+
prometheusENV, found := os.LookupEnv(envEnablePrometheusMetrics)
109+
if found {
110+
if strings.Compare(prometheusENV, "yes") == 0 || strings.Compare(prometheusENV, "true") == 0 {
111+
options.submitPrometheus = true
112+
prometheusRegister()
113+
}
114+
if strings.Compare(prometheusENV, "no") == 0 || strings.Compare(prometheusENV, "false") == 0 {
115+
options.submitPrometheus = false
116+
}
117+
}
118+
88119
metricUpdateIntervalEnv, found := os.LookupEnv("METRIC_UPDATE_INTERVAL")
89120
if !found {
90121
metricUpdateIntervalEnv = "30"
@@ -103,7 +134,7 @@ func main() {
103134
// should be name/identifier for the cluster if specified
104135
clusterID, _ := os.LookupEnv("AWS_CLUSTER_ID")
105136

106-
log.Infof("Starting CNIMetricsHelper. Sending metrics to CloudWatch: %v, LogLevel %s, metricUpdateInterval %d", options.submitCW, logConfig.LogLevel, metricUpdateInterval)
137+
log.Infof("Starting CNIMetricsHelper. Sending metrics to CloudWatch: %v, Prometheus: %v, LogLevel %s, metricUpdateInterval %d", options.submitCW, options.submitPrometheus, logConfig.LogLevel, metricUpdateInterval)
107138

108139
clientSet, err := k8sapi.GetKubeClientSet()
109140
if err != nil {
@@ -129,8 +160,13 @@ func main() {
129160
defer cw.Stop()
130161
}
131162

163+
if options.submitPrometheus {
164+
// Start prometheus server
165+
go prometheusmetrics.ServeMetrics(metricsPort)
166+
}
167+
132168
podWatcher := metrics.NewDefaultPodWatcher(k8sClient, log)
133-
var cniMetric = metrics.CNIMetricsNew(clientSet, cw, options.submitCW, log, podWatcher)
169+
var cniMetric = metrics.CNIMetricsNew(clientSet, cw, options.submitCW, options.submitPrometheus, log, podWatcher)
134170

135171
// metric loop
136172
for range time.Tick(time.Duration(metricUpdateInterval) * time.Second) {

‎cmd/cni-metrics-helper/metrics/cni_metrics.go

+19-13
Original file line numberDiff line numberDiff line change
@@ -165,24 +165,26 @@ var InterestingCNIMetrics = map[string]metricsConvert{
165165

166166
// CNIMetricsTarget defines data structure for kube-state-metric target
167167
type CNIMetricsTarget struct {
168-
interestingMetrics map[string]metricsConvert
169-
cwMetricsPublisher publisher.Publisher
170-
kubeClient kubernetes.Interface
171-
podWatcher *defaultPodWatcher
172-
submitCW bool
173-
log logger.Logger
168+
interestingMetrics map[string]metricsConvert
169+
cwMetricsPublisher publisher.Publisher
170+
kubeClient kubernetes.Interface
171+
podWatcher *defaultPodWatcher
172+
submitCW bool
173+
submitPrometheusMetrics bool
174+
log logger.Logger
174175
}
175176

176177
// CNIMetricsNew creates a new metricsTarget
177-
func CNIMetricsNew(k8sClient kubernetes.Interface, cw publisher.Publisher, submitCW bool, l logger.Logger,
178+
func CNIMetricsNew(k8sClient kubernetes.Interface, cw publisher.Publisher, submitCW bool, submitPrometheus bool, l logger.Logger,
178179
watcher *defaultPodWatcher) *CNIMetricsTarget {
179180
return &CNIMetricsTarget{
180-
interestingMetrics: InterestingCNIMetrics,
181-
cwMetricsPublisher: cw,
182-
kubeClient: k8sClient,
183-
podWatcher: watcher,
184-
submitCW: submitCW,
185-
log: l,
181+
interestingMetrics: InterestingCNIMetrics,
182+
cwMetricsPublisher: cw,
183+
kubeClient: k8sClient,
184+
podWatcher: watcher,
185+
submitCW: submitCW,
186+
submitPrometheusMetrics: submitPrometheus,
187+
log: l,
186188
}
187189
}
188190

@@ -220,3 +222,7 @@ func (t *CNIMetricsTarget) submitCloudWatch() bool {
220222
func (t *CNIMetricsTarget) getLogger() logger.Logger {
221223
return t.log
222224
}
225+
226+
func (t *CNIMetricsTarget) submitPrometheus() bool {
227+
return t.submitPrometheusMetrics
228+
}

‎cmd/cni-metrics-helper/metrics/cni_metrics_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func TestCNIMetricsNew(t *testing.T) {
5050
ctx := context.Background()
5151
_, _ = m.clientset.CoreV1().Pods("kube-system").Create(ctx, &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "aws-node-1"}}, metav1.CreateOptions{})
5252
//cniMetric := CNIMetricsNew(m.clientset, m.mockPublisher, m.discoverController, false, log)
53-
cniMetric := CNIMetricsNew(m.clientset, m.mockPublisher, false, testLog, m.podWatcher)
53+
cniMetric := CNIMetricsNew(m.clientset, m.mockPublisher, false, false, testLog, m.podWatcher)
5454
assert.NotNil(t, cniMetric)
5555
assert.NotNil(t, cniMetric.getCWMetricsPublisher())
5656
assert.NotEmpty(t, cniMetric.getInterestingMetrics())

‎cmd/cni-metrics-helper/metrics/metrics.go

+30
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727

2828
"github.com/aws/amazon-vpc-cni-k8s/pkg/publisher"
2929
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
30+
"github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics"
31+
"github.com/prometheus/client_golang/prometheus"
3032
)
3133

3234
type metricMatcher func(metric *dto.Metric) bool
@@ -38,6 +40,7 @@ type metricsTarget interface {
3840
getCWMetricsPublisher() publisher.Publisher
3941
getTargetList(ctx context.Context) ([]string, error)
4042
submitCloudWatch() bool
43+
submitPrometheus() bool
4144
getLogger() logger.Logger
4245
}
4346

@@ -354,6 +357,30 @@ func produceCloudWatchMetrics(t metricsTarget, families map[string]*dto.MetricFa
354357
}
355358
}
356359

360+
// Prometheus export supports only gauge metrics for now.
361+
362+
func producePrometheusMetrics(t metricsTarget, families map[string]*dto.MetricFamily, convertDef map[string]metricsConvert) {
363+
prometheusCNIMetrics := prometheusmetrics.GetSupportedPrometheusCNIMetricsMapping()
364+
if len(prometheusCNIMetrics) == 0 {
365+
t.getLogger().Infof("Skipping since prometheus mapping is missing")
366+
return
367+
}
368+
for key, family := range families {
369+
convertMetrics := convertDef[key]
370+
metricType := family.GetType()
371+
for _, action := range convertMetrics.actions {
372+
switch metricType {
373+
case dto.MetricType_GAUGE:
374+
metrics, ok := prometheusCNIMetrics[family.GetName()]
375+
if ok {
376+
metrics.(prometheus.Gauge).Set(action.data.curSingleDataPoint)
377+
}
378+
}
379+
}
380+
}
381+
382+
}
383+
357384
func resetMetrics(interestingMetrics map[string]metricsConvert) {
358385
for _, convert := range interestingMetrics {
359386
for _, act := range convert.actions {
@@ -426,4 +453,7 @@ func Handler(ctx context.Context, t metricsTarget) {
426453

427454
cw := t.getCWMetricsPublisher()
428455
produceCloudWatchMetrics(t, families, interestingMetrics, cw)
456+
if t.submitPrometheus() {
457+
producePrometheusMetrics(t, families, interestingMetrics)
458+
}
429459
}

‎cmd/cni-metrics-helper/metrics/metrics_test.go

+4
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ func (target *testMetricsTarget) submitCloudWatch() bool {
6161
return false
6262
}
6363

64+
func (target *testMetricsTarget) submitPrometheus() bool {
65+
return false
66+
}
67+
6468
func TestAPIServerMetric(t *testing.T) {
6569
testTarget := newTestMetricsTarget("cni_test1.data", InterestingCNIMetrics)
6670
ctx := context.Background()

0 commit comments

Comments
 (0)