ref: make everything configurable

Author: wbollock

traefik-mixin/alerts/alerts.libsonnet

@@ -1,33 +1,57 @@
 {
-  groups+: [
-    {
-      name: 'traefik',
-      rules: [
-        // TraefikConfigReloadFailuresIncreasing
-        {
-          alert: 'TraefikConfigReloadFailuresIncreasing',
-          expr: 'sum(rate(traefik_config_reloads_failure_total[5m])) > 0',
-          'for': '5m',
-          labels: {
-            severity: 'critical',
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'traefik',
+        rules: [
+          // TraefikConfigReloadFailuresIncreasing
+          {
+            alert: 'TraefikConfigReloadFailuresIncreasing',
+            expr: |||
+              sum by (%(sumByLabels)s) (rate(traefik_config_reloads_failure_total{%(timeSeriesLabels)s}[5m])) > 0
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            } + std.get($._config, 'alertLabels', {}),
+            annotations: {
+              description: 'Traefik is failing to reload its config',
+            } + std.get($._config, 'alertAnnotations', {}),
           },
-          annotations: {
-            description: 'Traefik is failing to reload its config',
+          // TraefikTLSCertificatesExpiring (critical)
+          {
+            alert: 'TraefikTLSCertificatesExpiring',
+            expr: |||
+              max by (%(maxByLabels)s) ((last_over_time(traefik_tls_certs_not_after{%(timeSeriesLabels)s}[5m]) - time()) / 86400) < %(traefik_tls_expiry_days_critical)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            } + std.get($._config, 'alertLabels', {}),
+            annotations: {
+              description: |||
+                The minimum number of days until a Traefik-served certificate expires is {{ printf "%%.0f" $value }} days on {{ $labels.sans }} which is below the critical threshold of %(traefik_tls_expiry_days_critical)s.
+              ||| % $._config,
+            } + std.get($._config, 'alertAnnotations', {}),
           },
-        },
-        // TraefikTLSCertificatesExpiring
-        {
-          alert: 'TraefikTLSCertificatesExpiring',
-          expr: 'max by (sans) ((last_over_time(traefik_tls_certs_not_after[5m]) - time()) / 86400) < 7',
-          'for': '5m',
-          labels: {
-            severity: 'critical',
+          // TraefikTLSCertificatesExpiring (warning)
+          {
+            alert: 'TraefikTLSCertificatesExpiringSoon',
+            expr: |||
+              max by (%(maxByLabels)s) ((last_over_time(traefik_tls_certs_not_after{%(timeSeriesLabels)s}[5m]) - time()) / 86400) < %(traefik_tls_expiry_days_warning)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            } + std.get($._config, 'alertLabels', {}),
+            annotations: {
+              description: |||
+                The minimum number of days until a Traefik-served certificate expires is {{ printf "%%.0f" $value }} days on {{ $labels.sans }} which is below the warning threshold of %(traefik_tls_expiry_days_warning)s.
+              ||| % $._config,
+            } + std.get($._config, 'alertAnnotations', {}),
           },
-          annotations: {
-            description: 'Traefik is serving certificates that will expire soon',
-          },
-        },
-      ],
-    },
-  ],
+        ],
+      },
+    ],
+  },
 }

traefik-mixin/config.libsonnet

@@ -0,0 +1,26 @@
+{
+  _config+:: {
+    // alerts thresholds
+    traefik_tls_expiry_days_critical: 7,
+    traefik_tls_expiry_days_warning: 14,
+    timeSeriesLabels: '',
+    // Example:
+    // timeSeriesLabels: "component=\"traefik\",environment=\"production\"",
+    // for config alert
+    sumByLabels: 'instance',
+    // for TLS alerts
+    maxByLabels: 'sans',
+    alertLabels: {},
+    // Example:
+    // alertLabels: {
+    //   environment: 'production',
+    //   component: 'traefik',
+    // },
+    alertAnnotations: {},
+    // Example:
+    // alertAnnotations: {
+    //   runbook: 'https://runbooks.example.com/traefik-tls',
+    //   grafana: 'https://grafana.example.com/d/traefik',
+    // },
+  },
+}

traefik-mixin/mixin.libsonnet

@@ -2,5 +2,5 @@
   grafanaDashboards+:: {
     'traefikdash.json': (import 'dashboards/traefikdash.json'),
   },
-  prometheusAlerts+:: (import 'alerts/alerts.libsonnet'),
-}
+} + (import 'alerts/alerts.libsonnet') +
+(import 'config.libsonnet')