feat: add prometheus rules for cnpg backups (#1837)

Co-authored-by: jeho <[email protected]>

Author: srodenhuis

package-lock.json

@@ -84,7 +84,7 @@
     "standard-version": "9.5.0",
     "ts-auto-mock": "3.6.2",
     "ts-jest": "27.1.5",
-    "ts-node": "10.8.1",
+    "ts-node": "^10.8.1",
     "ts-node-dev": "1.1.8",
     "tsc-alias": "^1.7.0",
     "tsconfig-paths": "3.14.1",

package.json

@@ -156,6 +156,18 @@ prometheus:
 additionalPrometheusRules:
   - name: blackbox
     {{- readFile "rules/blackbox.yaml" | nindent 4 }}
+  {{ if $v.platformBackups.database.gitea.enabled }}
+  - name: gitea-db-backup
+    {{- readFile "rules/gitea-db-backup.yaml" | nindent 4 }}
+  {{- end }}
+  {{ if $v.platformBackups.database.harbor.enabled }}
+  - name: harbor-db-backup
+    {{- readFile "rules/harbor-db-backup.yaml" | nindent 4 }}
+  {{- end }}
+  {{ if $v.platformBackups.database.keycloak.enabled }}
+  - name: keycloak-db-backup
+    {{- readFile "rules/keycloak-db-backup.yaml" | nindent 4 }}
+  {{- end }}
 alertmanager:
   enabled: {{ $a.enabled }}
   alertmanagerSpec:

values/prometheus-operator/prometheus-operator.gotmpl

@@ -0,0 +1,10 @@
+groups:
+  - name: gitea-db-backup
+    rules:
+      - alert: BackupGiteaDbFailing
+        expr: time() - cnpg_collector_last_failed_backup_timestamp{namespace="gitea",pod=~"gitea-db-1"} > 86400 or time() - cnpg_collector_last_failed_backup_timestamp{namespace="gitea",pod=~"gitea-db-2"} > 86400
+        labels:
+          severity: warning
+        annotations:
+          description: 'The Gitea database backup for DB {{ $labels.pod }} failed at {{ $value | humanizeTimestamp }}'
+          summary: Backup of {{ $labels.pod }} failed

values/prometheus-operator/rules/gitea-db-backup.yaml

@@ -0,0 +1,10 @@
+groups:
+  - name: harbor-db-backup
+    rules:
+      - alert: BackupHarborDbFailing
+        expr: time() - cnpg_collector_last_failed_backup_timestamp{namespace="harbor",pod=~"harbor-otomi-db-1"} > 86400 or time() - cnpg_collector_last_failed_backup_timestamp{namespace="harbor",pod=~"harbor-otomi-db-2"} > 86400
+        labels:
+          severity: warning
+        annotations:
+          description: 'The Harbor database backup for DB {{ $labels.pod }} failed at {{ $value | humanizeTimestamp }}'
+          summary: Backup of {{ $labels.pod }} failed

values/prometheus-operator/rules/harbor-db-backup.yaml

@@ -0,0 +1,10 @@
+groups:
+  - name: keycloak-db-backup
+    rules:
+      - alert: BackupKeycloakDbFailing
+        expr: time() - cnpg_collector_last_failed_backup_timestamp{namespace="keycloak",pod=~"keycloak-db-1"} > 86400 or time() - cnpg_collector_last_failed_backup_timestamp{namespace="keycloak",pod=~"keycloak-db-2"} > 86400
+        labels:
+          severity: warning
+        annotations:
+          description: 'The Keycloak database backup for DB {{ $labels.pod }} failed at {{ $value | humanizeTimestamp }}'
+          summary: Backup of {{ $labels.pod }} failed