feat: add prometheus rule to validate db backup executions (#1866)

Co-authored-by: jeho <[email protected]>

Author: srodenhuis

values/prometheus-operator/rules/gitea-db-backup.yaml

@@ -2,9 +2,10 @@ groups:
   - name: gitea-db-backup
     rules:
       - alert: BackupGiteaDbFailing
-        expr: time() - cnpg_collector_last_failed_backup_timestamp{namespace="gitea",pod=~"gitea-db-1"} > 86400 or time() - cnpg_collector_last_failed_backup_timestamp{namespace="gitea",pod=~"gitea-db-2"} > 86400
+        # 86400 sec => 24H
+        expr: (time() - max(cnpg_collector_last_available_backup_timestamp{namespace="gitea"}) > 86400)
         labels:
           severity: warning
         annotations:
           description: 'The Gitea database backup for DB {{ $labels.pod }} failed at {{ $value | humanizeTimestamp }}'
-          summary: Backup of {{ $labels.pod }} failed
+          summary: Backup of {{ $labels.pod }} failed

values/prometheus-operator/rules/harbor-db-backup.yaml

@@ -2,7 +2,8 @@ groups:
   - name: harbor-db-backup
     rules:
       - alert: BackupHarborDbFailing
-        expr: time() - cnpg_collector_last_failed_backup_timestamp{namespace="harbor",pod=~"harbor-otomi-db-1"} > 86400 or time() - cnpg_collector_last_failed_backup_timestamp{namespace="harbor",pod=~"harbor-otomi-db-2"} > 86400
+        # 86400 sec => 24H
+        expr: (time() - max(cnpg_collector_last_available_backup_timestamp{namespace="harbor"}) > 86400)
         labels:
           severity: warning
         annotations:

values/prometheus-operator/rules/keycloak-db-backup.yaml

@@ -2,7 +2,8 @@ groups:
   - name: keycloak-db-backup
     rules:
       - alert: BackupKeycloakDbFailing
-        expr: time() - cnpg_collector_last_failed_backup_timestamp{namespace="keycloak",pod=~"keycloak-db-1"} > 86400 or time() - cnpg_collector_last_failed_backup_timestamp{namespace="keycloak",pod=~"keycloak-db-2"} > 86400
+        # 86400 sec => 24H
+        expr: (time() - max(cnpg_collector_last_available_backup_timestamp{namespace="keycloak"}) > 86400)
         labels:
           severity: warning
         annotations: