[PLAT-17182][PLAT-17154][PLAT-16913][PLAT-16785][PLAT-14383] Multiple metric/alert related fixes

Summary:
Contains fixes for the following JIRAs:

[PLAT-17182] Reactor Delay 'Metric Graph in Prometheus' Does Not Show Outbound Queue or Outbound Transfer
[PLAT-17154] Raise the threshold for DB Queue Overflow Alert to 100
[PLAT-16913] Set defaultDurationSec: 300 for DDL atomicity alert
[PLAT-16785] Promethus Auth settings are present as runtime configs
[PLAT-14383] Backup metrics treats universe_name & universe_uuid as same string

Test Plan: Tested manually

Reviewers: vbansal

Reviewed By: vbansal

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D42950

Author: anmalysh-yb

managed/RUNTIME-FLAGS.md

@@ -127,9 +127,6 @@
 | "Devops command timeout" | "yb.devops.command_timeout" | "GLOBAL" | "Devops command timeout" | "Duration" |
 | "Node destroy command timeout" | "yb.node_ops.destroy_server_timeout" | "GLOBAL" | "Timeout for node destroy command before failing." | "Duration" |
 | "YBC Compatible DB Version" | "ybc.compatible_db_version" | "GLOBAL" | "Minimum YBDB version which supports YBC" | "String" |
-| "Prometheus auth enabled" | "yb.metrics.auth" | "GLOBAL" | "Enables basic authentication for Prometheus web UI/APIs access" | "Boolean" |
-| "Prometheus auth username" | "yb.metrics.auth_username" | "GLOBAL" | "Username, used for request authentication against embedded Prometheus" | "String" |
-| "Prometheus auth password" | "yb.metrics.auth_password" | "GLOBAL" | "Password, used for request authentication against embedded Prometheus" | "String" |
 | "Force YBC Shutdown during upgrade" | "ybc.upgrade.force_shutdown" | "GLOBAL" | "For YBC Shutdown during upgrade" | "Boolean" |
 | "Enable strict mode to ignore deprecated YBA APIs" | "yb.api.mode.strict" | "GLOBAL" | "Will ignore deprecated APIs" | "Boolean" |
 | "Enable safe mode to ignore preview YBA APIs" | "yb.api.mode.safe" | "GLOBAL" | "Will ignore preview APIs" | "Boolean" |

managed/src/main/java/com/yugabyte/yw/common/config/GlobalConfKeys.java

@@ -1122,23 +1122,23 @@ public class GlobalConfKeys extends RuntimeConfigKeysModule {
           "Prometheus auth enabled",
           "Enables basic authentication for Prometheus web UI/APIs access",
           ConfDataType.BooleanType,
-          ImmutableList.of(ConfKeyTags.PUBLIC));
+          ImmutableList.of(ConfKeyTags.INTERNAL));
   public static final ConfKeyInfo<String> metricsAuthUsername =
       new ConfKeyInfo<>(
           "yb.metrics.auth_username",
           ScopeType.GLOBAL,
           "Prometheus auth username",
           "Username, used for request authentication against embedded Prometheus",
           ConfDataType.StringType,
-          ImmutableList.of(ConfKeyTags.PUBLIC));
+          ImmutableList.of(ConfKeyTags.INTERNAL));
   public static final ConfKeyInfo<String> metricsAuthPassword =
       new ConfKeyInfo<>(
           "yb.metrics.auth_password",
           ScopeType.GLOBAL,
           "Prometheus auth password",
           "Password, used for request authentication against embedded Prometheus",
           ConfDataType.StringType,
-          ImmutableList.of(ConfKeyTags.PUBLIC));
+          ImmutableList.of(ConfKeyTags.INTERNAL));
   public static final ConfKeyInfo<Boolean> forceYbcShutdownDuringUpgrade =
       new ConfKeyInfo<>(
           "ybc.upgrade.force_shutdown",

managed/src/main/java/com/yugabyte/yw/common/metrics/MetricLabelsBuilder.java

@@ -15,7 +15,7 @@
 import com.yugabyte.yw.models.Customer;
 import com.yugabyte.yw.models.Universe;
 import com.yugabyte.yw.models.helpers.KnownAlertLabels;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -38,7 +38,7 @@ public class MetricLabelsBuilder {
     KnownAlertLabels.SOURCE_TYPE.labelName()
   };
 
-  private final Map<String, String> labels = new HashMap<>();
+  private final Map<String, String> labels = new LinkedHashMap<>();
 
   public static MetricLabelsBuilder create() {
     return new MetricLabelsBuilder();

managed/src/main/java/com/yugabyte/yw/controllers/AlertController.java

@@ -1191,7 +1191,8 @@ private String getAlertExpressionUrl(
             alert.getResolvedTime() != null
                 ? alert.getResolvedTime().getTime()
                 : System.currentTimeMillis());
-    return metricUrlProvider.getExpressionUrl(expression, startUnixTime, endUnixTime);
+    return metricUrlProvider.getExpressionUrl(
+        Collections.singletonList(expression), startUnixTime, endUnixTime);
   }
 
   @VisibleForTesting

managed/src/main/java/com/yugabyte/yw/metrics/MetricQueryExecutor.java

@@ -92,11 +92,12 @@ private JsonNode getMetrics(Map<String, String> queryParam) {
     return apiHelper.getRequest(queryUrl, headers, queryParam);
   }
 
-  private String getDirectURL(String queryExpr) {
+  private String getDirectURL(List<String> queryExpr) {
     long endUnixTime = Long.parseLong(queryParam.getOrDefault("end", "0"));
     long startUnixTime = Long.parseLong(queryParam.getOrDefault("start", "0"));
+    Long step = Long.parseLong(queryParam.get("step"));
 
-    return metricUrlProvider.getExpressionUrl(queryExpr, startUnixTime, endUnixTime);
+    return metricUrlProvider.getExpressionUrl(queryExpr, startUnixTime, endUnixTime, step);
   }
 
   @Override
@@ -147,6 +148,7 @@ public JsonNode call() {
       ArrayNode directURLs = responseJson.putArray("directURLs");
       responseJson.put(
           "metricsLinkUseBrowserFqdn", metricUrlProvider.getMetricsLinkUseBrowserFqdn());
+      List<String> queryExpressions = new ArrayList<>();
       for (Map.Entry<String, String> e : queries.entrySet()) {
         String metric = e.getKey();
         String queryExpr = e.getValue();
@@ -160,7 +162,7 @@ public JsonNode call() {
         }
         queryParam.put("query", queryExpr);
         try {
-          directURLs.add(getDirectURL(queryExpr));
+          queryExpressions.add(queryExpr);
         } catch (Exception de) {
           log.trace("Error getting direct url", de);
         }
@@ -179,6 +181,7 @@ public JsonNode call() {
           output.addAll(queryResponse.getGraphData(metric, configDefinition, metricSettings));
         }
       }
+      directURLs.add(getDirectURL(queryExpressions));
       if (isRecharts) {
         responseJson.set("data", Json.toJson(rechartsOutput));
       } else {

managed/src/main/java/com/yugabyte/yw/metrics/MetricUrlProvider.java

@@ -2,15 +2,17 @@
 
 package com.yugabyte.yw.metrics;
 
-import com.cronutils.utils.StringUtils;
 import com.yugabyte.yw.common.Util;
 import com.yugabyte.yw.common.config.GlobalConfKeys;
 import com.yugabyte.yw.common.config.RuntimeConfGetter;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
 import java.util.TimeZone;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 
 @Slf4j
 @Singleton
@@ -54,7 +56,12 @@ public String getMetricsExternalUrl() {
     return metricsExternalUrl;
   }
 
-  public String getExpressionUrl(String queryExpr, Long startUnixTime, Long endUnixTime) {
+  public String getExpressionUrl(List<String> queryExpr, Long startUnixTime, Long endUnixTime) {
+    return getExpressionUrl(queryExpr, startUnixTime, endUnixTime);
+  }
+
+  public String getExpressionUrl(
+      List<String> queryExpr, Long startUnixTime, Long endUnixTime, Long step) {
     String durationSecs = "3600s";
     String endString = "";
 
@@ -67,11 +74,30 @@ public String getExpressionUrl(String queryExpr, Long startUnixTime, Long endUni
       durationSecs = String.format("%ds", (endUnixTime - startUnixTime));
     }
 
+    StringBuilder result =
+        new StringBuilder(String.format("%s/graph?", this.getMetricsExternalUrl()));
+    for (int i = 0; i < queryExpr.size(); i++) {
+      if (i > 0) {
+        result.append("&");
+      }
+      String expression = queryExpr.get(i);
+      result.append(
+          String.format(
+              "g%d.expr=%s&g%d.tab=0&g%d.range_input=%s&g%d.end_input=%s",
+              i,
+              URLEncoder.encode(expression, StandardCharsets.UTF_8),
+              i,
+              i,
+              durationSecs,
+              i,
+              endString));
+      if (step != null) {
+        result.append(String.format("&g%d.step_input=%d", i, step));
+      }
+    }
     // Note: this is the URL as prometheus' web interface renders these metrics. It is
     // possible this breaks over time as we upgrade prometheus.
-    return String.format(
-        "%s/graph?g0.expr=%s&g0.tab=0&g0.range_input=%s&g0.end_input=%s",
-        this.getMetricsExternalUrl(), URLEncoder.encode(queryExpr), durationSecs, endString);
+    return result.toString();
   }
 
   public String getMetricsInternalUrl() {

managed/src/main/resources/alert/alert_templates.yml

@@ -460,7 +460,7 @@ templates:
     createForNewCustomer: true
     defaultThresholdMap:
       WARNING:
-        threshold: 0.0
+        threshold: 100.0
     targetType: UNIVERSE
     defaultThresholdCondition: GREATER_THAN
     defaultThresholdUnit: COUNT
@@ -660,6 +660,7 @@ templates:
     queryTemplate: yb_ddl_atomicity_check{universe_uuid="__universeUuid__"}
       {{ query_condition }} {{ query_threshold }}
     createForNewCustomer: true
+    defaultDurationSec: 300
     defaultThresholdMap:
       SEVERE:
         threshold: 1.0

managed/src/main/resources/db/migration/default_/postgres/V406__Alert_DB_Queues_Overflow_Fix.sql

@@ -0,0 +1,20 @@
+-- Copyright (c) YugaByte, Inc.
+
+-- Update DB_QUEUES_OVERFLOW alert threshold
+UPDATE alert_configuration
+SET thresholds = jsonb_set(
+        thresholds::jsonb,
+        '{WARNING,threshold}',
+        to_jsonb(greatest((thresholds::jsonb -> 'WARNING' ->> 'threshold')::double precision, 100))
+    )
+WHERE template = 'DB_QUEUES_OVERFLOW';
+
+-- Update DB_QUEUES_OVERFLOW alert duration
+UPDATE alert_configuration
+SET duration_sec = greatest(duration_sec, 300)
+WHERE template = 'DDL_ATOMICITY_CHECK';
+
+UPDATE alert_definition
+SET config_written = false
+WHERE configuration_uuid IN
+  (select uuid from alert_configuration where template in ('DB_QUEUES_OVERFLOW', 'DDL_ATOMICITY_CHECK'));