Kubernetes: Datadog Constant Tags (#15213)

* adding option for constant tags to be sent to datadog for split environments

* removing unnecessary space

* Adding tests on envconfig

* clean test

* adding empty string test for constant tags

* format files

* remove public class from test class

Co-authored-by: Guy Feldman <[email protected]>

Author: marcosmarxm

airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java

@@ -461,6 +461,12 @@ public interface Configs {
    */
   String getDDDogStatsDPort();
 
+  /**
+   * Set constant tags to be attached to all metrics. Useful for distinguishing between environments.
+   * Example: airbyte_instance:dev,k8s-cluster:aws-dev
+   */
+  List<String> getDDConstantTags();
+
   /**
    * Define whether to publish tracking events to Segment or log-only. Airbyte internal use.
    */

airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java

@@ -99,7 +99,7 @@ public class EnvConfigs implements Configs {
   private static final String CONTAINER_ORCHESTRATOR_IMAGE = "CONTAINER_ORCHESTRATOR_IMAGE";
   private static final String DD_AGENT_HOST = "DD_AGENT_HOST";
   private static final String DD_DOGSTATSD_PORT = "DD_DOGSTATSD_PORT";
-
+  public static final String DD_CONSTANT_TAGS = "DD_CONSTANT_TAGS";
   public static final String STATE_STORAGE_S3_BUCKET_NAME = "STATE_STORAGE_S3_BUCKET_NAME";
   public static final String STATE_STORAGE_S3_REGION = "STATE_STORAGE_S3_REGION";
   public static final String STATE_STORAGE_S3_ACCESS_KEY = "STATE_STORAGE_S3_ACCESS_KEY";
@@ -796,6 +796,15 @@ public String getDDDogStatsDPort() {
     return getEnvOrDefault(DD_DOGSTATSD_PORT, "");
   }
 
+  @Override
+  public List<String> getDDConstantTags() {
+    String tagsString = getEnvOrDefault(DD_CONSTANT_TAGS, "");
+    return Splitter.on(",")
+        .splitToStream(tagsString)
+        .filter(s -> !s.trim().isBlank())
+        .collect(Collectors.toList());
+  }
+
   @Override
   public TrackingStrategy getTrackingStrategy() {
     return getEnvOrDefault(TRACKING_STRATEGY, TrackingStrategy.LOGGING, s -> {

airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java

@@ -356,6 +356,20 @@ void testPublishMetrics() {
     assertFalse(config.getPublishMetrics());
   }
 
+  @Test
+  @DisplayName("Should parse constant tags")
+  void testDDConstantTags() {
+    assertEquals(List.of(), config.getDDConstantTags());
+
+    envMap.put(EnvConfigs.DD_CONSTANT_TAGS, " ");
+    assertEquals(List.of(), config.getDDConstantTags());
+
+    envMap.put(EnvConfigs.DD_CONSTANT_TAGS, "airbyte_instance:dev,k8s-cluster:eks-dev");
+    List<String> expected = List.of("airbyte_instance:dev", "k8s-cluster:eks-dev");
+    assertEquals(expected, config.getDDConstantTags());
+    assertEquals(2, config.getDDConstantTags().size());
+  }
+
   @Nested
   @DisplayName("CheckJobResourceSettings")
   class CheckJobResourceSettings {

airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DatadogClientConfiguration.java

@@ -5,6 +5,7 @@
 package io.airbyte.metrics.lib;
 
 import io.airbyte.config.Configs;
+import java.util.List;
 import lombok.AllArgsConstructor;
 
 /**
@@ -17,10 +18,13 @@ public class DatadogClientConfiguration {
   public final String ddPort;
   public final boolean publish;
 
+  public final List<String> constantTags;
+
   public DatadogClientConfiguration(final Configs configs) {
     this.ddAgentHost = configs.getDDAgentHost();
     this.ddPort = configs.getDDDogStatsDPort();
     this.publish = configs.getPublishMetrics();
+    this.constantTags = configs.getDDConstantTags();
   }
 
 }

airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java

@@ -55,6 +55,7 @@ public void initialize(final MetricEmittingApp app, final DatadogClientConfigura
         .prefix(app.getApplicationName())
         .hostname(config.ddAgentHost)
         .port(Integer.parseInt(config.ddPort))
+        .constantTags(config.constantTags.toArray(new String[0]))
         .build();
   }
 

airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricSingleton.java

@@ -46,6 +46,7 @@ public synchronized static void initialize(final MetricEmittingApp app, final Da
         .prefix(app.getApplicationName())
         .hostname(config.ddAgentHost)
         .port(Integer.parseInt(config.ddPort))
+        .constantTags(config.constantTags.toArray(new String[0]))
         .build();
   }
 

airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/DogStatsDMetricClientTest.java

@@ -4,6 +4,7 @@
 
 package io.airbyte.metrics.lib;
 
+import java.util.Collections;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -17,7 +18,7 @@ class DogStatsDMetricClientTest {
   @BeforeEach
   void setUp() {
     dogStatsDMetricClient = new DogStatsDMetricClient();
-    dogStatsDMetricClient.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", false));
+    dogStatsDMetricClient.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", false, Collections.emptyList()));
   }
 
   @AfterEach

airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/DogStatsDMetricSingletonTest.java

@@ -4,6 +4,7 @@
 
 package io.airbyte.metrics.lib;
 
+import java.util.Collections;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.DisplayName;
@@ -20,7 +21,8 @@ void tearDown() {
   @DisplayName("there should be no exception if we attempt to emit metrics while publish is false")
   void testPublishTrueNoEmitError() {
     Assertions.assertDoesNotThrow(() -> {
-      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", false));
+      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER,
+          new DatadogClientConfiguration("localhost", "1000", false, Collections.emptyList()));
       DogStatsDMetricSingleton.gauge(OssMetricsRegistry.KUBE_POD_PROCESS_CREATE_TIME_MILLISECS, 1);
     });
   }
@@ -29,7 +31,8 @@ void testPublishTrueNoEmitError() {
   @DisplayName("there should be no exception if we attempt to emit metrics while publish is true")
   void testPublishFalseNoEmitError() {
     Assertions.assertDoesNotThrow(() -> {
-      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", true));
+      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER,
+          new DatadogClientConfiguration("localhost", "1000", true, Collections.emptyList()));
       DogStatsDMetricSingleton.gauge(OssMetricsRegistry.KUBE_POD_PROCESS_CREATE_TIME_MILLISECS, 1);
     });
   }