Trace Failure Origins (#19550)

* Clarify local datadog instructions

* Add failure origins to trace

Author: jdpgrailsdev

airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java

@@ -63,6 +63,11 @@ public static final class Tags {
      */
     public static final String DOCKER_IMAGE_KEY = "docker_image";
 
+    /**
+     * Name of the APM trace tag that holds the failure origin(s) associated with the trace.
+     */
+    public static final String FAILURE_ORIGINS_KEY = "failure_origins";
+
     /**
      * Name of the APM trace tag that holds the job ID value associated with the trace.
      */

airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java

@@ -8,6 +8,7 @@
 import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME;
 import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY;
 import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY;
+import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.FAILURE_ORIGINS_KEY;
 import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY;
 import static io.airbyte.persistence.job.models.AttemptStatus.FAILED;
 
@@ -22,6 +23,7 @@
 import io.airbyte.config.Configs.WorkerEnvironment;
 import io.airbyte.config.DestinationConnection;
 import io.airbyte.config.FailureReason;
+import io.airbyte.config.FailureReason.FailureOrigin;
 import io.airbyte.config.JobConfig;
 import io.airbyte.config.JobOutput;
 import io.airbyte.config.JobSyncConfig;
@@ -56,6 +58,7 @@
 import io.airbyte.workers.run.TemporalWorkerRunFactory;
 import io.airbyte.workers.run.WorkerRun;
 import io.micronaut.context.annotation.Requires;
+import io.micronaut.core.util.CollectionUtils;
 import jakarta.inject.Singleton;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -67,6 +70,7 @@
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -287,12 +291,16 @@ public void jobFailure(final JobFailureInput input) {
   @Override
   public void attemptFailure(final AttemptFailureInput input) {
     try {
-      ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, input.getAttemptId(), JOB_ID_KEY, input.getJobId()));
-
       final int attemptId = input.getAttemptId();
       final long jobId = input.getJobId();
       final AttemptFailureSummary failureSummary = input.getAttemptFailureSummary();
 
+      ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, attemptId, JOB_ID_KEY, jobId));
+      if (CollectionUtils.isNotEmpty(failureSummary.getFailures())) {
+        ApmTraceUtils.addTagsToTrace(Map.of(FAILURE_ORIGINS_KEY, failureSummary.getFailures().stream().map(FailureReason::getFailureOrigin).map(
+            FailureOrigin::name).collect(Collectors.joining(","))));
+      }
+
       jobPersistence.failAttempt(jobId, attemptId);
       jobPersistence.writeAttemptFailureSummary(jobId, attemptId, failureSummary);
 

docker-compose.datadog.yaml

@@ -2,7 +2,7 @@
 # This exists _only_ for testing datadog integrations!
 #
 # Usage:
-# 1. create an API Key in datadog
+# 1. Create an organization API Key in DataDog (Organization Settings > API Keys)
 # 2. wget -O dd-java-agent.jar 'https://dtdg.co/latest-java-tracer'
 # 3. DD_API_KEY=[datadog api key] VERSION=dev docker-compose -f docker-compose.yaml -f docker-compose.datadog.yaml up -d
 version: "3.7"