fix(ci): e2e dd logs failure

.github/workflows/ci-integration-review.yml

@@ -113,7 +113,7 @@ jobs:
         with:
           timeout_minutes: 30
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }}
+          command: bash scripts/int-e2e-test.sh int ${{ matrix.service }}
 
   e2e-tests:
     needs: prep-pr
@@ -136,7 +136,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs
+          command: bash scripts/int-e2e-test.sh e2e datadog-logs
 
       - name: datadog-e2e-metrics
         if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-metrics')
@@ -146,7 +146,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics
+          command: bash scripts/int-e2e-test.sh e2e datadog-metrics
 
   update-pr-status:
     name: Signal result to PR

.github/workflows/e2e.yml

@@ -89,7 +89,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs
+          command: bash scripts/int-e2e-test.sh e2e datadog-logs
 
       - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
@@ -98,7 +98,7 @@ jobs:
         with:
           timeout_minutes: 35
           max_attempts: 3
-          command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics
+          command: bash scripts/int-e2e-test.sh e2e datadog-metrics
 
 
   e2e-test-suite:

.github/workflows/integration.yml

@@ -148,7 +148,7 @@ jobs:
                   "${{ needs.changes.outputs.all-int }}" == "true" || \
                   "$should_run" == "true" ]]; then
               echo "Running test for ${{ matrix.service }}"
-              bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }}
+              bash scripts/int-e2e-test.sh int ${{ matrix.service }}
             else
               echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met."
             fi

scripts/e2e/Dockerfile

@@ -1,8 +1,7 @@
 ARG RUST_VERSION
 ARG FEATURES
-ARG DEBIAN_RELEASE=slim-bookworm
 
-FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE}
+FROM docker.io/rust:${RUST_VERSION}-slim-bookworm
 
 RUN apt-get update && apt-get -y --no-install-recommends install \
     build-essential \
@@ -28,7 +27,7 @@ RUN git clone https://github.com/rui314/mold.git \
     && cmake --build . -j $(nproc) \
     && cmake --install .
 
-RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.72 --locked
+RUN cargo install cargo-nextest --version 0.9.72 --locked
 
 COPY scripts/environment/install-protoc.sh /
 COPY tests/data/ca/certs /certs

scripts/e2e/datadog-logs/compose.yaml

@@ -85,13 +85,13 @@ services:
   # which does the validation of consistency with the other fakeintake service.
   fakeintake-agent:
     # TODO: temporarily pegging the image as latest results in failures
-    image: docker.io/datadog/fakeintake:v77a06f2b
+    image: docker.io/datadog/fakeintake:ved764626
 
   # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner
   # which does the validation of consistency with the other fakeintake service.
   fakeintake-vector:
     # TODO: temporarily pegging the image as latest results in failures
-    image: docker.io/datadog/fakeintake:v77a06f2b
+    image: docker.io/datadog/fakeintake:ved764626
 
 networks:
   default:

scripts/ci-int-e2e-test.sh → scripts/int-e2e-test.sh

@@ -6,11 +6,6 @@
 
 set -u
 
-if [[ -z "${CI:-}" ]]; then
-  echo "Aborted: this script is for use in CI." >&2
-  exit 1
-fi
-
 if [ $# -ne 2 ]
 then
   echo "usage: $0 [int|e2e] TEST_NAME"
@@ -27,5 +22,10 @@ sleep 30
 cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}"
 RET=$?
 cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}"
-./scripts/upload-test-results.sh
+
+# Only upload test results if CI is defined
+if [[ -n "${CI:-}" ]]; then
+  ./scripts/upload-test-results.sh
+fi
+
 exit $RET

tests/data/e2e/datadog/logs/agent_only.yaml

@@ -27,3 +27,6 @@ logs_config:
   logs_no_ssl: true
   force_use_http: true
   batch_wait: 1
+
+# Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker
+dd_url: 'http://fakeintake-agent:80'

tests/data/e2e/datadog/logs/agent_vector.yaml

@@ -32,3 +32,6 @@ vector:
   logs:
     enabled: true
     url: "http://vector:8181"
+
+# Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker
+dd_url: 'http://fakeintake-agent:80'

tests/e2e/datadog/logs/mod.rs

@@ -1,11 +1,13 @@
 use serde_json::Value;
 use tracing::info;
 
-use vector::test_util::trace_init;
-
 use super::*;
+use std::time::Duration;
+use vector::test_util::trace_init;
 
 const LOGS_ENDPOINT: &str = "/api/v2/logs";
+const MAX_RETRIES: usize = 10;
+const WAIT_INTERVAL: Duration = Duration::from_secs(1);
 
 fn expected_log_events() -> usize {
     std::env::var("EXPECTED_LOG_EVENTS")
@@ -70,25 +72,38 @@ fn reduce_to_data(payloads: Vec<FakeIntakePayload<Value>>) -> Vec<Value> {
 async fn validate() {
     trace_init();
 
-    // Even with configuring docker service dependencies, we need a small buffer of time
-    // to ensure events flow through to fakeintake before asking for them
-    std::thread::sleep(std::time::Duration::from_secs(2));
+    // Retry until we have log payloads or hit max retries.
+    // This is to ensure events flow through to fakeintake before asking for them.
+    let mut agent_payloads = Vec::new();
+    for _ in 0..MAX_RETRIES {
+        info!("getting log payloads from agent-only pipeline");
+        agent_payloads = get_fakeintake_payloads::<FakeIntakeResponseJson>(
+            &fake_intake_agent_address(),
+            LOGS_ENDPOINT,
+        )
+        .await
+        .payloads;
+
+        if !agent_payloads.is_empty() {
+            break;
+        }
+
+        info!("No valid payloads yet, retrying...");
+        tokio::time::sleep(WAIT_INTERVAL).await;
+    }
 
-    info!("getting log payloads from agent-only pipeline");
-    let mut agent_payloads = get_fakeintake_payloads::<FakeIntakeResponseJson>(
-        &fake_intake_agent_address(),
-        LOGS_ENDPOINT,
-    )
-    .await
-    .payloads;
+    // If we still don't have valid payloads after retries, fail the test
+    if agent_payloads.is_empty() {
+        panic!("Failed to get valid log payloads from agent pipeline after {MAX_RETRIES} retries");
+    }
 
     // the logs endpoint receives an empty healthcheck payload in the beginning
     if !agent_payloads.is_empty() {
         agent_payloads.retain(|raw_payload| !raw_payload.data.as_array().unwrap().is_empty())
     }
 
     let mut agent_payloads = reduce_to_data(agent_payloads);
-
+    info!("pront 2 {agent_payloads:?}");
     common_assertions(&mut agent_payloads);
 
     info!("getting log payloads from agent-vector pipeline");

tests/e2e/datadog/mod.rs

@@ -62,6 +62,7 @@ where
     R: FakeIntakeResponseT + DeserializeOwned,
 {
     let url = &R::build_url(base, endpoint);
+    println!("pront 3 Fetching payloads from {}", url);
     Client::new()
         .request(Method::GET, url)
         .send()

tilt/Dockerfile

@@ -31,6 +31,8 @@ RUN --mount=type=cache,target=/vector/target \
 #
 # TARGET
 #
+ARG DEBIAN_RELEASE
+
 FROM docker.io/debian:${DEBIAN_RELEASE}-slim
 RUN apt-get update && apt-get -y --no-install-recommends install zlib1g && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /vector/vector /usr/bin/vector