fix(ci): address issues in integration test suite workflow (vectordotdev#17928)

- Author membership using the get-user-teams-membership doesn't work for
contributor PRs because the GHA requires a GH token with proper scopes,
which they don't have. Fix=check if secrets are defined in the
environment.
- This and a couple other workflows had a logic bug in that they could
be added to the merge queue if the main dep job were skipped (i.e due to
a bug in the workflow).

Author: neuronull

.github/workflows/integration.yml

@@ -46,25 +46,10 @@ jobs:
       int_tests: true
     secrets: inherit
 
-  detect-user:
-    name: Detect user team membership
-    runs-on: ubuntu-latest
-    outputs:
-      IS_TEAM_MEMBER: ${{ steps.author.outputs.isTeamMember }}
-    steps:
-      - name: Get PR author
-        if: github.event_name == 'pull_request'
-        id: author
-        uses: tspascoal/get-user-teams-membership@v2
-        with:
-          username: ${{ github.actor }}
-          team: 'Vector'
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }}
-
   integration-tests:
     name: Integration Tests
     runs-on: [linux, ubuntu-20.04-4core]
-    needs: [changes, detect-user]
+    needs: changes
     if: always() && (
       github.event_name == 'merge_group' || (
           needs.changes.outputs.all-int == 'true'
@@ -110,6 +95,17 @@ jobs:
 
       - run: docker image prune -af ; docker container prune -f
 
+      - name: Determine if secrets are defined (PR author is team member).
+        if: github.event_name == 'pull_request'
+        env:
+          GH_PAT_ORG: ${{ secrets.GH_PAT_ORG }}
+        run: |
+          if [[ "$GH_PAT_ORG" != "" ]] ; then
+            echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV"
+          else
+            echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV"
+          fi
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.amqp == 'true' }}
         name: amqp
         uses: nick-fields/retry@v2
@@ -119,7 +115,7 @@ jobs:
           command: bash scripts/ci-integration-test.sh amqp
 
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.appsignal == 'true') &&
-          (github.event_name != 'pull_request' || needs.detect-user.outputs.IS_TEAM_MEMBER == 'true')
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: appsignal
         uses: nick-fields/retry@v2
         with:
@@ -136,7 +132,7 @@ jobs:
           command: bash scripts/ci-integration-test.sh  aws
 
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.axiom == 'true') &&
-          (github.event_name != 'pull_request' || needs.detect-user.outputs.IS_TEAM_MEMBER == 'true')
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: axiom
         uses: nick-fields/retry@v2
         with:
@@ -161,7 +157,7 @@ jobs:
           command: bash scripts/ci-integration-test.sh  clickhouse
 
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.databend == 'true') &&
-          (github.event_name != 'pull_request' || needs.detect-user.outputs.IS_TEAM_MEMBER == 'true')
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: databend
         uses: nick-fields/retry@v2
         with:
@@ -170,7 +166,7 @@ jobs:
           command: bash scripts/ci-integration-test.sh  databend
 
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
-          (github.event_name != 'pull_request' || needs.detect-user.outputs.IS_TEAM_MEMBER == 'true')
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: datadog-agent
         uses: nick-fields/retry@v2
         with:
@@ -179,7 +175,7 @@ jobs:
           command: bash scripts/ci-integration-test.sh  datadog-agent
 
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
-          (github.event_name != 'pull_request' || needs.detect-user.outputs.IS_TEAM_MEMBER == 'true')
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: datadog-logs
         uses: nick-fields/retry@v2
         with:
@@ -188,7 +184,7 @@ jobs:
           command: bash scripts/ci-integration-test.sh  datadog-logs
 
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
-          (github.event_name != 'pull_request' || needs.detect-user.outputs.IS_TEAM_MEMBER == 'true')
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: datadog-metrics
         uses: nick-fields/retry@v2
         with:
@@ -197,7 +193,7 @@ jobs:
           command: bash scripts/ci-integration-test.sh  datadog-metrics
 
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
-          (github.event_name != 'pull_request' || needs.detect-user.outputs.IS_TEAM_MEMBER == 'true')
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: datadog-traces
         uses: nick-fields/retry@v2
         with:
@@ -398,6 +394,7 @@ jobs:
     runs-on: ubuntu-latest
     if: always()
     needs:
+      - changes
       - integration-tests
     env:
       FAILED: ${{ contains(needs.*.result, 'failure') }}

.github/workflows/k8s_e2e.yml

@@ -225,7 +225,11 @@ jobs:
   final-result:
     name: K8s E2E Suite
     runs-on: ubuntu-latest
-    needs: test-e2e-kubernetes
+    needs:
+      - changes
+      - build-x86_64-unknown-linux-gnu
+      - compute-k8s-test-plan
+      - test-e2e-kubernetes
     if: always()
     env:
       FAILED: ${{ contains(needs.*.result, 'failure') }}

.github/workflows/test.yml

@@ -127,7 +127,7 @@ jobs:
     name: Test Suite
     runs-on: ubuntu-20.04
     if: always()
-    needs: checks
+    needs: [changes, checks]
     env:
       FAILED: ${{ contains(needs.*.result, 'failure') }}
     steps: