Merge branch 'master' into btkcodedev/builder-contribute/source-appcues

Author: btkcodedev

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.57.4
+current_version = 0.64.3
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.devcontainer/python-connectors-generic/devcontainer.json

@@ -2,7 +2,7 @@
 {
   "name": "Python Development DevContainer (Generic)",
 
-  "image": "mcr.microsoft.com/devcontainers/python:0-3.10",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.10",
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker": {},
     "ghcr.io/devcontainers/features/python:1": {

.github/CODEOWNERS

@@ -4,23 +4,31 @@
 /airbyte-integrations/connectors/destination-milvus @airbytehq/ai-language-models
 /airbyte-integrations/connectors/destination-qdrant @airbytehq/ai-language-models
 /airbyte-integrations/connectors/destination-chroma @airbytehq/ai-language-models
+/airbyte-integrations/connectors/destination-snowflake-cortex @airbytehq/ai-language-models
 /airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based @airbytehq/ai-language-models
 
 # CI/CD
-/.github/ @airbytehq/connector-extensibility
-/airbyte-ci/ @airbytehq/connector-extensibility
+/.github/ @airbytehq/dev-tooling
+/airbyte-ci/ @airbytehq/dev-tooling
 
 # Python CDK and Connector Acceptance Tests
-/airbyte-cdk/python @airbytehq/connector-extensibility
-/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
-/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-extensibility @lazebnyi @oustynova
+/airbyte-cdk/python @airbytehq/python-team
+/airbyte-integrations/connector-templates/ @airbytehq/dev-marketplace-contributions
+/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-tooling
 
 # Build customization file change
-/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-extensibility
+/airbyte-integrations/connectors/**/build_customization.py @airbytehq/dev-tooling
 
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
+# Bulk CDK
+/airbyte-cdk/bulk @airbytehq/dbsources @airbytehq/destinations
+/airbyte-cdk/bulk/core/extract/ @airbytehq/dbsources
+/airbyte-cdk/bulk/core/load/ @airbytehq/destinations
+/airbyte-cdk/bulk/toolkits/extract-*/ @airbytehq/dbsources
+/airbyte-cdk/bulk/toolkits/load-*/ @airbytehq/destinations
+
 # Java CDK
 /airbyte-cdk/java/airbyte-cdk @airbytehq/dbsources @airbytehq/destinations
 /airbyte-cdk/java/airbyte-cdk/*-sources/ @airbytehq/dbsources
@@ -51,3 +59,10 @@
 /airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/destinations
+
+# Python critical connectors
+/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-hubspot/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-salesforce/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-shopify/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-stripe/ @airbytehq/python-team

.github/actions/airbyte-ci-requirements/action.yml

@@ -10,31 +10,41 @@ inputs:
     description: "Path to airbyte-ci source"
     required: false
     default: airbyte-ci/connectors/pipelines
+  is_fork:
+    description: "Whether the PR is from a fork"
+    required: false
+    default: "false"
 runs:
   using: "composite"
   steps:
     - name: Get changed files
-      uses: tj-actions/changed-files@v39
+      uses: tj-actions/changed-files@v44
+      # When the PR is from a fork, we always install from binary, so we don't need to check for changes
+      if: inputs.is_fork == 'false'
       id: changes
       with:
         files_yaml: |
           pipelines:
             - '${{ inputs.path_to_airbyte_ci_source }}/**'
 
-    - name: Determine how Airbyte CI should be installed
+    - name: "Determine how Airbyte CI should be installed"
       shell: bash
       id: determine-install-mode
+      # When the PR is from a fork, we always install from binary
+      if: inputs.is_fork == 'false'
       run: |
         if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
           echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
           echo "install-mode=source" >> $GITHUB_OUTPUT
+          echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV
         else
           echo "install-mode=binary" >> $GITHUB_OUTPUT
+          echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV
         fi
 
     - name: Install Airbyte CI from binary
       id: install-airbyte-ci-binary
-      if: steps.determine-install-mode.outputs.install-mode == 'binary'
+      if: steps.determine-install-mode.outputs.install-mode == 'binary' || ${{ inputs.is_fork }} == 'true'
       shell: bash
       run: |
         curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin

.github/actions/install-airbyte-ci/action.yml

@@ -9,16 +9,16 @@ inputs:
     required: true
   github_token:
     description: "GitHub token"
-    required: true
+    required: false
   dagger_cloud_token:
     description: "Dagger Cloud token"
-    required: true
+    required: false
   docker_hub_username:
     description: "Dockerhub username"
-    required: true
+    required: false
   docker_hub_password:
     description: "Dockerhub password"
-    required: true
+    required: false
   options:
     description: "Options for the subcommand"
     required: false
@@ -34,6 +34,14 @@ inputs:
     description: "GCP credentials for GCP Secret Manager"
     required: false
     default: ""
+  gcp_integration_tester_credentials:
+    description: "GCP credentials for integration tests"
+    required: false
+    default: ""
+  git_repo_url:
+    description: "Git repository URL"
+    default: https://github.com/airbytehq/airbyte.git
+    required: false
   git_branch:
     description: "Git branch to checkout"
     required: false
@@ -79,74 +87,110 @@ inputs:
   python_registry_token:
     description: "Python registry API token to publish python package"
     required: false
+  is_fork:
+    description: "Whether the PR is from a fork"
+    required: false
+    default: "false"
+  max_attempts:
+    description: "Number of attempts at running the airbyte-ci command"
+    required: false
+    default: 1
+  retry_wait_seconds:
+    description: "Number of seconds to wait between retry attempts"
+    required: false
+    default: 60
 
 runs:
   using: "composite"
   steps:
     - name: Get start timestamp
       id: get-start-timestamp
       shell: bash
-      run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
-
-    - name: Check if PR is from a fork
-      id: check-if-pr-is-from-fork
-      if: github.event_name == 'pull_request'
-      shell: bash
-      run: |
-        if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
-          echo "PR is from a fork. Exiting workflow..."
-          exit 78
-        fi
-
+      run: echo "start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
     - name: Docker login
       id: docker-login
       uses: docker/login-action@v3
+      if: ${{ inputs.docker_hub_username != '' && inputs.docker_hub_password != '' }}
       with:
         username: ${{ inputs.docker_hub_username }}
         password: ${{ inputs.docker_hub_password }}
-
     - name: Install Airbyte CI
       id: install-airbyte-ci
       uses: ./.github/actions/install-airbyte-ci
       with:
         airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url }}
-
+        is_fork: ${{ inputs.is_fork }}
     - name: Run airbyte-ci
       id: run-airbyte-ci
-      shell: bash
-      run: |
-        airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+      uses: nick-fields/retry@v3
       env:
+        CI: "True"
+        CI_GIT_USER: ${{ github.repository_owner }}
+        CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
+        PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+        # Next environment variables are workflow inputs based and can be set with empty values if the inputs are not required and passed
         CI_CONTEXT: "${{ inputs.context }}"
         CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }}
+        CI_GIT_REPO_URL: ${{ inputs.git_repo_url }}
         CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }}
         CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }}
         CI_JOB_KEY: ${{ inputs.ci_job_key }}
-        CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
         CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }}
-        CI: "True"
         DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}"
         DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }}
         DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }}
         GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }}
+        GCP_INTEGRATION_TESTER_CREDENTIALS: ${{ inputs.gcp_integration_tester_credentials }}
         GCS_CREDENTIALS: ${{ inputs.gcs_credentials }}
         METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }}
         METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }}
         PRODUCTION: ${{ inputs.production }}
-        PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
         PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
         PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }}
-        PYTHON_REGISTRY_CHECK_URL: ${{ inputs.python_registry_check_url }}
         S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
         S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
         SENTRY_DSN: ${{ inputs.sentry_dsn }}
-        SENTRY_ENVIRONMENT: ${{ steps.determine-install-mode.outputs.install-mode }}
         SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
         SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
         SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
-    # give the Dagger Engine more time to push cache data to Dagger Cloud
+      with:
+        shell: bash
+        max_attempts: ${{ inputs.max_attempts }}
+        retry_wait_seconds: ${{ inputs.retry_wait_seconds }}
+        # 360mn > 6 hours: it's the GitHub runner max job duration
+        timeout_minutes: 360
+        command: |
+          airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
     - name: Stop Engine
       id: stop-engine
       if: always()
       shell: bash
-      run: docker stop --time 300 $(docker ps --filter name="dagger-engine-*" -q)
+      run: |
+        mapfile -t containers < <(docker ps --filter name="dagger-engine-*" -q)
+        if [[ "${#containers[@]}" -gt 0 ]]; then
+          # give 5mn to the Dagger Engine to push cache data to Dagger Cloud
+          docker stop -t 300 "${containers[@]}";
+        fi
+
+    - name: Collect dagger engine logs
+      id: collect-dagger-engine-logs
+      if: always()
+      uses: jwalton/gh-docker-logs@v2
+      with:
+        dest: "./dagger_engine_logs"
+        images: "registry.dagger.io/engine"
+
+    - name: Tar logs
+      id: tar-logs
+      if: always()
+      shell: bash
+      run: tar cvzf ./dagger_engine_logs.tgz ./dagger_engine_logs
+
+    - name: Upload logs to GitHub
+      id: upload-dagger-engine-logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ github.job }}_dagger_engine_logs.tgz
+        path: ./dagger_engine_logs.tgz
+        retention-days: 7