Merge branch 'master' into leti/experiment-add-source-in-signup

* master: (59 commits)
  🪟🔧 Remove styled components (round 1) (#18766)
  Bump Airbyte version from 0.40.17 to 0.40.18 (#18827)
  ci: add job and run id to test reports (#18832)
  hide ConfigPersistence inside ConfigRepository to discourage use (#18803)
  Remove the bulk actions from ConfigPersistence (#18800)
  remove config persistence from seeding logic (#18749)
  Remove ConfigPersistence usage from SecretsMigrator (#18747)
  Add normalization changelog and bump normalization version in platform (#18813)
  🐛Source Exchange Rates: Fix handling error during check connection (#18726)
  🐛Destination Google Sheets: Fix empty headers list (#18729)
  Bump helm chart version reference to 0.40.40 (#18815)
  Use equalsIgnoreCase (#18810)
  [charts/airbyte-cron] Cleanup env vars (#18787)
  🐛 Source Facebook Marketing: reduce request limit after specific error (#18734)
  Parameterize test_empty_streams and test_stream_with_1_airbyte_column by destination (#18197)
  Correct coinmarket spec (#18790)
  ci: use custom test-reporter action to upload job results (#18004)
  Fix unit tests in source relational db (#18789)
  query to include data plane attributes (#18531)
  Mark/update notification settings design (#18159)
  ...

Author: letiescanciano

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.17
+current_version = 0.40.18
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -30,6 +30,8 @@ serialize =
 
 [bumpversion:file:charts/airbyte-bootloader/Chart.yaml]
 
+[bumpversion:file:charts/airbyte-cron/Chart.yaml]
+
 [bumpversion:file:charts/airbyte-server/Chart.yaml]
 
 [bumpversion:file:charts/airbyte-temporal/Chart.yaml]

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.17
+VERSION=0.40.18
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/actions/build-branch/action.yml

@@ -6,7 +6,7 @@ inputs:
     required: false
 outputs:
   branch_version_tag:
-    description: 'Tag used for jars and docker images. Either user specified or auto generated as `dev-<commit_hash>`'
+    description: "Tag used for jars and docker images. Either user specified or auto generated as `dev-<commit_hash>`"
     value: ${{ steps.parse-input.outputs.branch_version_tag }}
 runs:
   using: "composite"
@@ -17,8 +17,8 @@ runs:
       run: |-
         # if the *branch_version_tag* input param is not specified, then generate it as 'dev-<commit_hash>`
         #
-        [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "::set-output name=branch_version_tag::${{ inputs.branch_version_tag }}" \
-          || { short_hash=$(git rev-parse --short HEAD); echo "::set-output name=branch_version_tag::dev-$short_hash"; }
+        [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "branch_version_tag=${{ inputs.branch_version_tag }}" >> $GITHUB_OUTPUT \
+          || { short_hash=$(git rev-parse --short HEAD); echo "branch_version_tag=dev-$short_hash" >> $GITHUB_OUTPUT ; }
 
     - uses: actions/setup-java@v1
       with:
@@ -49,4 +49,3 @@ runs:
     - name: Publish to Maven Local
       run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew publishToMavenLocal
       shell: bash
-

.github/actions/ci-py-tests/action.yml

@@ -32,7 +32,7 @@ runs:
         REPORT_FOLDER="${{ inputs.module-folder }}/coverage/"
         ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
-        echo "::set-output name=coverage-paths::coverage/coverage.xml"
+        echo "coverage-paths=coverage/coverage.xml" >> $GITHUB_OUTPUT
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v2
@@ -48,7 +48,7 @@ runs:
         REPORT_FOLDER="${{ inputs.module-folder }}/reports/"
         ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
-        echo "::set-output name=mypy-logs::reports/mypy.log"
-        echo "::set-output name=black-diff::reports/black.diff"
-        echo "::set-output name=isort-diff::reports/isort.diff"
-        echo "::set-output name=flake8-logs::reports/flake.txt"
+        echo "mypy-logs=reports/mypy.log" >> $GITHUB_OUTPUT
+        echo "black-diff=reports/black.diff" >> $GITHUB_OUTPUT
+        echo "isort-diff=reports/isort.diff" >> $GITHUB_OUTPUT
+        echo "flake8-logs=reports/flake.txt" >> $GITHUB_OUTPUT

.github/actions/ci-tests-runner/action.yml

@@ -67,8 +67,8 @@ runs:
         IPS=($(hostname -I))
         LOCAL_IP_PORT="${IPS[0]}:${PORT}"
         gcloud compute start-iap-tunnel sonarqube-1-vm 80 --local-host-port=${LOCAL_IP_PORT} --zone=europe-central2-a --project dataline-integration-testing &
-        echo ::set-output name=pid::$!
-        echo "::set-output name=sonar-host::http://${LOCAL_IP_PORT}/"
+        echo pid=$! >> $GITHUB_OUTPUT
+        echo "sonar-host=http://${LOCAL_IP_PORT}/" >> $GITHUB_OUTPUT
         echo "::echo::on"
 
     - name: Python Tests
@@ -124,18 +124,18 @@ runs:
         fi
 
         # join the array to string format
-        echo ::set-output name=external_reports::$(IFS=, ; echo "${REPORT_FILES[*]}")
-        echo ::set-output name=options::$(IFS=' ' ; echo "${OPTIONS[*]}")
+        echo external_reports=$(IFS=, ; echo "${REPORT_FILES[*]}") >> $GITHUB_OUTPUT
+        echo options=$(IFS=' ' ; echo "${OPTIONS[*]}") >> $GITHUB_OUTPUT
 
     - name: Create SonarQube Project
       shell: bash
       id: create-sq-project
       run: |
         ci_sonar_qube --pr ${{ inputs.pull-request-id }} --create --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
-        echo "::set-output name=sq_project_name::$(ci_sonar_qube --pr ${{ inputs.pull-request-id }} --print_key --module ${{ inputs.module-name }})"
+        echo "sq_project_name=$(ci_sonar_qube --pr ${{ inputs.pull-request-id }} --print_key --module ${{ inputs.module-name }})" >> $GITHUB_OUTPUT
         ROOT_DIR=$(git rev-parse --show-toplevel)
         MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))")
-        echo "::set-output name=module_dir::${MODULE_DIR}"
+        echo "module_dir::${MODULE_DIR}" >> $GITHUB_OUTPUT
 
     - name: SonarQube Scan
 
@@ -168,7 +168,7 @@ runs:
         body="${body//'%'/'%25'}"
         body="${body//$'\n'/'%0A'}"
         body="${body//$'\r'/'%0D'}"
-        echo "::set-output name=sq-report::$body"
+        echo "sq-report=$body" >> $GITHUB_OUTPUT
 
     - name: Add Comment
       if: ${{ github.event_name == 'pull_request' }}

.github/actions/start-aws-runner/action.yml

@@ -9,7 +9,7 @@ inputs:
     required: true
   ec2-image-id:
     # github-self-hosted-runner-ubuntu-20-100g-disk-with-cypress-deps
-    default: "ami-0f23be2f917510c26"
+    default: "ami-005924fb76f7477ce"
     required: true
   ec2-instance-type:
     default: "c5.2xlarge"

.github/comment_templates/connector_dependency_template.md

@@ -0,0 +1,25 @@
+<!--- this comment is for `report-connectors-dependency.yml` identification, do not remove -->
+
+NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to run corresponding integration tests:
+
+<details>
+<summary>
+Sources ({num_sources})
+
+</summary>
+
+{sources}
+
+</details>
+
+<details>
+<summary>
+Destinations ({num_destinations})
+
+</summary>
+
+{destinations}
+
+</details>
+
+{others}

.github/workflows/commands-for-testing-tool.yml

@@ -26,12 +26,12 @@ jobs:
         id: getref
         run: |
           pr_info="$(curl ${{ github.event.issue.pull_request.url }})"
-          echo ::set-output name=ref::"$(echo $pr_info | jq -r '.head.ref')"
-          echo ::set-output name=repo::"$(echo $pr_info | jq -r '.head.repo.full_name')"
+          echo ref="$(echo $pr_info | jq -r '.head.ref')" >> $GITHUB_OUTPUT
+          echo repo="$(echo $pr_info | jq -r '.head.repo.full_name')" >> $GITHUB_OUTPUT
       - name: Get comment id
         id: comment-info
         run: |
-          echo ::set-output name=comment-id::"${{ github.event.comment.id }}"
+          echo comment-id="${{ github.event.comment.id }}" >> $GITHUB_OUTPUT
       - name: Get command
         id: regex
         uses: AsasInnab/regex-action@v1
@@ -139,4 +139,4 @@ jobs:
           comment-id: ${{ needs.set-params.outputs.comment-id }}
           body: |
             > :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
-          reactions: -1
+          reactions: -1

.github/workflows/create-release.yml

@@ -32,7 +32,7 @@ jobs:
           # The sed removes carriage returns so that the body is easier to parse later, and
           #   escapes backticks so that they are not executed as commands.
           PR_BODY=$(printf '%s' "$PR" | jq '.[0].body' | sed 's/\\r//g' | sed 's/`/\\`/g')
-          echo ::set-output name=pr_body::${PR_BODY}
+          echo pr_body=${PR_BODY} >> $GITHUB_OUTPUT
       - name: Extract Changelog
         id: extract_changelog
         shell: bash
@@ -61,7 +61,7 @@ jobs:
         shell: bash
         run: |
           VERSION=$(grep -w VERSION .env | cut -d"=" -f2)
-          echo ::set-output name=VERSION::${VERSION}
+          echo VERSION=${VERSION} >> $GITHUB_OUTPUT
       - name: Create Release
         id: create_release
         uses: ncipollo/release-action@v1

.github/workflows/deploy-docs-site.yml

@@ -40,12 +40,11 @@ jobs:
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
         with:
-          # 'C03BEADRPNY' channel => '#oss-master-build-failure'
+          # 'C032Y32T065' channel => '#docs'
           args: >-
             {\"channel\":\"C03BEADRPNY\", \"blocks\":[
             {\"type\":\"divider\"},
             {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs build fails on the latest master :bangbang: \n\n\"}},
             {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
             {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
             {\"type\":\"divider\"}]}
-

.github/workflows/gradle.yml

@@ -86,6 +86,7 @@ jobs:
             ${{ secrets.SUPERTOPHER_PAT }} \
             ${{ secrets.DAVINCHIA_PAT }}
 
+
   # Uncomment to debug.
   #  changes-output:
   #    name: "Debug Change Detection Logic"
@@ -237,6 +238,37 @@ jobs:
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
 
+      - name: Publish Connectors Base Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        id: connectors-test-results
+        if: always()
+        with:
+          junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
+          comment_mode: off
+          json_file: connectors_base_results.json
+          json_test_case_results: true
+          check_name: "Connectors Base Test Results"
+          
+      - name: Setup Google Cloud SDK
+        if: always()
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GKE_TEST_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Prep Test Results For GCS
+        if: always()
+        run: |
+          python tools/bin/prep_test_results_for_gcs.py --json connectors_base_results.json  --jobid $GITHUB_JOB --runid $GITHUB_RUN_ID
+        
+      - name: Upload Test Results to GCS
+        if: always()
+        run: | 
+          gcs_bucket_name="dev-ab-ci-run-results"
+          filename=$(echo "${{ fromJSON( steps.connectors-test-results.outputs.json ).check_url }}" | sed 's@.*/@@')  
+          echo "$filename"
+          gsutil -h "Cache-Control:public" cp connectors_base_results.jsonl "gs://$gcs_bucket_name/oss/$filename.jsonl"
+        
       - name: Generate Test Report
         uses: dorny/test-reporter@v1
         if: always()
@@ -524,6 +556,42 @@ jobs:
       - name: Automatic Migration Acceptance Test
         run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
 
+      - uses: actions/setup-python@v2
+        if: always()
+        with:
+          python-version: "3.9"
+
+      - name: Publish Platform Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        id: platform-results
+        if: always()
+        with:
+          junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
+          comment_mode: off
+          json_file: platform_results.json
+          json_test_case_results: true
+          check_name: "Platform Test Results"
+
+      - name: Setup Google Cloud SDK
+        if: always()
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GKE_TEST_SA_KEY }}
+          export_default_credentials: true
+      
+      - name: Prep Test Results For GCS
+        if: always()
+        run: |
+          python tools/bin/prep_test_results_for_gcs.py --json platform_results.json  --jobid $GITHUB_JOB --runid $GITHUB_RUN_ID
+        
+      - name: Upload Test Results to GCS
+        if: always()
+        run: | 
+          gcs_bucket_name="dev-ab-ci-run-results"
+          filename=$(echo "${{ fromJSON( steps.platform-results.outputs.json ).check_url }}" | sed 's@.*/@@')  
+          echo "$filename"
+          gsutil -h "Cache-Control:public" cp platform_results.jsonl "gs://$gcs_bucket_name/oss/$filename.jsonl"
+        
       - name: Generate Test Report
         uses: dorny/test-reporter@v1
         if: always()    # run this step even if previous step failed
@@ -543,15 +611,6 @@ jobs:
           key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
           secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
 
-      - name: Upload test results to Github for analysis 
-        if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
-        uses: actions/upload-artifact@v3
-        with:
-          path: |
-            /actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml
-            /actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml
-          name: test-results-build
-
   # In case of self-hosted EC2 errors, remove this block.
   stop-platform-build-runner:
     name: "Platform: Stop Build EC2 Runner"
@@ -683,6 +742,42 @@ jobs:
         run: |
           CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
 
+      - uses: actions/setup-python@v2
+        if: always()
+        with:
+          python-version: "3.9"
+
+      - name: Publish Kube Test Results
+        id: kube-results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          junit_files: "/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml\n/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml"
+          comment_mode: off
+          json_file: kube_results.json
+          json_test_case_results: true
+          check_name: "Kube Test Results"
+      
+      - name: Setup Google Cloud SDK
+        if: always()
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GKE_TEST_SA_KEY }}
+          export_default_credentials: true
+      
+      - name: Prep Test Results For GCS
+        if: always()
+        run: |
+          python tools/bin/prep_test_results_for_gcs.py --json kube_results.json  --jobid $GITHUB_JOB --runid $GITHUB_RUN_ID
+        
+      - name: Upload Test Results to GCS
+        if: always()
+        run: | 
+          gcs_bucket_name="dev-ab-ci-run-results"
+          filename=$(echo "${{ fromJSON( steps.kube-results.outputs.json ).check_url }}" | sed 's@.*/@@')  
+          echo "$filename"
+          gsutil -h "Cache-Control:public" cp kube_results.jsonl "gs://$gcs_bucket_name/oss/$filename.jsonl"
+      
       - name: Generate Test Report
         uses: dorny/test-reporter@v1
         if: always()    # run this step even if previous step failed
@@ -701,20 +796,13 @@ jobs:
           key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
           secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
 
-      - name: Upload test results to Github for analysis 
-        if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
-        uses: actions/upload-artifact@v3
-        with:
-          path: |
-            /actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml
-            /actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml
-          name: test-results-kube
-
       - uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: Kubernetes Logs
           path: /tmp/kubernetes_logs/*
+
+
   # In case of self-hosted EC2 errors, remove this block.
   stop-kube-acceptance-test-runner:
     name: "Platform: Stop Kube Acceptance Test EC2 Runner"
@@ -861,13 +949,6 @@ jobs:
 #          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
 #        run: |
 #          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
-#      - name: Generate Test Report
-#        uses: dorny/test-reporter@v1
-#        if: always()    # run this step even if previous step failed
-#        with:
-#          name: Platform Helm E2E Test Report
-#          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
-#          reporter: java-junit
 #
 #      - uses: actions/upload-artifact@v2
 #        if: failure()

.github/workflows/publish-command.yml

@@ -175,7 +175,7 @@ jobs:
           stripped_connector="$(echo "${{ github.event.inputs.connector }}" | tr -d ' ')"
           middle=${stripped_connector//,/$replace}
           full="$start$middle$end"
-          echo "::set-output name=connectorjson::$full"
+          echo "connectorjson=$full" >> $GITHUB_OUTPUT
   write-initial-output-to-comment:
     name: Set up git comment
     if: github.event.inputs.comment-id