Merge branch 'master' into alex/support_3.8

Author: girarda

.editorconfig

@@ -137,13 +137,3 @@ ij_yaml_sequence_on_new_line = false
 ij_yaml_space_before_colon = false
 ij_yaml_spaces_within_braces = true
 ij_yaml_spaces_within_brackets = true
-
-# don't interfere with prettier default formatting for webapp
-# see https://prettier.io/docs/en/api.html#prettierresolveconfigfilepath--options
-# for options that we're disabling
-[/airbyte-webapp/**]
-end_of_line = unset
-indent_style = unset
-indent_size = unset
-tab_width = unset
-max_line_length = unset

.github/labeler.yml

@@ -13,12 +13,6 @@ area/platform:
   - charts/*
   - charts/**/*
 
-area/frontend:
-  - airbyte-webapp/*
-  - airbyte-webapp/**/*
-  - airbyte-webapp-e2e-tests/*
-  - airbyte-webapp-e2e-tests/**/*
-
 area/api:
   - airbyte-api/*
   - airbyte-api/**/*

.github/workflows/gradle.yml

@@ -50,6 +50,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       build: ${{ steps.filter.outputs.build }}
+      cdk: ${{ steps.filter.outputs.cdk }}
       cli: ${{ steps.filter.outputs.cli }}
       connectors: ${{ steps.filter.outputs.connectors }}
       db: ${{ steps.filter.outputs.db }}
@@ -69,11 +70,12 @@ jobs:
               - '*.gradle'
               - 'deps.toml'
               - 'airbyte-config/**'
+            cdk:
+              - 'airbyte-cdk/**'
             cli:
               - 'airbyte-api/**'
               - 'octavia-cli/**'
             connectors:
-              - 'airbyte-cdk/**'
               - 'airbyte-integrations/**'
               - 'airbyte-connector-test-harnesses/acceptance-test-harness/**'
             db:
@@ -156,6 +158,67 @@ jobs:
           attempt_limit: 3
           attempt_delay: 5000 # in ms
 
+  cdk-build:
+    needs: changes
+    runs-on: ubuntu-latest
+    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
+    if: needs.changes.outputs.cdk == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
+    name: "Airbyte CDK: Build"
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+
+      - name: Cache Build Artifacts
+        uses: ./.github/actions/cache-build-artifacts
+        with:
+          cache-key: ${{ secrets.CACHE_VERSION }}
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "17"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m \
+            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Format
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CDK ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
+
+      - name: Commit Formatting Changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Automated Commit - Formatting Changes
+          commit_user_name: Octavia Squidington III
+          commit_user_email: [email protected]
+
+      - name: Build
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CDK ./gradlew build --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
+
+
   # Connectors Base
   # In case of self-hosted EC2 errors, remove this block.
   start-connectors-base-build-runner:

.github/workflows/publish-cdk-command-manually.yml

@@ -31,6 +31,35 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  # We are using these runners because they are the same as the one for `publish-command.yml`
+  # One problem we had using `ubuntu-latest` for example is that the user is not root and some commands would fail in
+  # `manage.sh` (specifically `apt-get`)
+  start-publish-docker-image-runner-0:
+    name: Start Build EC2 Runner 0
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          repository: airbytehq/airbyte
+          ref: master
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
+            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ env.PAT }}
+          label: ${{ github.run_id }}-publisher
+
   bump-version:
     if: github.event.inputs.release-type != 'none'
     runs-on: ubuntu-latest
@@ -54,7 +83,7 @@ jobs:
       - name: Commit and Push Changes
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
-          file_pattern: airbyte-cdk/python/setup.py airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md
+          file_pattern: airbyte-cdk/python/setup.py airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md airbyte-cdk/python/Dockerfile
           commit_message: 🤖 Bump ${{ github.event.inputs.release-type }} version of Airbyte CDK
           commit_user_name: Octavia Squidington III
           commit_user_email: [email protected]
@@ -104,7 +133,7 @@ jobs:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
       - name: Build CDK Package
-        run: SUB_BUILD=CONNECTORS_BASE ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
+        run: SUB_BUILD=CDK ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
       - name: Post failure to Slack channel dev-connectors-extensibility
         if: ${{ failure() }}
         uses: slackapi/[email protected]
@@ -165,6 +194,71 @@ jobs:
         env:
           TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
           TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+      - name: Post failure to Slack channel dev-connectors-extensibility
+        if: ${{ failure() }}
+        uses: slackapi/[email protected]
+        continue-on-error: true
+        with:
+          channel-id: C04J1M66D8B
+          payload: |
+            {
+                "text": "Error during `publish-cdk` while publishing Airbyte CDK!",
+                "blocks": [
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "Error while publishing Airbyte CDK!"
+                        }
+                    },
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
+                        }
+                    }
+                ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+
+
+  publish-docker-image:
+    timeout-minutes: 240
+    needs:
+      - start-publish-docker-image-runner-0
+      - publish-cdk
+    runs-on: runner-pool-${{ github.run_id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          repository: airbytehq/airbyte
+          ref: master
+      - name: Install Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "17"
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Publish source-declarative-manifest
+        id: publish
+        env:
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
+          TZ: UTC
+        uses: Wandalen/wretry.action@master
+        with:
+          command: |
+            docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
+            ./tools/integrations/manage.sh publish airbyte-cdk/python false
+          attempt_limit: 3
+          attempt_delay: 5000 in # ms
       - name: Post success to Slack channel dev-connectors-extensibility
         uses: slackapi/[email protected]
         continue-on-error: true
@@ -220,3 +314,33 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-publish-docker-image-runner-0:
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    name: Stop Build EC2 Runner
+    needs:
+      - start-publish-docker-image-runner-0 # required to get output from the start-runner job
+      - publish-docker-image # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
+            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
+      - name: Stop EC2 runner
+        uses: airbytehq/[email protected]
+        with:
+          mode: stop
+          github-token: ${{ env.PAT }}
+          label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }}
+          ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }}

.github/workflows/test-command.yml

@@ -23,6 +23,9 @@ on:
         description: "Set a specific connector acceptance test version to use. Enter 'dev' to test, build and use a local version of Connector Acceptance Test."
         required: false
         default: "latest"
+      local_cdk:
+        description: "Run Connector Acceptance Tests against the CDK version on the current branch."
+        required: false
 jobs:
   uuid:
     name: "Custom UUID of workflow run"
@@ -119,7 +122,7 @@ jobs:
           S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
         uses: Wandalen/wretry.action@master
         with:
-          command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
+          command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} ${{ github.event.inputs.local_cdk }}
           attempt_limit: 3
           attempt_delay: 10000 # in ms
       - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}

.gitignore

@@ -63,9 +63,6 @@ crash.log
 resources/examples/airflow/logs/*
 !resources/examples/airflow/logs/.gitkeep
 
-# Cloud Demo
-!airbyte-webapp/src/packages/cloud/data
-
 # Summary.md keeps getting added and we just don't like it
 docs/SUMMARY.md