Merge remote-tracking branch 'origin/master' into alvova/9052-zendesk-support

Author: annalvova05

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.9-alpha
+current_version = 0.35.15-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.9-alpha
+VERSION=0.35.15-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.env.dev

@@ -27,4 +27,3 @@ SYNC_JOB_MAX_TIMEOUT_DAYS=3
 
 # Sentry
 SENTRY_DSN=""
-

.github/workflows/platform-project-automation.yml

@@ -0,0 +1,34 @@
+# See https://github.com/marketplace/actions/project-beta-automations for guidance
+
+name: Platform Project Automation
+on:
+  issues:
+    types: [labeled]
+
+env:
+  GH_PROJECT_TOKEN: ${{ secrets.PARKER_PAT_FOR_PLATFORM_PROJECT_AUTOMATION }}
+  ORG: airbytehq
+  PROJECT_ID: 6 # https://github.com/orgs/airbytehq/projects/6/views/8
+  FIELD_STATUS: Status
+  STATUS_TODO: Todo
+  FIELD_DATE_ADDED: Date Added
+
+jobs:
+  add-area-platform-issues-to-platform-project:
+    runs-on: ubuntu-latest
+    name: Add area/platform issue to Platform Project
+    steps:
+      - name: Set current date env var
+        id: set_date
+        run: echo ::set-output name=CURRENT_DATE::$(date +'%Y-%m-%dT%H:%M:%S%z')
+      
+      - name: Add issue to project if labelled with area/platform
+        uses: leonsteinhaeuser/[email protected]
+        if: contains(github.event.issue.labels.*.name, 'area/platform')
+        with:
+          gh_token: ${{ env.GH_PROJECT_TOKEN }}
+          organization: ${{ env.ORG }}
+          project_id: ${{ env.PROJECT_ID }}
+          resource_node_id: ${{ github.event.issue.node_id }}
+          operation_mode: custom_field
+          custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'

.github/workflows/publish-command.yml

@@ -67,10 +67,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
-      - name: Install Unzip for Databricks
-        if: github.event.inputs.connector == 'connectors/destination-databricks'
-        run: |
-          apt-get update && apt-get install -y unzip
       - name: Install Java
         uses: actions/setup-java@v1
         with:
@@ -94,15 +90,39 @@ jobs:
           ci_credentials ${{ github.event.inputs.connector }}
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
-      - run: |
+      - name: Prepare Sentry Release
+        if: startsWith(github.event.inputs.connector, 'connectors')
+        run: |
+          curl -sL https://sentry.io/get-cli/ | bash
+          source tools/lib/lib.sh
+          DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile
+          echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV
+          echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV
+      - name: Create Sentry Release
+        if: startsWith(github.event.inputs.connector, 'connectors')
+        run: |
+          sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
+          SENTRY_ORG: airbyte-5j
+          SENTRY_PROJECT: airbyte-connectors
+      - name: Publish ${{ github.event.inputs.connector }}
+        run: |
           echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
           ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
-        name: publish ${{ github.event.inputs.connector }}
         id: publish
         env:
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
+      - name: Finalize Sentry release
+        if: startsWith(github.event.inputs.connector, 'connectors')
+        run: |
+          sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}"
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
+          SENTRY_ORG: airbyte-5j
+          SENTRY_PROJECT: airbyte-connectors
       - name: Add Success Comment
         if: github.event.inputs.comment-id && success()
         uses: peter-evans/create-or-update-comment@v1

.github/workflows/publish-external-command.yml

@@ -0,0 +1,113 @@
+name: Publish External Connector Image
+on:
+  workflow_dispatch:
+    inputs:
+      connector:
+        description: "Airbyte Connector image"
+        required: true
+      version:
+        description: "Airbyte Connector version"
+        required: true
+      comment-id:
+        description: "The comment-id of the slash command. Used to update the comment with the status."
+        required: false
+
+jobs:
+  ## Gradle Build
+  # In case of self-hosted EC2 errors, remove this block.
+  start-publish-image-runner:
+    name: Start Build EC2 Runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          # 80 gb disk
+          ec2-image-id: ami-0d648081937c75a73
+  publish-image:
+    needs: start-publish-image-runner
+    runs-on: ${{ needs.start-publish-image-runner.outputs.label }}
+    environment: more-secrets
+    steps:
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
+          export_default_credentials: true
+      - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
+      - run: |
+          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
+          ./tools/integrations/manage.sh publish_external ${{ github.event.inputs.connector }} ${{ github.event.inputs.version }}
+        name: publish ${{ github.event.inputs.connector }}
+        id: publish
+        env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
+          TZ: UTC
+      - name: Add Success Comment
+        if: github.event.inputs.comment-id && success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Add Failure Comment
+        if: github.event.inputs.comment-id && !success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Slack Notification - Failure
+        if: failure()
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildozer
+          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
+          SLACK_COLOR: DC143C
+          SLACK_TITLE: "Failed to publish connector ${{ github.event.inputs.connector }} from branch ${{ github.ref	 }}"
+          SLACK_FOOTER: ""
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-publish-image-runner:
+    name: Stop Build EC2 Runner
+    needs:
+      - start-publish-image-runner # required to get output from the start-runner job
+      - publish-image # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-publish-image-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }}

.github/workflows/slash-commands.yml

@@ -19,6 +19,7 @@ jobs:
             test
             test-performance
             publish
+            publish-external
             publish-cdk
             gke-kube-test
             run-specific-test

.github/workflows/test-command.yml

@@ -62,10 +62,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ${{ github.event.inputs.repo }}
-      - name: Install Unzip for Databricks
-        if: github.event.inputs.connector == 'connectors/destination-databricks'
-        run: |
-          apt-get update && apt-get install -y unzip
       - name: Install Java
         uses: actions/setup-java@v1
         with:

.pre-commit-config.yaml

@@ -11,11 +11,17 @@ repos:
     rev: 21.11b1
     hooks:
       - id: black
+        args: ["--line-length=140"]
   - repo: https://github.com/timothycrosley/isort
     rev: 5.10.1
     hooks:
       - id: isort
-        args: ["--dont-follow-links", "--jobs=-1"]
+        args:
+          [
+            "--settings-path=tools/python/.isort.cfg",
+            "--dont-follow-links",
+            "--jobs=-1",
+          ]
         additional_dependencies: ["colorama"]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.0
@@ -34,12 +40,14 @@ repos:
     rev: v0.0.1a2.post1
     hooks:
       - id: pyproject-flake8
+        args: ["--config=tools/python/.flake8"]
         additional_dependencies: ["mccabe"]
         alias: flake8
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.910-1
     hooks:
       - id: mypy
+        args: ["--config-file=tools/python/.mypy.ini"]
         exclude: |
           (?x)^.*(
               octavia-cli/unit_tests/|

airbyte-api/src/main/openapi/config.yaml

@@ -18,6 +18,7 @@ info:
     * All backwards incompatible changes will happen in major version bumps. We will not make backwards incompatible changes in minor version bumps. Examples of non-breaking changes (includes but not limited to...):
       * Adding fields to request or response bodies.
       * Adding new HTTP endpoints.
+    * All `web_backend` APIs are not considered public APIs and are not guaranteeing backwards compatibility.
 
   version: "1.0.0"
   title: Airbyte Configuration API
@@ -53,7 +54,9 @@ tags:
   - name: db_migration
     description: Database migration related resources.
   - name: web_backend
-    description: Connection between sources and destinations.
+    description: |
+      Endpoints for the Airbyte web application. Those APIs should not be called outside the web application implementation and are not
+      guaranteeing any backwards compatibility.
   - name: health
     description: Healthchecks
   - name: deployment
@@ -182,6 +185,29 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/workspaces/update_name:
+    post:
+      tags:
+        - workspace
+      summary: Update workspace name
+      operationId: updateWorkspaceName
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/WorkspaceUpdateName"
+        required: true
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WorkspaceRead"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "422":
+          $ref: "#/components/responses/InvalidInputResponse"
   /v1/workspaces/tag_feedback_status_as_done:
     post:
       tags:
@@ -1969,6 +1995,16 @@ components:
           type: boolean
         feedbackDone:
           type: boolean
+    WorkspaceUpdateName:
+      type: object
+      required:
+        - workspaceId
+        - name
+      properties:
+        workspaceId:
+          $ref: "#/components/schemas/WorkspaceId"
+        name:
+          type: string
     WorkspaceUpdate:
       type: object
       required:

airbyte-bootloader/Dockerfile

@@ -5,6 +5,6 @@ ENV APPLICATION airbyte-bootloader
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.9-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.15-alpha.tar /app
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.15-alpha/bin/${APPLICATION}"]

airbyte-bootloader/build.gradle

@@ -11,7 +11,9 @@ dependencies {
     implementation project(':airbyte-db:lib')
     implementation project(":airbyte-json-validation")
     implementation project(':airbyte-scheduler:persistence')
+    implementation project(':airbyte-scheduler:models')
 
+    implementation 'io.temporal:temporal-sdk:1.6.0'
     implementation "org.flywaydb:flyway-core:7.14.0"
 
     testImplementation "org.testcontainers:postgresql:1.15.3"