Merge branch 'master' into dev

* master: (142 commits)
  Highlight removed and added streams in Connection form (airbytehq#13392)
  🐛  Source Amplitude: Fixed JSON Validator `date-time` validation (airbytehq#13373)
  🐛 Source Mixpanel: publish v0.1.17 (airbytehq#13450)
  Fixed reverted PR: Fix cancel button when it doesn't provide feedback to the user + UX improvements (airbytehq#13388)
  🎉 Source Freshdesk: Added new streams (airbytehq#13332)
  Prepare YamlSeedConfigPersistence for dependency injection (airbytehq#13384)
  helm chart: Support nodeSelector, tolerations and affinity on the booloader pod (airbytehq#11467)
  airbyte-api: add jackson model annotations to remove null values from responses (airbytehq#13370)
  Change stage to `beta` (airbytehq#13422)
  🐛 Source Google Sheets: Retry on server errors (airbytehq#13446)
  Improve kube deploy process. (airbytehq#13397)
  Helm chart dependencies fix (airbytehq#13432)
  🐛 Source HubSpot: Transform `contact_lists` data to comply with schema (airbytehq#13218)
  airbytehq#11758: Source Google Ads to GA (airbytehq#13441)
  Add more pr actions to tag pull requests (airbytehq#13437)
  Source Google Ads: drop schema field that filters out the data from stream (airbytehq#13423)
  Updates error view with new design (airbytehq#13197)
  Source MSSQL: correct enum Standard method (airbytehq#13419)
  Update postgres doc about cdc publication (airbytehq#13433)
  run source acceptance tests against image built from branch (airbytehq#13401)
  ...

Author: chebalski

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.39.1-alpha
+current_version = 0.39.8-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.39.1-alpha
+VERSION=0.39.8-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/ISSUE_TEMPLATE/bug-report.md

@@ -8,6 +8,7 @@ assignees: ''
 ---
 <!-- 
 Welcome to Airbyte!
+
 We're really appreciate your report and know that this will help us build an 
 amazing tool. If you want to contribute yourself, you can find a good place 
 to start by searching for the good-first-issues label or maybe... by trying 
@@ -28,7 +29,6 @@ You can remove the examples bellow and fill out with your information.
 - **Deployment**: example are Docker or Kubernetes deploy env
 - **Source Connector and version**: (if applicable example Salesforce 0.2.3) <!-- Found in the admin page in the UI in the Source tab. -->
 - **Destination Connector and version**: (if applicable example Postgres 0.3.3) <!-- Found in the admin page in the UI in the Destination tab. -->
-- **Severity**: Very Low / Low / Medium / High / Critical
 - **Step where error happened**: Deploy / Sync job / Setup new connection / Update connector / Upgrade Airbyte
 
 ## Current Behavior
@@ -39,8 +39,11 @@ You can remove the examples bellow and fill out with your information.
 
 ## Logs
 <!--
-If applicable: Please upload the complete log files from the failing operation. 
-We ask for complete log files because these files capture essential metadata about your environment. The team will read the full stack trace of errors and try to find the root cause of the error. For a sync failure, you can download the log in the sync page, and for other errors please download the server and scheduler logs in the Settings Page -> Configuration.
+Please upload the complete log files from the failing operation. 
+
+We ask for complete log files because these files capture essential metadata about your environment.
+The team will read the full stack trace of errors and try to find the root cause of the error. 
+For a sync failure, you can download the log in the sync page, and for other errors please download the server and scheduler logs in the Settings Page -> Configuration.
 You can remove this after uploading the file.
 -->
 

.github/actions/build-and-push-branch/action.yml

@@ -4,6 +4,9 @@ inputs:
   branch_version_tag:
     description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-<commit_hash>" to pass AirbyteVersion validation)'
     required: false
+  dockerhub_username:
+    description: "Used to log in to dockerhub for pushing images"
+    required: true
   dockerhub_token:
     description: "Used to log in to dockerhub for pushing images"
     required: true
@@ -19,7 +22,7 @@ runs:
     - name: Login to Docker (on Master)
       uses: docker/login-action@v1
       with:
-        username: airbytebot
+        username: ${{ inputs.dockerhub_username }}
         password: ${{ inputs.dockerhub_token }}
 
     - name: Push Docker Images

.github/workflows/build-connector-command.yml

@@ -186,7 +186,8 @@ jobs:
         id: build
         env:
           PR_NUMBER: ${{ github.event.number }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
   #      - name: Test ${{ github.event.inputs.connector }}

.github/workflows/gke-kube-test-command.yml

@@ -116,7 +116,8 @@ jobs:
         env:
           USER: root
           HOME: /home/runner
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           ACTION_RUN_ID: ${{github.run_id}}
         run: |
           ./tools/bin/gke-kube-acceptance-test/acceptance_test_kube_gke.sh

.github/workflows/gradle.yml

@@ -204,6 +204,9 @@ jobs:
       - name: Install Pyenv
         run: python3 -m pip install virtualenv==16.7.9 --user
 
+      - name: Install automake
+        run: apt-get install -y automake build-essential libtool libtool-bin autoconf
+
       - name: Set up CI Gradle Properties
         run: |
           mkdir -p ~/.gradle/

.github/workflows/publish-command.yml

@@ -65,14 +65,19 @@ jobs:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
-          # 80 gb disk
-          ec2-image-id: ami-0d648081937c75a73
   publish-image:
     timeout-minutes: 240
     needs: start-publish-image-runner
     runs-on: ${{ needs.start-publish-image-runner.outputs.label }}
     environment: more-secrets
     steps:
+      - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v0
         with:
@@ -88,13 +93,6 @@ jobs:
       - name: Validate input workflow format
         if: steps.regex.outputs.first_match != github.event.inputs.connector
         run: echo "The connector provided has an invalid format!" && exit 1
-      - name: Link comment to workflow run
-        if: github.event.inputs.comment-id
-        uses: peter-evans/create-or-update-comment@v1
-        with:
-          comment-id: ${{ github.event.inputs.comment-id }}
-          body: |
-            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
       - name: Checkout Airbyte
         uses: actions/checkout@v2
         with:
@@ -155,12 +153,12 @@ jobs:
           SENTRY_PROJECT: airbyte-connectors
       - name: Publish ${{ github.event.inputs.connector }}
         run: |
-          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
+          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
           ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
         id: publish
         env:
-          DOCKER_USERNAME: airbytebot
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
       - name: Finalize Sentry release

.github/workflows/publish-external-command.yml

@@ -85,12 +85,13 @@ jobs:
           repository: ${{ gituhb.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
       - run: |
-          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
+          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
           ./tools/integrations/manage.sh publish_external ${{ github.event.inputs.connector }} ${{ github.event.inputs.version }}
         name: publish ${{ github.event.inputs.connector }}
         id: publish
         env:
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
       - name: Add Success Comment

.github/workflows/release-airbyte-os.yml

@@ -70,7 +70,8 @@ jobs:
       - name: Release Airbyte
         id: release_airbyte
         env:
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
           CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }}
           CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
@@ -100,7 +101,8 @@ jobs:
       - name: Release Octavia
         id: release_octavia
         env:
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
         run: ./tools/bin/release_version_octavia.sh
 

.github/workflows/shared-pulls.yml

@@ -1,7 +1,7 @@
 name: "Shared Pull Requests"
 on:
   pull_request_target:
-    types: [opened]
+    types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened]
 
 jobs:
   shared-pulls:

.github/workflows/test-command.yml

@@ -64,6 +64,13 @@ jobs:
     runs-on: ${{ needs.start-test-runner.outputs.label }}
     environment: more-secrets
     steps:
+      - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
       - name: Search for valid connector name format
         id: regex
         uses: AsasInnab/regex-action@v1
@@ -74,13 +81,6 @@ jobs:
       - name: Validate input workflow format
         if: steps.regex.outputs.first_match != github.event.inputs.connector
         run: echo "The connector provided has an invalid format!" && exit 1
-      - name: Link comment to workflow run
-        if: github.event.inputs.comment-id
-        uses: peter-evans/create-or-update-comment@v1
-        with:
-          comment-id: ${{ github.event.inputs.comment-id }}
-          body: |
-            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
       - name: Checkout Airbyte
         uses: actions/checkout@v2
         with:
@@ -113,7 +113,7 @@ jobs:
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
 
-      - name: test ${{ github.event.inputs.connector }}
+      - name: Test ${{ github.event.inputs.connector }}
         run: |
           ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
         id: test

LICENSE_SHORT

@@ -1 +1 @@
-Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+Copyright (c) 2022 Airbyte, Inc., all rights reserved.

airbyte-analytics/src/main/java/io/airbyte/analytics/Deployment.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-analytics/src/main/java/io/airbyte/analytics/LoggingTrackingClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-analytics/src/main/java/io/airbyte/analytics/SegmentTrackingClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingClientSingleton.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingIdentity.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-analytics/src/test/java/io/airbyte/analytics/TrackingClientSingletonTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.analytics;

airbyte-api/build.gradle

@@ -17,9 +17,9 @@ task generateApiServer(type: GenerateTask) {
     inputSpec = specFile
     outputDir = serverOutputDir
 
-    apiPackage = "io.airbyte.api"
-    invokerPackage = "io.airbyte.api.invoker"
-    modelPackage = "io.airbyte.api.model"
+    apiPackage = "io.airbyte.api.generated"
+    invokerPackage = "io.airbyte.api.invoker.generated"
+    modelPackage = "io.airbyte.api.model.generated"
 
     importMappings = [
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
@@ -36,7 +36,15 @@ task generateApiServer(type: GenerateTask) {
     configOptions = [
             dateLibrary  : "java8",
             generatePom  : "false",
-            interfaceOnly: "true"
+            interfaceOnly: "true",
+            /*
+            JAX-RS generator does not respect nullable properties defined in the OpenApi Spec.
+            It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json.
+            The below Jackson annotation is made to only keep non null values in serialized json.
+            We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above.
+            Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details.
+            */
+            additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)"
     ]
 }
 compileJava.dependsOn tasks.generateApiServer
@@ -51,9 +59,9 @@ task generateApiClient(type: GenerateTask) {
     inputSpec = specFile
     outputDir = clientOutputDir
 
-    apiPackage = "io.airbyte.api.client"
-    invokerPackage = "io.airbyte.api.client.invoker"
-    modelPackage = "io.airbyte.api.client.model"
+    apiPackage = "io.airbyte.api.client.generated"
+    invokerPackage = "io.airbyte.api.client.invoker.generated"
+    modelPackage = "io.airbyte.api.client.model.generated"
 
     importMappings = [
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
@@ -84,9 +92,9 @@ task generateApiDocs(type: GenerateTask) {
     inputSpec = specFile
     outputDir = docsOutputDir
 
-    apiPackage = "io.airbyte.api.client"
-    invokerPackage = "io.airbyte.api.client.invoker"
-    modelPackage = "io.airbyte.api.client.model"
+    apiPackage = "io.airbyte.api.client.generated"
+    invokerPackage = "io.airbyte.api.client.invoker.generated"
+    modelPackage = "io.airbyte.api.client.model.generated"
 
     importMappings = [
             'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',

airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java

@@ -1,10 +1,22 @@
 /*
- * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
 package io.airbyte.api.client;
 
-import io.airbyte.api.client.invoker.ApiClient;
+import io.airbyte.api.client.generated.ConnectionApi;
+import io.airbyte.api.client.generated.DbMigrationApi;
+import io.airbyte.api.client.generated.DestinationApi;
+import io.airbyte.api.client.generated.DestinationDefinitionApi;
+import io.airbyte.api.client.generated.DestinationDefinitionSpecificationApi;
+import io.airbyte.api.client.generated.HealthApi;
+import io.airbyte.api.client.generated.JobsApi;
+import io.airbyte.api.client.generated.OperationApi;
+import io.airbyte.api.client.generated.SourceApi;
+import io.airbyte.api.client.generated.SourceDefinitionApi;
+import io.airbyte.api.client.generated.SourceDefinitionSpecificationApi;
+import io.airbyte.api.client.generated.WorkspaceApi;
+import io.airbyte.api.client.invoker.generated.ApiClient;
 
 /**
  * This class is meant to consolidate all our API endpoints into a fluent-ish client. Currently, all