Merge branch 'master' into leti/experiment-add-source-in-signup

* master: (38 commits)
  New Source: Gridly (#18342)
  🎉 New Source: Alpha Vantage (#18320)
  ci_integration_test.sh: cut GITHUB_STEP_SUMMARY (#18895)
  🎉 New Source: Datadog [python cdk] (#18150)
  Hide Reject all button in consent dialog (#18596)
  feat: add doc url to track event (#18690)
  fix: install java in oss catalog deploy action (#18887)
  [CI] Speed up check_images_exist (#18873)
  Extract open API (#18879)
  Remove unused interfaces (#18880)
  add action for deploying oss connector catalog to GCS (#18633)
  feat: generate full connector catalog json (#18562)
  Add unsupported_protocol_version column to Connection (#18876)
  Extract OAuth API (#18818)
  update images to have non-transparent background (#18874)
  DiscoverSchema endpoints calculates diff and breaking change (#18571)
  Validate protocol version on connector update (#18639)
  Bmoric/extract notification api (#18812)
  Show version and changelog status for affected connectors (#18845)
  Bmoric/extract logs api (#18621)
  ...

Author: letiescanciano

.github/comment_templates/connector_dependency_template.md

@@ -1,25 +1,54 @@
 <!--- this comment is for `report-connectors-dependency.yml` identification, do not remove -->
 
-NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to run corresponding integration tests:
+NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do the following as needed:
+- Run integration tests
+- Bump connector version
+- Add changelog
+- Publish the new version
 
-<details>
+<details {source_open}>
 <summary>
-Sources ({num_sources})
+
+### {source_status_summary} Sources ({num_sources})
 
 </summary>
 
-{sources}
+| Connector | Version | Changelog | Publish |
+| --- | :---: | :---: | :---: |
+{source_rows}
 
 </details>
 
-<details>
+<details {destination_open}>
 <summary>
-Destinations ({num_destinations})
+
+### {destination_status_summary} Destinations ({num_destinations})
 
 </summary>
 
-{destinations}
+| Connector | Version | Changelog | Publish |
+| --- | :---: | :---: | :---: |
+{destination_rows}
 
 </details>
 
 {others}
+
+<details>
+
+<summary>
+
+### Notes
+
+</summary>
+
+| Category | Status | Actionable Item |
+| --- | :---: | --- |
+| Version | ❌<br/>mismatch | The version of the connector is different from its normal variant. Please bump the version of the connector. |
+| | ⚠<br/>doc not found | The connector does not seem to have a documentation file. This can be normal (e.g. basic connector like `source-jdbc` is not published or documented). Please double-check to make sure that it is not a bug. |
+| Changelog | ⚠<br/>doc not found | The connector does not seem to have a documentation file. This can be normal (e.g. basic connector like `source-jdbc` is not published or documented). Please double-check to make sure that it is not a bug. |
+| | ❌<br/>changelog missing | There is no chnagelog for the current version of the connector. If you are the author of the current version, please add a changelog. |
+| Publish | ⚠<br/>not in seed | The connector is not in the seed file (e.g. `source_definitions.yaml`), so its publication status cannot be checked. This can be normal (e.g. some connectors are cloud-specific, and only listed in the cloud seed file). Please double-check to make sure that it is not a bug. |
+| | ❌<br/>diff seed version | The connector exists in the seed file, but the latest version is not listed there. This usually means that the latest version is not published. Please use the `/publish` command to publish the latest version. |
+
+</details>

.github/workflows/create-release.yml

@@ -32,13 +32,15 @@ jobs:
           # The sed removes carriage returns so that the body is easier to parse later, and
           #   escapes backticks so that they are not executed as commands.
           PR_BODY=$(printf '%s' "$PR" | jq '.[0].body' | sed 's/\\r//g' | sed 's/`/\\`/g')
-          echo pr_body=${PR_BODY} >> $GITHUB_OUTPUT
+          echo "pr_body<<EOF" >> $GITHUB_ENV
+          echo "$PR_BODY" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
       - name: Extract Changelog
         id: extract_changelog
         shell: bash
         run: |-
           set -x
-          PR_BODY=${{ steps.fetch_pr_body.outputs.pr_body}}
+          PR_BODY=${{ env.pr_body}}
           if [[ $PR_BODY = "null" ]]; then
             echo "No PR body exists for this commit, so a release cannot be generated."
             exit 1

.github/workflows/deploy-oss-catalog.yml

@@ -0,0 +1,37 @@
+name: Deploy OSS Connector Catalog to GCS
+
+on:
+  push:
+    branches:
+      - master
+    paths: 
+      - airbyte-config/init/src/main/resources/seed
+
+  workflow_dispatch:
+
+jobs:
+  deploy-catalog:
+    name: "Deploy Catalog"
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    concurrency: deploy-oss-connector-catalog
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Setup Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.PROD_SPEC_CACHE_SA_KEY }}
+          export_default_credentials: true
+      - name: Install Java
+        uses: actions/setup-java@v1
+        with:
+          java-version: "17"
+      - name: Generate catalog
+        run: SUB_BUILD=PLATFORM ./gradlew :airbyte-config:init:processResources
+      - name: Upload catalog to GCS
+        shell: bash
+        run: |
+          gcs_bucket_name="prod-airbyte-cloud-connector-metadata-service"
+          catalog_path="airbyte-config/init/src/main/resources/seed/oss_catalog.json"
+          gsutil -h "Cache-Control:public, max-age=10" cp "$catalog_path" "gs://$gcs_bucket_name/oss_catalog.json"

.github/workflows/publish-helm-charts.yml

@@ -109,7 +109,10 @@ jobs:
         id: changelog
         run: |
           cd ./airbyte/
-          echo "::set-output name=changelog::$(PAGER=cat git log $(git describe --tags --match "*-helm" $(git rev-list --tags --max-count=1))..HEAD --oneline --decorate=no)"
+          changelog=$(PAGER=cat git log $(git describe --tags --match "*-helm" $(git rev-list --tags --max-count=1))..HEAD --oneline --decorate=no)
+          echo "changelog<<EOF" >> $GITHUB_ENV
+          echo "$changelog" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v4
@@ -122,7 +125,7 @@ jobs:
             ## What
             Bump version reference in all Chart.yaml files to ${{ needs.generate-semantic-version.outputs.next-version }}
             CHANGELOG:
-            ${{ steps.changelog.outputs.changelog }}
+            ${{ env.changelog }}
           commit-message: Bump helm chart version reference to ${{ needs.generate-semantic-version.outputs.next-version }}
           delete-branch: true
 

.github/workflows/release-airbyte-os.yml

@@ -135,8 +135,9 @@ jobs:
         run: |
           chmod +x tools/bin/pr_body.sh
           body=$(./tools/bin/pr_body.sh)
-          body="${body//$'\n'/'%0A'}"
-          echo PR_BODY=$body >> $GITHUB_OUTPUT
+          echo "PR_BODY<<EOF" >> $GITHUB_ENV
+          echo "$body" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
       - name: Create Pull Request
         id: cpr
         uses: peter-evans/create-pull-request@v3
@@ -146,7 +147,7 @@ jobs:
           branch-suffix: random
           delete-branch: true
           title: Bump Airbyte version from ${{ steps.bump_version.outputs.PREV_VERSION }} to ${{ steps.bump_version.outputs.NEW_VERSION }}
-          body: ${{ steps.pr_body.outputs.PR_BODY }}
+          body: ${{ env.PR_BODY }}
           commit-message: Bump Airbyte version from ${{ steps.bump_version.outputs.PREV_VERSION }} to ${{ steps.bump_version.outputs.NEW_VERSION }}
       - name: PR Details
         run: |

.gitignore

@@ -71,6 +71,9 @@ docs/SUMMARY.md
 # Files generated by unit tests
 **/specs_secrets_mask.yaml
 
+# Files generated for uploading to GCS
+airbyte-config/**/resources/seed/oss_catalog.json
+
 # Helm charts .tgz dependencies
 charts/**/charts
 

airbyte-api/src/main/openapi/config.yaml

@@ -49,8 +49,10 @@ tags:
     description: Destination related resources.
   - name: connection
     description: Connection between sources and destinations.
-  - name: oauth
-    description: OAuth related resources to delegate access from user.
+  - name: destination_oauth
+    description: Source OAuth related resources to delegate access from user.
+  - name: source_oauth
+    description: Source OAuth related resources to delegate access from user.
   - name: db_migration
     description: Database migration related resources.
   - name: web_backend
@@ -1762,7 +1764,7 @@ paths:
   /v1/source_oauths/oauth_params/create:
     post:
       tags:
-        - oauth
+        - source_oauth
       summary: >
         Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected
         into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
@@ -1785,7 +1787,7 @@ paths:
   /v1/source_oauths/get_consent_url:
     post:
       tags:
-        - oauth
+        - source_oauth
       summary: Given a source connector definition ID, return the URL to the consent screen where to redirect the user to.
       operationId: getSourceOAuthConsent
       requestBody:
@@ -1808,7 +1810,7 @@ paths:
   /v1/source_oauths/complete_oauth:
     post:
       tags:
-        - oauth
+        - source_oauth
       summary: Given a source def ID generate an access/refresh token etc.
       operationId: completeSourceOAuth
       requestBody:
@@ -1831,7 +1833,7 @@ paths:
   /v1/destination_oauths/get_consent_url:
     post:
       tags:
-        - oauth
+        - destination_oauth
       summary: Given a destination connector definition ID, return the URL to the consent screen where to redirect the user to.
       operationId: getDestinationOAuthConsent
       requestBody:
@@ -1854,7 +1856,7 @@ paths:
   /v1/destination_oauths/complete_oauth:
     post:
       tags:
-        - oauth
+        - destination_oauth
       summary: Given a destination def ID generate an access/refresh token etc.
       operationId: completeDestinationOAuth
       requestBody:
@@ -1877,7 +1879,7 @@ paths:
   /v1/destination_oauths/oauth_params/create:
     post:
       tags:
-        - oauth
+        - destination_oauth
       summary: >
         Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected
         into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with
@@ -2822,6 +2824,9 @@ components:
       properties:
         sourceId:
           $ref: "#/components/schemas/SourceId"
+        connectionId:
+          type: string
+          format: uuid
         disable_cache:
           type: boolean
     SourceUpdate:
@@ -2883,6 +2888,10 @@ components:
         catalogId:
           type: string
           format: uuid
+        catalogDiff:
+          $ref: "#/components/schemas/CatalogDiff"
+        breakingChange:
+          type: boolean
     SourceSearch:
       type: object
       properties:
@@ -3375,6 +3384,8 @@ components:
           type: boolean
         nonBreakingChangesPreference:
           $ref: "#/components/schemas/NonBreakingChangesPreference"
+        breakingChange:
+          type: boolean
     WebBackendConnectionUpdate:
       type: object
       description: Used to apply a patch-style update to a connection, which means that null properties remain unchanged

airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

@@ -141,7 +141,7 @@ void testBootloaderAppBlankDb() throws Exception {
       val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
       // this line should change with every new migration
       // to show that you meant to make a new migration to the prod database
-      assertEquals("0.40.12.001", configsMigrator.getLatestMigration().getVersion().getVersion());
+      assertEquals("0.40.18.001", configsMigrator.getLatestMigration().getVersion().getVersion());
 
       val jobsPersistence = new DefaultJobPersistence(jobDatabase);
       assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get());

airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersionRange.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.commons.version;
+
+public record AirbyteProtocolVersionRange(Version min, Version max) {
+
+  public boolean isSupported(final Version v) {
+    final Integer major = getMajor(v);
+    return getMajor(min) <= major && major <= getMajor(max);
+  }
+
+  private Integer getMajor(final Version v) {
+    return Integer.valueOf(v.getMajorVersion());
+  }
+
+}

airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteProtocolVersionRangeTest.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.commons.version;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+class AirbyteProtocolVersionRangeTest {
+
+  @Test
+  void checkRanges() {
+    final AirbyteProtocolVersionRange range = new AirbyteProtocolVersionRange(new Version("1.2.3"), new Version("4.3.2"));
+    assertTrue(range.isSupported(new Version("2.0.0")));
+    assertTrue(range.isSupported(new Version("1.2.3")));
+    assertTrue(range.isSupported(new Version("4.3.2")));
+
+    // We should only be requiring major to be within range
+    assertTrue(range.isSupported(new Version("1.0.0")));
+    assertTrue(range.isSupported(new Version("4.4.0")));
+
+    assertFalse(range.isSupported(new Version("0.2.3")));
+    assertFalse(range.isSupported(new Version("5.0.0")));
+  }
+
+  @Test
+  void checkRangeWithOnlyOneMajor() {
+    final AirbyteProtocolVersionRange range = new AirbyteProtocolVersionRange(new Version("2.0.0"), new Version("2.1.2"));
+
+    assertTrue(range.isSupported(new Version("2.0.0")));
+    assertTrue(range.isSupported(new Version("2.5.0")));
+
+    assertFalse(range.isSupported(new Version("1.0.0")));
+    assertFalse(range.isSupported(new Version("3.0.0")));
+  }
+
+}

airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigPersistence.java

@@ -5,7 +5,6 @@
 package io.airbyte.config.persistence;
 
 import io.airbyte.config.AirbyteConfig;
-import io.airbyte.config.ConfigWithMetadata;
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.IOException;
 import java.util.List;
@@ -21,11 +20,6 @@ public interface ConfigPersistence {
 
   <T> List<T> listConfigs(AirbyteConfig configType, Class<T> clazz) throws JsonValidationException, IOException;
 
-  <T> ConfigWithMetadata<T> getConfigWithMetadata(AirbyteConfig configType, String configId, Class<T> clazz)
-      throws ConfigNotFoundException, JsonValidationException, IOException;
-
-  <T> List<ConfigWithMetadata<T>> listConfigsWithMetadata(AirbyteConfig configType, Class<T> clazz) throws JsonValidationException, IOException;
-
   <T> void writeConfig(AirbyteConfig configType, String configId, T config) throws JsonValidationException, IOException;
 
   <T> void writeConfigs(AirbyteConfig configType, Map<String, T> configs) throws IOException, JsonValidationException;