Merge branch 'master' into osheg/10148-bigquery-docs

Author: alexandr-shegeda

.github/workflows/platform-project-automation.yml

@@ -8,7 +8,8 @@ on:
 env:
   GH_PROJECT_TOKEN: ${{ secrets.PARKER_PAT_FOR_PLATFORM_PROJECT_AUTOMATION }}
   ORG: airbytehq
-  PROJECT_ID: 6 # https://github.com/orgs/airbytehq/projects/6/views/8
+  CLOUD_PROJECT_ID: 6 # https://github.com/orgs/airbytehq/projects/6/views/8
+  OSS_PROJECT_ID: 11 # https://github.com/orgs/airbytehq/projects/11
   FIELD_STATUS: Status
   STATUS_TODO: Todo
   FIELD_DATE_ADDED: Date Added
@@ -21,14 +22,44 @@ jobs:
       - name: Set current date env var
         id: set_date
         run: echo ::set-output name=CURRENT_DATE::$(date +'%Y-%m-%dT%H:%M:%S%z')
-      
-      - name: Add issue to project if labelled with area/platform
+
+      - name: Add issue to the cloud project if labelled with area/platform and not area/platform-oss
+        uses: leonsteinhaeuser/[email protected]
+        if: contains(github.event.issue.labels.*.name, 'area/platform') && !contains(github.event.issue.labels.*.name, 'area/platform-oss')
+        with:
+          gh_token: ${{ env.GH_PROJECT_TOKEN }}
+          organization: ${{ env.ORG }}
+          project_id: ${{ env.CLOUD_PROJECT_ID }}
+          resource_node_id: ${{ github.event.issue.node_id }}
+          operation_mode: custom_field
+          custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'
+      - name: Add issue to the cloud oss if labelled with area/platform and not area/platform-cloud
+        uses: leonsteinhaeuser/[email protected]
+        if: contains(github.event.issue.labels.*.name, 'area/platform') && !contains(github.event.issue.labels.*.name, 'area/platform-cloud')
+        with:
+          gh_token: ${{ env.GH_PROJECT_TOKEN }}
+          organization: ${{ env.ORG }}
+          project_id: ${{ env.OSS_PROJECT_ID }}
+          resource_node_id: ${{ github.event.issue.node_id }}
+          operation_mode: custom_field
+          custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'
+      - name: Add issue to project if labelled with area/platform-cloud
+        uses: leonsteinhaeuser/[email protected]
+        if: contains(github.event.issue.labels.*.name, 'area/platform-cloud')
+        with:
+          gh_token: ${{ env.GH_PROJECT_TOKEN }}
+          organization: ${{ env.ORG }}
+          project_id: ${{ env.CLOUD_PROJECT_ID }}
+          resource_node_id: ${{ github.event.issue.node_id }}
+          operation_mode: custom_field
+          custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'
+      - name: Add issue to project if labelled with area/platform-oss
         uses: leonsteinhaeuser/[email protected]
-        if: contains(github.event.issue.labels.*.name, 'area/platform')
+        if: contains(github.event.issue.labels.*.name, 'area/platform-oss')
         with:
           gh_token: ${{ env.GH_PROJECT_TOKEN }}
           organization: ${{ env.ORG }}
-          project_id: ${{ env.PROJECT_ID }}
+          project_id: ${{ env.OSS_PROJECT_ID }}
           resource_node_id: ${{ github.event.issue.node_id }}
           operation_mode: custom_field
           custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'

.github/workflows/release-airbyte-os.yml

@@ -63,15 +63,10 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: "16.13.0"
-
       # necessary to install pip
       - uses: actions/setup-python@v2
         with:
           python-version: "3.7"
-      - name: Save Old Version
-        id: old_version
-        run: |
-          echo ::set-output name=OLD_VERSION::$(grep -w VERSION .env | cut -d"=" -f2)
       - name: Release Airbyte
         id: release_airbyte
         env:
@@ -81,14 +76,63 @@ jobs:
           CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
         run: |
           ./tools/bin/release_version.sh
-      - name: Save New Version
-        id: new_version
-        run: |
-          echo ::set-output name=NEW_VERSION::$(grep -w VERSION .env | cut -d"=" -f2)
+
+  # We are releasing octavia from a separate job because:
+  # - The self hosted runner used in releaseAirbyte does not have the docker buildx command to build multi-arch images
+  # - Plaform build requires Python 3.7, Octavia Build requires 3.8
+  releaseOctavia:
+    runs-on: ubuntu-latest
+    environment: more-secrets
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-java@v1
+        with:
+          java-version: "17"
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: "16.13.0"
+      # octavia-cli build requires Python 3.8.
+      # We use 3.8 in this project because 3.7 is not supported on Apple M1.
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Release Octavia
+        id: release_octavia
+        env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
+        run: ./tools/bin/release_version_octavia.sh
+
+  createPullRequest:
+    needs:
+      - releaseAirbyte
+      - releaseOctavia
+    runs-on: ubuntu-latest
+    environment: more-secrets
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      # necessary to install pip
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.7"
+      - name: Bump version
+        id: bump_version
+        env:
+          PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
+        run: ./tools/bin/bump_version.sh
       - name: Get PR Body
         id: pr_body
         env:
-          PREV_VERSION: ${{ steps.old_version.outputs.OLD_VERSION }}
+          PREV_VERSION: ${{ steps.bump_version.outputs.PREV_VERSION }}
+          NEW_VERSION: ${{ steps.bump_version.outputs.NEW_VERSION }}
+          GIT_REVISION: ${{ steps.bump_version.outputs.GIT_REVISION }}
         run: |
           chmod +x tools/bin/pr_body.sh
           body=$(./tools/bin/pr_body.sh)
@@ -102,9 +146,9 @@ jobs:
           branch: bump-version
           branch-suffix: random
           delete-branch: true
-          title: Bump Airbyte version from ${{ steps.old_version.outputs.OLD_VERSION }} to ${{ steps.new_version.outputs.NEW_VERSION }}
+          title: Bump Airbyte version from ${{ steps.bump_version.outputs.PREV_VERSION }} to ${{ steps.bump_version.outputs.NEW_VERSION }}
           body: ${{ steps.pr_body.outputs.PR_BODY }}
-          commit-message: Bump Airbyte version from ${{ steps.old_version.outputs.OLD_VERSION }} to ${{ steps.new_version.outputs.NEW_VERSION }}
+          commit-message: Bump Airbyte version from ${{ steps.bump_version.outputs.PREV_VERSION }} to ${{ steps.bump_version.outputs.NEW_VERSION }}
       - name: PR Details
         run: |
           echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"

.github/workflows/slash-commands.yml

@@ -18,7 +18,7 @@ jobs:
         id: scd
         uses: peter-evans/slash-command-dispatch@v2
         with:
-          token: ${{ secrets.SUPERTOPHER_PAT }}
+          token: ${{ secrets.DAVINCHIA_PAT }}
           commands: |
             test
             test-performance

airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

@@ -86,9 +86,12 @@ public BootloaderApp() {
         final Database configDatabase =
             new ConfigsDatabaseInstance(configs.getConfigDatabaseUser(), configs.getConfigDatabasePassword(), configs.getConfigDatabaseUrl())
                 .getAndInitialize();
-        final JsonSecretsProcessor jsonSecretsProcessor = new JsonSecretsProcessor();
+        final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder()
+            .maskSecrets(!featureFlags.exposeSecretsInExport())
+            .copySecrets(true)
+            .build();
         final ConfigPersistence configPersistence =
-            DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor, featureFlags);
+            DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor);
         configPersistence.loadData(YamlSeedConfigPersistence.getDefault());
         LOGGER.info("Loaded seed data..");
       } catch (final IOException e) {
@@ -115,8 +118,11 @@ public void load() throws Exception {
       runFlywayMigration(configs, configDatabase, jobDatabase);
       LOGGER.info("Ran Flyway migrations...");
 
-      final JsonSecretsProcessor jsonSecretsProcessor = new JsonSecretsProcessor();
-      final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor, featureFlags);
+      final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder()
+          .maskSecrets(!featureFlags.exposeSecretsInExport())
+          .copySecrets(false)
+          .build();
+      final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor);
       final ConfigRepository configRepository =
           new ConfigRepository(configPersistence, configDatabase);
 

airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

@@ -72,7 +72,7 @@ void testBootloaderAppBlankDb() throws Exception {
         container.getPassword(),
         container.getJdbcUrl()).getInitialized();
     val jobsMigrator = new JobsDatabaseMigrator(jobDatabase, this.getClass().getName());
-    assertEquals("0.35.40.001", jobsMigrator.getLatestMigration().getVersion().getVersion());
+    assertEquals("0.35.62.001", jobsMigrator.getLatestMigration().getVersion().getVersion());
 
     val configDatabase = new ConfigsDatabaseInstance(
         mockedConfigs.getConfigDatabaseUser(),

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -15,7 +15,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 0.1.3
+  dockerImageTag: 0.1.4
   documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api
@@ -252,7 +252,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.2.23
+  dockerImageTag: 0.2.25
   documentationUrl: https://docs.airbyte.io/integrations/sources/github
   icon: github.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -2380,7 +2380,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.2.23"
+- dockerImage: "airbyte/source-github:0.2.25"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:

airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java

@@ -22,7 +22,6 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Sets;
 import io.airbyte.commons.enums.Enums;
-import io.airbyte.commons.features.FeatureFlags;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.util.MoreIterators;
 import io.airbyte.commons.version.AirbyteVersion;
@@ -81,7 +80,6 @@ public class DatabaseConfigPersistence implements ConfigPersistence {
 
   private final ExceptionWrappingDatabase database;
   private final JsonSecretsProcessor jsonSecretsProcessor;
-  private final FeatureFlags featureFlags;
   private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseConfigPersistence.class);
 
   /**
@@ -90,20 +88,16 @@ public class DatabaseConfigPersistence implements ConfigPersistence {
    *
    * @param database - database where configs are stored
    * @param jsonSecretsProcessor - for filtering secrets in export
-   * @param featureFlags - feature flags that govern secret export behavior
    * @return database config persistence wrapped in validation decorators
    */
   public static ConfigPersistence createWithValidation(final Database database,
-                                                       final JsonSecretsProcessor jsonSecretsProcessor,
-                                                       final FeatureFlags featureFlags) {
-    return new ClassEnforcingConfigPersistence(
-        new ValidatingConfigPersistence(new DatabaseConfigPersistence(database, jsonSecretsProcessor, featureFlags)));
+                                                       final JsonSecretsProcessor jsonSecretsProcessor) {
+    return new ValidatingConfigPersistence(new DatabaseConfigPersistence(database, jsonSecretsProcessor));
   }
 
-  public DatabaseConfigPersistence(final Database database, final JsonSecretsProcessor jsonSecretsProcessor, final FeatureFlags featureFlags) {
+  public DatabaseConfigPersistence(final Database database, final JsonSecretsProcessor jsonSecretsProcessor) {
     this.database = new ExceptionWrappingDatabase(database);
     this.jsonSecretsProcessor = jsonSecretsProcessor;
-    this.featureFlags = featureFlags;
   }
 
   @Override
@@ -1431,18 +1425,15 @@ public Map<String, Stream<JsonNode>> dumpConfigs() throws IOException {
           sourceConnectionWithMetadata
               .stream()
               .map(configWithMetadata -> {
-                if (featureFlags.exposeSecretsInExport()) {
-                  return Jsons.jsonNode(configWithMetadata.getConfig());
-                }
-
                 try {
                   final UUID sourceDefinitionId = configWithMetadata.getConfig().getSourceDefinitionId();
                   final StandardSourceDefinition standardSourceDefinition = getConfig(
                       ConfigSchema.STANDARD_SOURCE_DEFINITION,
                       sourceDefinitionId.toString(),
                       StandardSourceDefinition.class);
                   final JsonNode connectionSpecs = standardSourceDefinition.getSpec().getConnectionSpecification();
-                  final JsonNode sanitizedConfig = jsonSecretsProcessor.maskSecrets(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpecs);
+                  final JsonNode sanitizedConfig =
+                      jsonSecretsProcessor.prepareSecretsForOutput(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpecs);
                   return sanitizedConfig;
                 } catch (final ConfigNotFoundException | JsonValidationException | IOException e) {
                   throw new RuntimeException(e);
@@ -1454,18 +1445,15 @@ public Map<String, Stream<JsonNode>> dumpConfigs() throws IOException {
       final Stream<JsonNode> jsonNodeStream = destinationConnectionWithMetadata
           .stream()
           .map(configWithMetadata -> {
-            if (featureFlags.exposeSecretsInExport()) {
-              return Jsons.jsonNode(configWithMetadata.getConfig());
-            }
-
             try {
               final UUID destinationDefinition = configWithMetadata.getConfig().getDestinationDefinitionId();
               final StandardDestinationDefinition standardDestinationDefinition = getConfig(
                   ConfigSchema.STANDARD_DESTINATION_DEFINITION,
                   destinationDefinition.toString(),
                   StandardDestinationDefinition.class);
               final JsonNode connectionSpec = standardDestinationDefinition.getSpec().getConnectionSpecification();
-              final JsonNode sanitizedConfig = jsonSecretsProcessor.maskSecrets(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpec);
+              final JsonNode sanitizedConfig =
+                  jsonSecretsProcessor.prepareSecretsForOutput(Jsons.jsonNode(configWithMetadata.getConfig()), connectionSpec);
               return sanitizedConfig;
             } catch (final ConfigNotFoundException | JsonValidationException | IOException e) {
               throw new RuntimeException(e);