Add normalization to destination definition and actor definition table

airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml

@@ -58,3 +58,10 @@ properties:
   protocolVersion:
     type: string
     description: the Airbyte Protocol version supported by the connector
+  normalizationRepository:
+    type: string
+  normalizationTag:
+    type: string
+  supportsDbt:
+    type: boolean
+    default: false

airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java

@@ -130,6 +130,9 @@ static void writeStandardDestinationDefinition(final List<StandardDestinationDef
                 standardDestinationDefinition.getResourceRequirements() == null ? null
                     : JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getResourceRequirements())))
             .set(Tables.ACTOR_DEFINITION.UPDATED_AT, timestamp)
+            .set(Tables.ACTOR_DEFINITION.NORMALIZATION_REPOSITORY, standardDestinationDefinition.getNormalizationRepository())
+            .set(Tables.ACTOR_DEFINITION.NORMALIZATION_TAG, standardDestinationDefinition.getNormalizationTag())
+            .set(Tables.ACTOR_DEFINITION.SUPPORTS_DBT, standardDestinationDefinition.getSupportsDbt())
             .where(Tables.ACTOR_DEFINITION.ID.eq(standardDestinationDefinition.getDestinationDefinitionId()))
             .execute();
 
@@ -159,6 +162,9 @@ static void writeStandardDestinationDefinition(final List<StandardDestinationDef
                     : JSONB.valueOf(Jsons.serialize(standardDestinationDefinition.getResourceRequirements())))
             .set(Tables.ACTOR_DEFINITION.CREATED_AT, timestamp)
             .set(Tables.ACTOR_DEFINITION.UPDATED_AT, timestamp)
+            .set(Tables.ACTOR_DEFINITION.NORMALIZATION_REPOSITORY, standardDestinationDefinition.getNormalizationRepository())
+            .set(Tables.ACTOR_DEFINITION.NORMALIZATION_TAG, standardDestinationDefinition.getNormalizationTag())
+            .set(Tables.ACTOR_DEFINITION.SUPPORTS_DBT, standardDestinationDefinition.getSupportsDbt())
             .execute();
       }
     });

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -30,6 +30,9 @@
   dockerImageTag: 1.2.5
   documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
   icon: bigquery.svg
+  normalizationRepository: airbyte/normalization-bigquery
+  normalizationTag: 0.2.23
+  supportsDbt: true
   resourceRequirements:
     jobSpecific:
       - jobType: sync
@@ -70,6 +73,9 @@
   dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse
   releaseStage: alpha
+  normalizationRepository: airbyte/normalization-clickhouse
+  normalizationTag: 0.2.23
+  supportsDbt: true
 - name: Cloudflare R2
   destinationDefinitionId: 0fb07be9-7c3b-4336-850d-5efc006152ee
   dockerRepository: airbyte/destination-r2
@@ -179,6 +185,9 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/mssql
   icon: mssql.svg
   releaseStage: alpha
+  normalizationRepository: airbyte/normalization-mssql
+  normalizationTag: 0.2.23
+  supportsDbt: true
 - name: MeiliSearch
   destinationDefinitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e
   dockerRepository: airbyte/destination-meilisearch
@@ -200,20 +209,29 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/mysql
   icon: mysql.svg
   releaseStage: alpha
+  normalizationRepository: airbyte/normalization-mysql
+  normalizationTag: 0.2.23
+  supportsDbt: true
 - name: Oracle
   destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72
   dockerRepository: airbyte/destination-oracle
   dockerImageTag: 0.1.19
   documentationUrl: https://docs.airbyte.com/integrations/destinations/oracle
   icon: oracle.svg
   releaseStage: alpha
+  normalizationRepository: airbyte/normalization-oracle
+  normalizationTag: 0.2.23
+  supportsDbt: true
 - name: Postgres
   destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503
   dockerRepository: airbyte/destination-postgres
   dockerImageTag: 0.3.26
   documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres
   icon: postgresql.svg
   releaseStage: alpha
+  normalizationRepository: airbyte/normalization-postgres
+  normalizationTag: 0.2.23
+  supportsDbt: true
 - name: Pulsar
   destinationDefinitionId: 2340cbba-358e-11ec-8d3d-0242ac130203
   dockerRepository: airbyte/destination-pulsar
@@ -241,6 +259,9 @@
   dockerImageTag: 0.3.50
   documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift
   icon: redshift.svg
+  normalizationRepository: airbyte/normalization-redshift
+  normalizationTag: 0.2.23
+  supportsDbt: true
   resourceRequirements:
     jobSpecific:
       - jobType: sync
@@ -280,6 +301,9 @@
   dockerImageTag: 0.4.38
   documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake
   icon: snowflake.svg
+  normalizationRepository: airbyte/normalization-snowflake
+  normalizationTag: 0.2.23
+  supportsDbt: true
   resourceRequirements:
     jobSpecific:
       - jobType: sync
@@ -329,3 +353,6 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/tidb
   icon: tidb.svg
   releaseStage: alpha
+  normalizationRepository: airbyte/normalization-tidb
+  normalizationTag: 0.2.23
+  supportsDbt: true

airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_15_001__AddActorDefinitionNormalizationAndDbtColumns.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.db.instance.configs.migrations;
+
+import org.flywaydb.core.api.migration.BaseJavaMigration;
+import org.flywaydb.core.api.migration.Context;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// TODO: update migration description in the class name
+public class V0_40_15_001__AddActorDefinitionNormalizationAndDbtColumns extends BaseJavaMigration {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(V0_40_15_001__AddActorDefinitionNormalizationAndDbtColumns.class);
+
+  @Override
+  public void migrate(final Context context) throws Exception {
+    LOGGER.info("Running migration: {}", this.getClass().getSimpleName());
+
+    // Warning: please do not use any jOOQ generated code to write a migration.
+    // As database schema changes, the generated jOOQ code can be deprecated. So
+    // old migration may not compile if there is any generated code.
+    final DSLContext ctx = DSL.using(context.getConnection());
+    addNormalizationRepositoryColumn(ctx);
+    addNormalizationTagColumn(ctx);
+    addSupportsDbtColumn(ctx);
+  }
+
+  private void addNormalizationRepositoryColumn(final DSLContext ctx) {
+    ctx.alterTable("actor_definition")
+        .addColumnIfNotExists(DSL.field(
+            "normalization_repository",
+            SQLDataType.VARCHAR(256).nullable(true)))
+        .execute();
+  }
+
+  private void addNormalizationTagColumn(final DSLContext ctx) {
+    ctx.alterTable("actor_definition")
+        .addColumnIfNotExists(DSL.field(
+            "normalization_tag",
+            SQLDataType.VARCHAR(256).nullable(true)))
+        .execute();
+  }
+
+  public static void addSupportsDbtColumn(final DSLContext ctx) {
+    ctx.alterTable("actor_definition")
+        .addColumnIfNotExists(DSL.field("supports_dbt",
+            SQLDataType.BOOLEAN.nullable(true)))
+        .execute();
+  }
+
+}

airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt

@@ -68,23 +68,26 @@ Referenced by:
 
 
                                Table "public.actor_definition"
-        Column         |           Type           | Collation | Nullable |      Default
------------------------+--------------------------+-----------+----------+-------------------
- id                    | uuid                     |           | not null |
- name                  | character varying(256)   |           | not null |
- docker_repository     | character varying(256)   |           | not null |
- docker_image_tag      | character varying(256)   |           | not null |
- documentation_url     | character varying(256)   |           |          |
- icon                  | character varying(256)   |           |          |
- actor_type            | actor_type               |           | not null |
- source_type           | source_type              |           |          |
- spec                  | jsonb                    |           | not null |
- created_at            | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
- updated_at            | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
- tombstone             | boolean                  |           | not null | false
- release_stage         | release_stage            |           |          |
- release_date          | date                     |           |          |
- resource_requirements | jsonb                    |           |          |
+        Column              |           Type           | Collation | Nullable |      Default
+----------------------------+--------------------------+-----------+----------+-------------------
+ id                         | uuid                     |           | not null |
+ name                       | character varying(256)   |           | not null |
+ docker_repository          | character varying(256)   |           | not null |
+ docker_image_tag           | character varying(256)   |           | not null |
+ documentation_url          | character varying(256)   |           |          |
+ icon                       | character varying(256)   |           |          |
+ actor_type                 | actor_type               |           | not null |
+ source_type                | source_type              |           |          |
+ spec                       | jsonb                    |           | not null |
+ created_at                 | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
+ updated_at                 | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
+ tombstone                  | boolean                  |           | not null | false
+ release_stage              | release_stage            |           |          |
+ release_date               | date                     |           |          |
+ resource_requirements      | jsonb                    |           |          |
+ normalization_repository   | character varying(256)   |           |          |
+ normalization_tag          | character varying(256)   |           |          |
+ supports_dbt               | boolean                  |           |          |
 Indexes:
     "actor_definition_pkey" PRIMARY KEY, btree (id)
 Referenced by: