Publish PR 9029: clickhouse normalization (#9072)

* add normalization-clickhouse docker build step

* bump normalization version

* small changes gradle

* fix settings gradle

* fix eof file

* correct clickhouse normalization

* Refactor jinja template for scd (#9278)

* merge chris code and regenerate sql files

Co-authored-by: James Zhao <[email protected]>
Co-authored-by: Edward Gao <[email protected]>
Co-authored-by: Christophe Duong <[email protected]>

Author: 4 people

airbyte-integrations/bases/base-normalization/.dockerignore

@@ -8,3 +8,4 @@
 !dbt-project-template-mssql
 !dbt-project-template-mysql
 !dbt-project-template-oracle
+!dbt-project-template-clickhouse

airbyte-integrations/bases/base-normalization/Dockerfile

@@ -28,5 +28,5 @@ WORKDIR /airbyte
 ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh"
 ENTRYPOINT ["/airbyte/entrypoint.sh"]
 
-LABEL io.airbyte.version=0.1.61
+LABEL io.airbyte.version=0.1.62
 LABEL io.airbyte.name=airbyte/normalization

airbyte-integrations/bases/base-normalization/build.gradle

@@ -69,10 +69,15 @@ task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) {
     configure buildAirbyteDocker('oracle')
     dependsOn assemble
 }
+task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) {
+    configure buildAirbyteDocker('clickhouse')
+    dependsOn assemble
+}
 
 airbyteDocker.dependsOn(airbyteDockerMSSql)
 airbyteDocker.dependsOn(airbyteDockerMySql)
 airbyteDocker.dependsOn(airbyteDockerOracle)
+airbyteDocker.dependsOn(airbyteDockerClickhouse)
 
 task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) {
     module = "pytest"
@@ -86,6 +91,7 @@ task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs
     dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker'
     dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker'
     dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker'
+    dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
 }
 
 integrationTest.dependsOn("customIntegrationTestPython")

airbyte-integrations/bases/base-normalization/docker-compose.yaml

@@ -10,3 +10,5 @@ services:
     image: airbyte/normalization-mysql:${VERSION}
   normalization-oracle:
     image: airbyte/normalization-oracle:${VERSION}
+  normalization-clickhouse:
+    image: airbyte/normalization-clickhouse:${VERSION}

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql

@@ -12,11 +12,12 @@
 
   as (
 
+-- depends_on: ref('dedup_cdc_excluded_stg')
 with
 
 input_data as (
     select *
-    from _airbyte_test_normalization.dedup_cdc_excluded_ab3
+    from _airbyte_test_normalization.dedup_cdc_excluded_stg
     -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded
 ),
 
@@ -45,15 +46,15 @@ scd_data as (
         _ab_cdc_updated_at,
         _ab_cdc_deleted_at,
       _airbyte_emitted_at as _airbyte_start_at,
+      case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row,
       anyOrNull(_airbyte_emitted_at) over (
         partition by id
         order by
             _airbyte_emitted_at is null asc,
             _airbyte_emitted_at desc,
             _airbyte_emitted_at desc, _ab_cdc_updated_at desc
-        ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING
+            ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING
       ) as _airbyte_end_at,
-      case when _airbyte_active_row_num = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row,
       _airbyte_ab_id,
       _airbyte_emitted_at,
       _airbyte_dedup_cdc_excluded_hashid
@@ -65,7 +66,7 @@ dedup_data as (
         -- additionally, we generate a unique key for the scd table
         row_number() over (
             partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, accurateCastOrNull(_ab_cdc_deleted_at, 'String'), accurateCastOrNull(_ab_cdc_updated_at, 'String')
-            order by _airbyte_ab_id
+            order by _airbyte_active_row desc, _airbyte_ab_id
         ) as _airbyte_row_num,
         assumeNotNull(hex(MD5(
 

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql

@@ -12,11 +12,12 @@
 
   as (
 
+-- depends_on: ref('dedup_exchange_rate_stg')
 with
 
 input_data as (
     select *
-    from _airbyte_test_normalization.dedup_exchange_rate_ab3
+    from _airbyte_test_normalization.dedup_exchange_rate_stg
     -- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate
 ),
 
@@ -54,15 +55,15 @@ scd_data as (
         NZD,
         USD,
       date as _airbyte_start_at,
+      case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row,
       anyOrNull(date) over (
         partition by id, currency, cast(NZD as String)
         order by
             date is null asc,
             date desc,
             _airbyte_emitted_at desc
-        ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING
+            ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING
       ) as _airbyte_end_at,
-      case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row,
       _airbyte_ab_id,
       _airbyte_emitted_at,
       _airbyte_dedup_exchange_rate_hashid
@@ -74,7 +75,7 @@ dedup_data as (
         -- additionally, we generate a unique key for the scd table
         row_number() over (
             partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at
-            order by _airbyte_ab_id
+            order by _airbyte_active_row desc, _airbyte_ab_id
         ) as _airbyte_row_num,
         assumeNotNull(hex(MD5(
 

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql

@@ -12,11 +12,12 @@
 
   as (
 
+-- depends_on: ref('renamed_dedup_cdc_excluded_stg')
 with
 
 input_data as (
     select *
-    from _airbyte_test_normalization.renamed_dedup_cdc_excluded_ab3
+    from _airbyte_test_normalization.renamed_dedup_cdc_excluded_stg
     -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded
 ),
 
@@ -41,15 +42,15 @@ scd_data as (
     ))) as _airbyte_unique_key,
         id,
       _airbyte_emitted_at as _airbyte_start_at,
+      case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row,
       anyOrNull(_airbyte_emitted_at) over (
         partition by id
         order by
             _airbyte_emitted_at is null asc,
             _airbyte_emitted_at desc,
             _airbyte_emitted_at desc
-        ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING
+            ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING
       ) as _airbyte_end_at,
-      case when _airbyte_active_row_num = 1 then 1 else 0 end as _airbyte_active_row,
       _airbyte_ab_id,
       _airbyte_emitted_at,
       _airbyte_renamed_dedup_cdc_excluded_hashid
@@ -61,7 +62,7 @@ dedup_data as (
         -- additionally, we generate a unique key for the scd table
         row_number() over (
             partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at
-            order by _airbyte_ab_id
+            order by _airbyte_active_row desc, _airbyte_ab_id
         ) as _airbyte_row_num,
         assumeNotNull(hex(MD5(
 

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql

@@ -13,6 +13,7 @@
   as (
 
 -- Final base SQL model
+-- depends_on: test_normalization.dedup_cdc_excluded_scd
 select
     _airbyte_unique_key,
     id,

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql

@@ -13,6 +13,7 @@
   as (
 
 -- Final base SQL model
+-- depends_on: test_normalization.dedup_exchange_rate_scd
 select
     _airbyte_unique_key,
     id,

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql

@@ -13,6 +13,7 @@
   as (
 
 -- Final base SQL model
+-- depends_on: test_normalization.renamed_dedup_cdc_excluded_scd
 select
     _airbyte_unique_key,
     id,