Skip to content

Commit b6926d4

Browse files
authored
🚨 Snowflake produces permanent tables 🚨 (#9063)
1 parent c5d4a97 commit b6926d4

File tree

33 files changed

+224
-60
lines changed

33 files changed

+224
-60
lines changed

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@
179179
- name: Snowflake
180180
destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
181181
dockerRepository: airbyte/destination-snowflake
182-
dockerImageTag: 0.3.24
182+
dockerImageTag: 0.4.0
183183
documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
184184
icon: snowflake.svg
185185
- name: MariaDB ColumnStore

airbyte-config/init/src/main/resources/seed/destination_specs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3767,7 +3767,7 @@
37673767
supported_destination_sync_modes:
37683768
- "overwrite"
37693769
- "append"
3770-
- dockerImage: "airbyte/destination-snowflake:0.3.24"
3770+
- dockerImage: "airbyte/destination-snowflake:0.4.0"
37713771
spec:
37723772
documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake"
37733773
connectionSpecification:

airbyte-integrations/bases/base-normalization/.dockerignore

+1
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@
99
!dbt-project-template-mysql
1010
!dbt-project-template-oracle
1111
!dbt-project-template-clickhouse
12+
!dbt-project-template-snowflake

airbyte-integrations/bases/base-normalization/.gitignore

+11
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,31 @@ integration_tests/normalization_test_output/**/*.yml
1919
# We keep a minimal/restricted subset of sql files for all destinations to avoid noise in diff
2020
# Simple Streams
2121
!integration_tests/normalization_test_output/**/dedup_exchange_rate*.sql
22+
!integration_tests/normalization_test_output/**/DEDUP_EXCHANGE_RATE*.sql
2223
!integration_tests/normalization_test_output/**/exchange_rate.sql
24+
!integration_tests/normalization_test_output/**/EXCHANGE_RATE.sql
2325
!integration_tests/normalization_test_output/**/test_simple_streams/first_output/airbyte_views/**/multiple_column_names_conflicts_stg.sql
2426
# Nested Streams
2527
# Parent table
2628
!integration_tests/normalization_test_output/**/nested_stream_with*_names_ab*.sql
2729
!integration_tests/normalization_test_output/**/nested_stream_with*_names_scd.sql
2830
!integration_tests/normalization_test_output/**/nested_stream_with*_names.sql
31+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES_AB*.sql
32+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES_SCD.sql
33+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_NAMES.sql
2934
# Nested table
3035
!integration_tests/normalization_test_output/**/nested_stream_with_*_partition_ab1.sql
3136
!integration_tests/normalization_test_output/**/nested_stream_with_*_data_ab1.sql
3237
!integration_tests/normalization_test_output/**/nested_stream_with*_partition_scd.sql
3338
!integration_tests/normalization_test_output/**/nested_stream_with*_data_scd.sql
3439
!integration_tests/normalization_test_output/**/nested_stream_with*_partition.sql
3540
!integration_tests/normalization_test_output/**/nested_stream_with*_data.sql
41+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH_*_PARTITION_AB1.sql
42+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH_*_DATA_AB1.sql
43+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_PARTITION_SCD.sql
44+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_DATA_SCD.sql
45+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_PARTITION.sql
46+
!integration_tests/normalization_test_output/**/NESTED_STREAM_WITH*_DATA.sql
3647

3748
# but we keep all sql files for Postgres
3849
!integration_tests/normalization_test_output/postgres/**/*.sql

airbyte-integrations/bases/base-normalization/build.gradle

+5
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,16 @@ task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) {
7373
configure buildAirbyteDocker('clickhouse')
7474
dependsOn assemble
7575
}
76+
task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) {
77+
configure buildAirbyteDocker('snowflake')
78+
dependsOn assemble
79+
}
7680

7781
airbyteDocker.dependsOn(airbyteDockerMSSql)
7882
airbyteDocker.dependsOn(airbyteDockerMySql)
7983
airbyteDocker.dependsOn(airbyteDockerOracle)
8084
airbyteDocker.dependsOn(airbyteDockerClickhouse)
85+
airbyteDocker.dependsOn(airbyteDockerSnowflake)
8186

8287
task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) {
8388
module = "pytest"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# This file is necessary to install dbt-utils with dbt deps
2+
# the content will be overwritten by the transform function
3+
4+
# Name your package! Package names should contain only lowercase characters
5+
# and underscores. A good package name should reflect your organization's
6+
# name or the intended use of these models
7+
name: "airbyte_utils"
8+
version: "1.0"
9+
config-version: 2
10+
11+
# This setting configures which "profile" dbt uses for this project. Profiles contain
12+
# database connection information, and should be configured in the ~/.dbt/profiles.yml file
13+
profile: "normalize"
14+
15+
# These configurations specify where dbt should look for different types of files.
16+
# The `source-paths` config, for example, states that source models can be found
17+
# in the "models/" directory. You probably won't need to change these!
18+
source-paths: ["models"]
19+
docs-paths: ["docs"]
20+
analysis-paths: ["analysis"]
21+
test-paths: ["tests"]
22+
data-paths: ["data"]
23+
macro-paths: ["macros"]
24+
25+
target-path: "../build" # directory which will store compiled SQL files
26+
log-path: "../logs" # directory which will store DBT logs
27+
modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
28+
29+
clean-targets: # directories to be removed by `dbt clean`
30+
- "build"
31+
- "dbt_modules"
32+
33+
quoting:
34+
database: true
35+
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
36+
# all schemas should be unquoted
37+
schema: false
38+
identifier: true
39+
40+
# You can define configurations for models in the `source-paths` directory here.
41+
# Using these configurations, you can enable or disable models, change how they
42+
# are materialized, and more!
43+
models:
44+
+transient: false
45+
airbyte_utils:
46+
+materialized: table
47+
generated:
48+
airbyte_ctes:
49+
+tags: airbyte_internal_cte
50+
+materialized: ephemeral
51+
airbyte_incremental:
52+
+tags: incremental_tables
53+
+materialized: incremental
54+
+on_schema_change: sync_all_columns
55+
airbyte_tables:
56+
+tags: normalized_tables
57+
+materialized: table
58+
airbyte_views:
59+
+tags: airbyte_internal_views
60+
+materialized: view
61+
62+
dispatch:
63+
- macro_namespace: dbt_utils
64+
search_order: ["airbyte_utils", "dbt_utils"]

airbyte-integrations/bases/base-normalization/docker-compose.build.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,10 @@ services:
3636
context: .
3737
labels:
3838
io.airbyte.git-revision: ${GIT_REVISION}
39+
normalization-snowflake:
40+
image: airbyte/normalization-snowflake:${VERSION}
41+
build:
42+
dockerfile: snowflake.Dockerfile
43+
context: .
44+
labels:
45+
io.airbyte.git-revision: ${GIT_REVISION}

airbyte-integrations/bases/base-normalization/docker-compose.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,5 @@ services:
1212
image: airbyte/normalization-oracle:${VERSION}
1313
normalization-clickhouse:
1414
image: airbyte/normalization-clickhouse:${VERSION}
15+
normalization-snowflake:
16+
image: airbyte/normalization-snowflake:${VERSION}

airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py

+4
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,8 @@ def get_normalization_image(destination_type: DestinationType) -> str:
374374
return "airbyte/normalization-oracle:dev"
375375
elif DestinationType.CLICKHOUSE.value == destination_type.value:
376376
return "airbyte/normalization-clickhouse:dev"
377+
elif DestinationType.SNOWFLAKE.value == destination_type.value:
378+
return "airbyte/normalization-snowflake:dev"
377379
else:
378380
return "airbyte/normalization:dev"
379381

@@ -445,6 +447,8 @@ def run_check_dbt_command(normalization_image: str, command: str, cwd: str, forc
445447
"Configuration paths exist in your dbt_project.yml", # When no cte / view are generated
446448
"Error loading config file: .dockercfg: $HOME is not defined", # ignore warning
447449
"depends on a node named 'disabled_test' which was not found", # Tests throwing warning because it is disabled
450+
"The requested image's platform (linux/amd64) does not match the detected host platform "
451+
+ "(linux/arm64/v8) and no specific platform was requested", # temporary patch until we publish images for arm64
448452
]:
449453
if except_clause in str_line:
450454
is_exception = True

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml

100755100644
+13-12
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
# Name your package! Package names should contain only lowercase characters
55
# and underscores. A good package name should reflect your organization's
66
# name or the intended use of these models
7-
name: 'airbyte_utils'
8-
version: '1.0'
7+
name: "airbyte_utils"
8+
version: "1.0"
99
config-version: 2
1010

1111
# This setting configures which "profile" dbt uses for this project. Profiles contain
1212
# database connection information, and should be configured in the ~/.dbt/profiles.yml file
13-
profile: 'normalize'
13+
profile: "normalize"
1414

1515
# These configurations specify where dbt should look for different types of files.
1616
# The `source-paths` config, for example, states that source models can be found
@@ -22,25 +22,26 @@ test-paths: ["tests"]
2222
data-paths: ["data"]
2323
macro-paths: ["macros"]
2424

25-
target-path: "../build" # directory which will store compiled SQL files
26-
log-path: "../logs" # directory which will store DBT logs
27-
modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
25+
target-path: "../build" # directory which will store compiled SQL files
26+
log-path: "../logs" # directory which will store DBT logs
27+
modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
2828

29-
clean-targets: # directories to be removed by `dbt clean`
30-
- "build"
31-
- "dbt_modules"
29+
clean-targets: # directories to be removed by `dbt clean`
30+
- "build"
31+
- "dbt_modules"
3232

3333
quoting:
3434
database: true
35-
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
36-
# all schemas should be unquoted
35+
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
36+
# all schemas should be unquoted
3737
schema: false
3838
identifier: true
3939

4040
# You can define configurations for models in the `source-paths` directory here.
4141
# Using these configurations, you can enable or disable models, change how they
4242
# are materialized, and more!
4343
models:
44+
+transient: false
4445
airbyte_utils:
4546
+materialized: table
4647
generated:
@@ -60,4 +61,4 @@ models:
6061

6162
dispatch:
6263
- macro_namespace: dbt_utils
63-
search_order: ['airbyte_utils', 'dbt_utils']
64+
search_order: ["airbyte_utils", "dbt_utils"]

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as
44
(select * from(
55

66
-- Final base SQL model

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as
44
(select * from(
55

66
with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1 as (

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as
44
(select * from(
55

66
with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1 as (

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as
44
(select * from(
55

66
with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1 as (

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as
44
(select * from(
55

66
-- depends_on: ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_STG')

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml

100755100644
+13-12
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
# Name your package! Package names should contain only lowercase characters
55
# and underscores. A good package name should reflect your organization's
66
# name or the intended use of these models
7-
name: 'airbyte_utils'
8-
version: '1.0'
7+
name: "airbyte_utils"
8+
version: "1.0"
99
config-version: 2
1010

1111
# This setting configures which "profile" dbt uses for this project. Profiles contain
1212
# database connection information, and should be configured in the ~/.dbt/profiles.yml file
13-
profile: 'normalize'
13+
profile: "normalize"
1414

1515
# These configurations specify where dbt should look for different types of files.
1616
# The `source-paths` config, for example, states that source models can be found
@@ -22,25 +22,26 @@ test-paths: ["tests"]
2222
data-paths: ["data"]
2323
macro-paths: ["macros"]
2424

25-
target-path: "../build" # directory which will store compiled SQL files
26-
log-path: "../logs" # directory which will store DBT logs
27-
modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
25+
target-path: "../build" # directory which will store compiled SQL files
26+
log-path: "../logs" # directory which will store DBT logs
27+
modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
2828

29-
clean-targets: # directories to be removed by `dbt clean`
30-
- "build"
31-
- "dbt_modules"
29+
clean-targets: # directories to be removed by `dbt clean`
30+
- "build"
31+
- "dbt_modules"
3232

3333
quoting:
3434
database: true
35-
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
36-
# all schemas should be unquoted
35+
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
36+
# all schemas should be unquoted
3737
schema: false
3838
identifier: true
3939

4040
# You can define configurations for models in the `source-paths` directory here.
4141
# Using these configurations, you can enable or disable models, change how they
4242
# are materialized, and more!
4343
models:
44+
+transient: false
4445
airbyte_utils:
4546
+materialized: table
4647
generated:
@@ -60,4 +61,4 @@ models:
6061

6162
dispatch:
6263
- macro_namespace: dbt_utils
63-
search_order: ['airbyte_utils', 'dbt_utils']
64+
search_order: ["airbyte_utils", "dbt_utils"]

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as
44
(select * from(
55

66
-- Final base SQL model

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as
44
(select * from(
55

66
-- depends_on: ref('DEDUP_EXCHANGE_RATE_STG')

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as
44
(select * from(
55

66
with __dbt__cte__EXCHANGE_RATE_AB1 as (

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22

3-
create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as
3+
create or replace table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as
44
(select * from(
55

66
with __dbt__cte__EXCHANGE_RATE_AB1 as (

airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py

+2
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ def setup_test_dir(integration_type: str) -> str:
134134
copy_tree("../dbt-project-template-mysql", test_root_dir)
135135
elif integration_type == DestinationType.ORACLE.value:
136136
copy_tree("../dbt-project-template-oracle", test_root_dir)
137+
elif integration_type == DestinationType.SNOWFLAKE.value:
138+
copy_tree("../dbt-project-template-snowflake", test_root_dir)
137139
return test_root_dir
138140

139141

airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py

+3
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,9 @@ def setup_test_dir(destination_type: DestinationType, test_resource_name: str) -
192192
elif destination_type.value == DestinationType.CLICKHOUSE.value:
193193
copy_tree("../dbt-project-template-clickhouse", test_root_dir)
194194
dbt_project_yaml = "../dbt-project-template-clickhouse/dbt_project.yml"
195+
elif destination_type.value == DestinationType.SNOWFLAKE.value:
196+
copy_tree("../dbt-project-template-snowflake", test_root_dir)
197+
dbt_project_yaml = "../dbt-project-template-snowflake/dbt_project.yml"
195198
dbt_test_utils.copy_replace(dbt_project_yaml, os.path.join(test_root_dir, "dbt_project.yml"))
196199
return test_root_dir
197200

0 commit comments

Comments
 (0)