Merge pull request #101 from fishtown-analytics/fix/bq-macros-syntax

WIP fix: updated macros for bq compatibility

Author: axelazaid

.circleci/config.yml

@@ -10,6 +10,11 @@ jobs:
     steps:
       - checkout
 
+      - run:
+          run: setup_creds
+          command: |
+            echo $GCLOUD_SERVICE_KEY | base64 --decode --ignore-garbage > ${HOME}/gcloud-service-key.json
+
       - restore_cache:
           key: deps1-{{ .Branch }}
 
@@ -25,7 +30,7 @@ jobs:
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
 
       - run:
-          name: "Run tests (postgres)"
+          name: "Run Tests - Postgres"
           environment:
             CI_DBT_USER: root
             CI_DBT_PASS: ''
@@ -34,10 +39,47 @@ jobs:
           command: |
             . venv/bin/activate
             cd integration_tests
-            dbt deps
-            dbt seed --full-refresh
-            dbt run --full-refresh --exclude test_insert_by_period
-            dbt test --exclude test_insert_by_period
+            dbt deps --target postgres
+            dbt seed --target postgres --full-refresh
+            dbt run --target postgres
+            dbt test --target postgres
+
+      - run:
+          name: "Run Tests - Redshift"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps --target redshift
+            dbt seed --target redshift --full-refresh
+            dbt run --target redshift
+            dbt test --target redshift
+
+      - run:
+          name: "Run Tests - Snowflake"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps --target snowflake
+            dbt seed --target snowflake --full-refresh
+            dbt run --target snowflake --exclude test_unpivot
+            dbt test --target snowflake --exclude test_unpivot
+
+      - run:
+          name: "Run Tests - BigQuery"
+          environment:
+              GCLOUD_SERVICE_KEY_PATH: "/home/circleci/gcloud-service-key.json"
+
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps --target bigquery
+            dbt seed --target bigquery --full-refresh
+            dbt run --target bigquery
+            dbt test --target bigquery
+
 
       - save_cache:
           key: deps1-{{ .Branch }}

integration_tests/ci/sample.profiles.yml

@@ -7,14 +7,43 @@ config:
     use_colors: True
 
 integration_tests:
-  target: ci
+  target: postgres
   outputs:
-    ci:
+    postgres:
       type: postgres
       host: localhost
       user: "{{ env_var('CI_DBT_USER') }}"
       pass: "{{ env_var('CI_DBT_PASS') }}"
       port: "{{ env_var('CI_DBT_PORT') }}"
       dbname: "{{ env_var('CI_DBT_DBNAME') }}"
-      schema: dbt_utils
+      schema: dbt_utils_integration_tests_postgres
+      threads: 1
+
+    redshift:
+      type: redshift
+      host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}"
+      user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}"
+      pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
+      dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
+      port: 5439
+      schema: dbt_utils_integration_tests_redshift
+      threads: 1
+
+    bigquery:
+      type: bigquery
+      method: service-account
+      keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}"
+      project: 'dbt-integration-tests'
+      schema: dbt_utils_integration_tests_bigquery
+      threads: 1
+
+    snowflake:
+      type: snowflake
+      account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}"
+      user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}"
+      password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}"
+      role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
+      database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
+      warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
+      schema: dbt_utils_integration_tests_snowflake
       threads: 1

integration_tests/data/cross_db/data_split_part.csv

@@ -1,4 +1,4 @@
 parts,split_on,result_1,result_2,result_3
 a|b|c,|,a,b,c
 1|2|3,|,1,2,3
-,|,,
+,|,,,

integration_tests/data/datetime/data_date_spine.csv

@@ -7,4 +7,4 @@ date_day
 2018-01-06
 2018-01-07
 2018-01-08
-2018-01-09
+2018-01-09

integration_tests/dbt_project.yml

@@ -14,3 +14,24 @@ target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
     - "target"
     - "dbt_modules"
+    
+seeds:
+    dbt_utils_integration_tests:
+        enabled: true
+        
+        cross_db:
+            data_date_trunc:
+                column_types:
+                    updated_at: timestamp
+                    day: date
+                    month: date
+        
+            data_dateadd:
+                column_types:
+                    from_time: timestamp
+                    result: timestamp
+                    
+            data_datediff:
+                column_types:
+                    first_date: timestamp
+                    second_date: timestamp

integration_tests/models/cross_db_utils/test_date_trunc.sql

@@ -6,15 +6,15 @@ with data as (
 )
 
 select
-    {{ dbt_utils.date_trunc('day', 'updated_at') }} as actual,
+    cast({{dbt_utils.date_trunc('day', 'updated_at') }} as date) as actual,
     day as expected
 
 from data
 
 union all
 
 select
-    {{ dbt_utils.date_trunc('month', 'updated_at') }} as actual,
+    cast({{ dbt_utils.date_trunc('month', 'updated_at') }} as date) as actual,
     month as expected
 
 from data

integration_tests/models/cross_db_utils/test_dateadd.sql

@@ -7,10 +7,10 @@ with data as (
 
 select
     case
-        when datepart = 'hour' then {{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }}
-        when datepart = 'day' then {{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }}
-        when datepart = 'month' then {{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }}
-        when datepart = 'year' then {{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }}
+        when datepart = 'hour' then cast({{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}})
+        when datepart = 'day' then cast({{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}})
+        when datepart = 'month' then cast({{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}})
+        when datepart = 'year' then cast({{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}})
         else null
     end as actual,
     result as expected

integration_tests/models/datetime/test_date_spine.sql

@@ -11,6 +11,11 @@ with date_spine as (
     {% if target.type == 'postgres' %}
         {{ log("WARNING: Not testing - datediff macro is unsupported on Postgres", info=True) }}
         select * from {{ ref('data_date_spine') }}
+        
+    {% elif target.type == 'bigquery' %}
+        select cast(date_day as date) as date_day
+        from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }})
+    
     {% else %}
         {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}
     {% endif %}

integration_tests/models/materializations/expected_insert_by_period.sql

@@ -1,6 +1,8 @@
 {{
 	config(
-		materialized = 'view')
+		materialized = 'view',
+		enabled=(target.type == 'redshift')
+	)
 }}
 
 select *

integration_tests/models/materializations/test_insert_by_period.sql

@@ -4,7 +4,9 @@
 		period = 'month',
 		timestamp_field = 'created_at',
 		start_date = '2018-01-01',
-		stop_date = '2018-06-01')
+		stop_date = '2018-06-01',
+		enabled=(target.type == 'redshift')
+	)
 }}
 
 with events as (

integration_tests/models/schema_tests/test_recency.sql

@@ -1,3 +1,12 @@
 
+{% if target.type == 'postgres' %}
+
 select
     {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today
+
+{% else %}
+
+select
+    cast({{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as datetime) as today
+    
+{% endif %}

integration_tests/models/sql/test_get_column_values.sql

@@ -2,6 +2,8 @@
 {% set columns = dbt_utils.get_column_values(ref('data_get_column_values'), 'field') %}
 
 
+{% if target.type == 'snowflake' %}
+
 select
     {% set columns = columns if columns is iterable else [] %}
     {% for column in columns -%}
@@ -12,3 +14,18 @@ select
     {%- endfor %}
 
 from {{ ref('data_get_column_values') }}
+
+{% else %}
+
+select
+    {% set columns = columns if columns is iterable else [] %}
+    {% for column in columns -%}
+
+        {{dbt_utils.safe_cast("sum(case when field = '" ~ column ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ column }}
+        {%- if not loop.last %},{% endif -%}
+
+    {%- endfor %}
+
+from {{ ref('data_get_column_values') }}
+
+{% endif %}

integration_tests/models/sql/test_get_tables_by_prefix_and_union.sql

@@ -1,9 +1,11 @@
+{{config( materialized = 'table')}}
+
 
 {% if target.type == 'snowflake' %}
 
     {% set tables = dbt_utils.get_tables_by_prefix((target.schema | upper), 'data_events_') %}
     {{ dbt_utils.union_tables(tables) }}
-
+    
 {% else %}
 
     {% set tables = dbt_utils.get_tables_by_prefix(target.schema, 'data_events_') %}

integration_tests/models/sql/test_groupby.sql

@@ -1,8 +1,24 @@
+with test_data as (
+    
+    select
+    
+        {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string() )}} as column_1,
+        {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string() )}} as column_2
+    
+),
+
+grouped as (
+
+    select 
+        *,
+        count(*) as total
+
+    from test_data
+    {{ dbt_utils.group_by(2) }}
+    
+)
+
+select * from grouped
 
 
-select
-    'a' as column_1,
-    'b' as column_2,
-    count(*) as total
 
-{{ dbt_utils.group_by(2) }}

integration_tests/models/sql/test_nullcheck_table.sql

@@ -1,15 +1,41 @@
-
+{{ config( materialized = "table" ) }}
+ 
+-- TO DO: remove if-statement
+ 
 {% set tbl = ref('data_nullcheck_table') %}
 
+
 with nulled as (
 
     {{ dbt_utils.nullcheck_table(tbl.schema, tbl.name) }}
 
 )
 
+{% if target.type == 'snowflake' %}
+
 select
     field_1::varchar as field_1,
     field_2::varchar as field_2,
     field_3::varchar as field_3
 
 from nulled
+
+{% else %}
+
+select
+
+    {{ dbt_utils.safe_cast('field_1',
+        dbt_utils.type_string()
+    )}} as field_1,
+
+    {{ dbt_utils.safe_cast('field_2',
+        dbt_utils.type_string()
+    )}} as field_2,
+
+    {{ dbt_utils.safe_cast('field_3',
+        dbt_utils.type_string()
+    )}} as field_3
+
+from nulled
+
+{% endif %}

integration_tests/models/sql/test_unpivot.sql

@@ -1 +1,6 @@
-{{ dbt_utils.unpivot(table=ref('data_unpivot'), cast_to='varchar', exclude=['customer_id','created_at']) }}
+{{ dbt_utils.unpivot(
+    table=ref('data_unpivot'), 
+    cast_to=dbt_utils.type_string(), 
+    exclude=['customer_id','created_at']
+
+) }}

macros/cross_db_utils/date_trunc.sql

@@ -7,5 +7,10 @@
 {% endmacro %}
 
 {% macro bigquery__date_trunc(datepart, date) %}
-    date_trunc({{date}}, {{datepart}})
+    timestamp_trunc(
+        cast({{date}} as timestamp), 
+        {{datepart}}
+    )
+    
+
 {% endmacro %}

macros/cross_db_utils/dateadd.sql

@@ -16,8 +16,8 @@
 
 {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %}
 
-    date_add(
-        cast( {{ from_date_or_timestamp }} as date),
+        datetime_add(
+            cast( {{ from_date_or_timestamp }} as datetime),
         interval {{ interval }} {{ datepart }}
         )
 

macros/cross_db_utils/datediff.sql

@@ -16,11 +16,11 @@
 
 {% macro bigquery__datediff(first_date, second_date, datepart) %}
 
-    date_diff(
-        {{second_date}},
-        {{first_date}},
+    datetime_diff(
+        cast({{second_date}} as datetime),
+        cast({{first_date}} as datetime),
         {{datepart}}
-        )
+    ) 
 
 {% endmacro %}