Merge pull request #82 from fishtown-analytics/bq/union

make union work on bq

Author: drewbanin

.circleci/config.yml

@@ -18,7 +18,9 @@ jobs:
           command: |
             python3 -m venv venv
             . venv/bin/activate
+
             pip install dbt
+
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
 
@@ -33,8 +35,8 @@ jobs:
             . venv/bin/activate
             cd integration_tests
             dbt deps
-            dbt seed
-            dbt run
+            dbt seed --full-refresh
+            dbt run --full-refresh
             dbt test
 
       - save_cache:

integration_tests/Makefile

@@ -1,22 +1,22 @@
 
 test-postgres:
-	dbt seed --target postgres
-	dbt run --target postgres
+	dbt seed --target postgres --full-refresh
+	dbt run --target postgres --full-refresh
 	dbt test --target postgres
 
 test-redshift:
-	dbt seed --target redshift
-	dbt run --target redshift
+	dbt seed --target redshift --full-refresh
+	dbt run --target redshift --full-refresh
 	dbt test --target redshift
 
 test-snowflake:
-	dbt seed --target snowflake
-	dbt run --target snowflake
+	dbt seed --target snowflake --full-refresh
+	dbt run --target snowflake --full-refresh
 	dbt test --target snowflake
 
 test-bigquery:
-	dbt seed --target bigquery
-	dbt run --target bigquery
+	dbt seed --target bigquery --full-refresh
+	dbt run --target bigquery --full-refresh
 	dbt test --target bigquery
 
 test-all: test-postgres test-redshift test-snowflake test-bigquery

integration_tests/data/sql/data_union_expected.csv

@@ -0,0 +1,6 @@
+id,name,favorite_color
+1,"drew",
+2,"bob",
+3,"alice",
+1,,"green"
+2,,"pink"

integration_tests/data/sql/data_union_table_1.csv

@@ -0,0 +1,4 @@
+id,name
+1,drew
+2,bob
+3,alice

integration_tests/data/sql/data_union_table_2.csv

@@ -0,0 +1,3 @@
+id,favorite_color
+1,green
+2,pink

integration_tests/models/sql/schema.yml

@@ -56,3 +56,8 @@ test_surrogate_key:
     constraints:
         assert_equal:
             - {actual: actual, expected: expected}
+
+test_union:
+    constraints:
+        dbt_utils.equality:
+            - ref('data_union_expected')

integration_tests/models/sql/test_get_column_values.sql

@@ -3,6 +3,7 @@
 
 
 select
+    {% set columns = columns if columns is iterable else [] %}
     {% for column in columns -%}
 
         sum(case when field = '{{ column }}' then 1 else 0 end) as count_{{ column }}

integration_tests/models/sql/test_union.sql

@@ -0,0 +1,8 @@
+
+select
+    id,
+    name,
+    favorite_color
+
+from {{ ref('test_union_base') }}
+

integration_tests/models/sql/test_union_base.sql

@@ -0,0 +1,6 @@
+
+{{ dbt_utils.union_tables([
+        ref('data_union_table_1'),
+        ref('data_union_table_2')]
+) }} 
+

macros/cross_db_utils/literal.sql

@@ -0,0 +1,8 @@
+
+{% macro string_literal(value) %}
+  {{ adapter_macro('dbt_utils.string_literal', value) }}
+{% endmacro %}
+
+{% macro default__string_literal(value) -%}
+    '{{ value }}'
+{%- endmacro %}

macros/sql/union.sql

@@ -56,15 +56,15 @@
         (
             select
 
-                '{{ table }}'::text as _dbt_source_table,
+                {{ dbt_utils.safe_cast(dbt_utils.string_literal(table), dbt_utils.type_string()) }} as _dbt_source_table,
 
                 {% for col_name in ordered_column_names -%}
 
                     {%- set col = column_superset[col_name] %}
                     {%- set col_type = column_override.get(col.column, col.data_type) %}
                     {%- set col_name = adapter.quote(col_name) if col_name in table_columns[table] else 'null' %}
 
-                    {{ col_name }}::{{ col_type }} as {{ col.quoted }} {% if not loop.last %},{% endif %}
+                    {{ dbt_utils.safe_cast(col_name, col_type) }} as {{ col.quoted }} {% if not loop.last %},{% endif %}
                 {%- endfor %}
 
             from {{ table }}