Feature: Fetch Macro

README.md

@@ -214,6 +214,24 @@ models:
 
 ---
 ### SQL helpers
+#### get_query_results_as_dict ([source](macros/sql/get_query_results_as_dict.sql))
+This macro returns a dictionary from a sql query, so that you don't need to interact with the Agate library to operate on teh reuslt
+
+Usage:
+```
+-- Returns a dictionary of the users table where the state is California
+{% set california_cities = dbt_utils.get_query_results_as_dict("select * from" ~ ref('cities') ~ "where state = 'CA' and city is not null ") %}
+select
+  city,
+{% for city in california_cities %}
+  sum(case when city = {{ city }} then 1 else 0 end) as users_in_{{ city }},
+{% endfor %}
+  count(*) as total
+from {{ ref('users') }}
+
+group by 1
+```
+
 #### get_column_values ([source](macros/sql/get_column_values.sql))
 This macro returns the unique values for a column in a given [relation](https://docs.getdbt.com/docs/api-variable#section-relation).
 It takes an options `default` argument for compiling when the relation does not already exist.

integration_tests/Makefile

@@ -1,24 +1,20 @@
 
 test-postgres:
-	dbt compile --target postgres
 	dbt seed --target postgres --full-refresh
 	dbt run --target postgres --full-refresh --exclude test_insert_by_period
 	dbt test --target postgres --exclude test_insert_by_period
 
 test-redshift:
-	dbt compile --target redshift
 	dbt seed --target redshift --full-refresh
 	dbt run --target redshift --full-refresh
 	dbt test --target redshift
 
 test-snowflake:
-	dbt compile --target snowflake
 	dbt seed --target snowflake --full-refresh
 	dbt run --target snowflake --full-refresh
 	dbt test --target snowflake
 
 test-bigquery:
-	dbt compile --target bigquery
 	dbt seed --target bigquery --full-refresh
 	dbt run --target bigquery --full-refresh
 	dbt test --target bigquery

integration_tests/data/sql/data_get_query_results_as_dict.csv

@@ -0,0 +1,4 @@
+col_1,col_2,col_3
+1,a,True
+2,b,False
+3,c,

integration_tests/dbt_project.yml

@@ -4,8 +4,10 @@ version: '1.0'
 
 profile: 'integration_tests'
 
+require-dbt-version: ">=0.15.0"
+
 source-paths: ["models"]
-analysis-paths: ["analysis"] 
+analysis-paths: ["analysis"]
 test-paths: ["tests"]
 data-paths: ["data"]
 macro-paths: ["macros"]
@@ -14,24 +16,26 @@ target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
     - "target"
     - "dbt_modules"
-    
+
 seeds:
-    dbt_utils_integration_tests:
-        enabled: true
-
-        cross_db:
-            data_date_trunc:
-                column_types:
-                    updated_at: timestamp
-                    day: date
-                    month: date
-
-            data_dateadd:
-                column_types:
-                    from_time: timestamp
-                    result: timestamp
-
-            data_datediff:
-                column_types:
-                    first_date: timestamp
-                    second_date: timestamp
+  quote_columns: false
+  dbt_utils_integration_tests:
+    enabled: true
+
+    cross_db:
+      data_date_trunc:
+        column_types:
+          updated_at: timestamp
+          day: date
+          month: date
+
+      data_dateadd:
+        column_types:
+          from_time: timestamp
+          result: timestamp
+
+      data_datediff:
+        column_types:
+          first_date: timestamp
+          second_date: timestamp
+

integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql

@@ -0,0 +1,81 @@
+-- depends_on: {{ ref('data_get_query_results_as_dict') }}
+
+{% set expected_dictionary={
+    'col_1': [1, 2, 3],
+    'col_2': ['a', 'b', 'c'],
+    'col_3': [True, False, none]
+} %}
+
+{#- Handle snowflake casing silliness -#}
+{% if target.type == 'snowflake' %}
+{% set expected_dictionary={
+    'COL_1': [1, 2, 3],
+    'COL_2': ['a', 'b', 'c'],
+    'COL_3': [True, False, none]
+} %}
+{% endif %}
+
+
+{% set actual_dictionary=dbt_utils.get_query_results_as_dict(
+    "select * from " ~ ref('data_get_query_results_as_dict')
+) %}
+{#-
+For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary.
+Instead, we'll manually check that the values of these dictionaries are equivalent.
+-#}
+
+{% set ns = namespace(
+    pass=True,
+    err_msg = ""
+) %}
+{% if execute %}
+{#- Check that the dictionaries have the same keys -#}
+{% set expected_keys=expected_dictionary.keys() | list | sort %}
+{% set actual_keys=actual_dictionary.keys() | list | sort %}
+
+{% if expected_keys != actual_keys %}
+    {% set ns.pass=False %}
+    {% set ns.err_msg %}
+    The two dictionaries have different keys:
+      expected_dictionary has keys: {{ expected_keys }}
+      actual_dictionary has keys: {{ actual_keys }}
+    {% endset %}
+
+{% else %}
+
+{% for key, value in expected_dictionary.items() %}
+    {% set expected_length=expected_dictionary[key] | length %}
+    {% set actual_length=actual_dictionary[key] | length %}
+
+    {% if expected_length != actual_length %}
+        {% set ns.pass=False %}
+        {% set ns.err_msg %}
+    The {{ key }} column has different lengths:
+      expected_dictionary[{{ key }}] has length {{ expected_length }}
+      actual_dictionary[{{ key }}] has length {{ actual_length }}
+        {% endset %}
+
+    {% else %}
+
+        {% for i in range(value | length) %}
+            {% set expected_value=expected_dictionary[key][i] %}
+            {% set actual_value=actual_dictionary[key][i] %}
+            {% if expected_value != actual_value %}
+                {% set ns.pass=False %}
+                {% set ns.err_msg %}
+    The {{ key }} column has differing values:
+      expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }}
+      actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }}
+                {% endset %}
+
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+
+{% endfor %}
+
+{% endif %}
+
+{{ log(ns.err_msg, info=True) }}
+select 1 {% if ns.pass %} limit 0 {% endif %}
+{% endif %}

macros/sql/get_query_results_as_dict.sql

@@ -0,0 +1,21 @@
+{% macro get_query_results_as_dict(query) %}
+{# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #}
+
+    {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%}
+
+        {{ query }}
+
+    {%- endcall -%}
+
+    {% set sql_results={} %}
+
+    {%- if execute -%}
+        {% set sql_results_table = load_result('get_query_results').table.columns %}
+        {% for column_name, column in sql_results_table.items() %}
+            {% do sql_results.update({column_name: column.values()}) %}
+        {% endfor %}
+    {%- endif -%}
+
+    {{ return(sql_results) }}
+
+{% endmacro %}