Skip to content

Commit 6e8c92a

Browse files
mtcarlonemtcarlone
authored
Merge pull request #488 from brooklyn-data/bug/handle-microbatch
Bug: Handle microbatch datetime objeect
2 parents 857193f + 3d82175 commit 6e8c92a

File tree

9 files changed

+191
-8
lines changed

9 files changed

+191
-8
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ Pipfile.lock
1111
env.sh
1212

1313
.python_version
14+
.python-version
1415
.vscode
1516
.venv
17+
pyproject.toml
1618

1719
.DS_Store

‎integration_test_project/models/microbatch.sql

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{{
2+
config(
3+
materialized = 'incremental',
4+
incremental_strategy = 'microbatch',
5+
unique_key = 'transaction_id',
6+
event_time = 'transaction_ts',
7+
begin = (modules.datetime.datetime.now().replace(hour=6, minute=0, second=0) - modules.datetime.timedelta(2)).isoformat(),
8+
batch_size = 'day'
9+
)
10+
}}
11+
12+
with transactions as (
13+
select * from {{ ref('microbatch_transaction') }}
14+
)
15+
16+
, stores as (
17+
select * from {{ ref('microbatch_store') }}
18+
)
19+
20+
select
21+
transactions.transaction_id,
22+
transactions.transaction_ts__daily as transaction_ts,
23+
stores.store_name,
24+
transactions.unit_price * transactions.transaction_qty as total_price,
25+
transactions.product_id
26+
from transactions
27+
left join stores
28+
on transactions.store_id = stores.store_id

‎integration_test_project/models/microbatch_store.sql

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
select
2+
5 as store_id,
3+
'Lower Manhattan' as store_name
4+
5+
union all
6+
7+
select
8+
8 as store_id,
9+
'Hells Kitchen' as store_name

‎integration_test_project/models/microbatch_transaction.sql

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{{
2+
config(
3+
materialized = 'table',
4+
event_time = 'transaction_ts__hourly'
5+
)
6+
}}
7+
8+
select
9+
transaction_id,
10+
transaction_ts__hourly,
11+
transaction_ts__daily,
12+
transaction_qty,
13+
unit_price,
14+
store_id,
15+
product_id,
16+
product_type,
17+
product_detail
18+
from {{ ref('microbatch_transaction_base') }}

‎integration_test_project/models/microbatch_transaction_base.sql

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{{
2+
config(
3+
materialized = 'view',
4+
)
5+
}}
6+
7+
with
8+
mb_transactions as (
9+
select
10+
{{ dbt.cast('transaction_id', api.Column.translate_type('integer') ) }} as transaction_id,
11+
{{ dbt.cast('transaction_date', api.Column.translate_type('date') ) }} as transaction_date,
12+
{{ dbt.cast('transaction_time', api.Column.translate_type('string') ) }} as transaction_time,
13+
{{ dbt.cast('transaction_ts', api.Column.translate_type('timestamp') ) }} as transaction_ts,
14+
{{ dbt.cast('transaction_qty', api.Column.translate_type('integer') ) }} as transaction_qty,
15+
{{ dbt.cast('store_id', api.Column.translate_type('integer') ) }} as store_id,
16+
{{ dbt.cast('store_location', api.Column.translate_type('string') ) }} as store_location,
17+
{{ dbt.cast('product_id', api.Column.translate_type('integer') ) }} as product_id,
18+
{{ dbt.cast('unit_price', api.Column.translate_type('numeric') ) }} as unit_price,
19+
{{ dbt.cast('product_category', api.Column.translate_type('string') ) }} as product_category,
20+
{{ dbt.cast('product_type', api.Column.translate_type('string') ) }} as product_type,
21+
{{ dbt.cast('product_detail', api.Column.translate_type('string') ) }} as product_detail
22+
from {{ ref('microbatch_seed') }}
23+
)
24+
25+
, transaction_interval as (
26+
select
27+
transaction_id,
28+
case left(transaction_time, 2)
29+
when '07' then 0
30+
when '08' then -1
31+
else -2
32+
end as transaction_interval
33+
from mb_transactions
34+
)
35+
36+
/* do this to prevent and db errors in case we can't self reference ...*/
37+
, transaction_time_today as (
38+
select
39+
transaction_id,
40+
transaction_time,
41+
{{ dbt.cast(dbt.current_timestamp(), api.Column.translate_type('date') ) }} as todays_date
42+
from mb_transactions
43+
)
44+
45+
, transaction_time_today_string as (
46+
select
47+
transaction_id,
48+
transaction_time,
49+
todays_date,
50+
{{ dbt.cast('todays_date', api.Column.translate_type('string') ) }} as todays_date__str
51+
from transaction_time_today
52+
)
53+
54+
, transaction_times as (
55+
select
56+
transaction_id,
57+
todays_date,
58+
{{ dbt.concat(['todays_date__str', "' '", 'transaction_time']) }} as transaction_time__ts
59+
from transaction_time_today_string
60+
)
61+
62+
select
63+
t.*,
64+
{{ dbt.safe_cast('tt.transaction_time__ts', api.Column.translate_type('timestamp')) }} as transaction_ts__hourly,
65+
66+
{{ dbt.dateadd(
67+
datepart='day',
68+
interval='ti.transaction_interval',
69+
from_date_or_timestamp='tt.todays_date'
70+
)}} as transaction_ts__daily
71+
from mb_transactions as t
72+
left join transaction_times as tt
73+
on t.transaction_id = tt.transaction_id
74+
left join transaction_interval as ti
75+
on t.transaction_id = ti.transaction_id

‎integration_test_project/seeds/microbatch_seed.csv

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
transaction_id,transaction_date,transaction_time,transaction_ts,transaction_qty,store_id,store_location,product_id,unit_price,product_category,product_type,product_detail
2+
1,2023-01-01,07:06:11,2023-01-01 07:06:11,2,5,Lower Manhattan,32,3.0,Coffee,Gourmet brewed coffee,Ethiopia Rg
3+
2,2023-01-01,07:08:56,2023-01-01 07:08:56,2,5,Lower Manhattan,57,3.1,Tea,Brewed Chai tea,Spicy Eye Opener Chai Lg
4+
3,2023-01-01,07:14:04,2023-01-01 07:14:04,2,5,Lower Manhattan,59,4.5,Drinking Chocolate,Hot chocolate,Dark chocolate Lg
5+
4,2023-01-01,07:20:24,2023-01-01 07:20:24,1,5,Lower Manhattan,22,2.0,Coffee,Drip coffee,Our Old Time Diner Blend Sm
6+
5,2023-01-01,07:22:41,2023-01-01 07:22:41,2,5,Lower Manhattan,57,3.1,Tea,Brewed Chai tea,Spicy Eye Opener Chai Lg
7+
6,2023-01-01,07:22:41,2023-01-01 07:22:41,1,5,Lower Manhattan,77,3.0,Bakery,Scone,Oatmeal Scone
8+
7,2023-01-01,07:25:49,2023-01-01 07:25:49,1,5,Lower Manhattan,22,2.0,Coffee,Drip coffee,Our Old Time Diner Blend Sm
9+
8,2023-01-01,07:33:34,2023-01-01 07:33:34,2,5,Lower Manhattan,28,2.0,Coffee,Gourmet brewed coffee,Columbian Medium Roast Sm
10+
9,2023-01-01,07:39:13,2023-01-01 07:39:13,1,5,Lower Manhattan,39,4.25,Coffee,Barista Espresso,Latte Rg
11+
10,2023-01-01,07:39:34,2023-01-01 07:39:34,2,5,Lower Manhattan,58,3.5,Drinking Chocolate,Hot chocolate,Dark chocolate Rg
12+
11,2023-01-01,07:43:05,2023-01-01 07:43:05,1,5,Lower Manhattan,56,2.55,Tea,Brewed Chai tea,Spicy Eye Opener Chai Rg
13+
12,2023-01-01,07:44:35,2023-01-01 07:44:35,2,5,Lower Manhattan,33,3.5,Coffee,Gourmet brewed coffee,Ethiopia Lg
14+
13,2023-01-01,07:45:51,2023-01-01 07:45:51,1,5,Lower Manhattan,51,3.0,Tea,Brewed Black tea,Earl Grey Lg
15+
14,2023-01-01,07:48:19,2023-01-01 07:48:19,1,5,Lower Manhattan,57,3.1,Tea,Brewed Chai tea,Spicy Eye Opener Chai Lg
16+
15,2023-01-01,07:52:36,2023-01-01 07:52:36,2,5,Lower Manhattan,87,3.0,Coffee,Barista Espresso,Ouro Brasileiro shot
17+
16,2023-01-01,07:59:58,2023-01-01 07:59:58,2,5,Lower Manhattan,47,3.0,Tea,Brewed Green tea,Serenity Green Tea Lg
18+
17,2023-01-01,07:59:58,2023-01-01 07:59:58,1,5,Lower Manhattan,79,3.75,Bakery,Scone,Jumbo Savory Scone
19+
18,2023-01-01,08:00:18,2023-01-01 08:00:18,1,8,Hells Kitchen,42,2.5,Tea,Brewed herbal tea,Lemon Grass Rg
20+
19,2023-01-01,08:00:39,2023-01-01 08:00:39,2,8,Hells Kitchen,59,4.5,Drinking Chocolate,Hot chocolate,Dark chocolate Lg
21+
20,2023-01-01,08:11:45,2023-01-01 08:11:45,1,8,Hells Kitchen,61,4.75,Drinking Chocolate,Hot chocolate,Sustainably Grown Organic Lg
22+
21,2023-01-01,08:17:27,2023-01-01 08:17:27,2,8,Hells Kitchen,33,3.5,Coffee,Gourmet brewed coffee,Ethiopia Lg
23+
22,2023-01-01,08:24:26,2023-01-01 08:24:26,2,5,Lower Manhattan,56,2.55,Tea,Brewed Chai tea,Spicy Eye Opener Chai Rg
24+
23,2023-01-01,08:24:26,2023-01-01 08:24:26,1,5,Lower Manhattan,69,3.25,Bakery,Biscotti,Hazelnut Biscotti
25+
24,2023-01-01,08:29:38,2023-01-01 08:29:38,1,8,Hells Kitchen,56,2.55,Tea,Brewed Chai tea,Spicy Eye Opener Chai Rg
26+
25,2023-01-01,08:31:23,2023-01-01 08:31:23,1,8,Hells Kitchen,40,3.75,Coffee,Barista Espresso,Cappuccino
27+
26,2023-01-01,08:33:08,2023-01-01 08:33:08,1,5,Lower Manhattan,43,3.0,Tea,Brewed herbal tea,Lemon Grass Lg
28+
27,2023-01-01,08:33:08,2023-01-01 08:33:08,1,5,Lower Manhattan,76,3.5,Bakery,Biscotti,Chocolate Chip Biscotti
29+
28,2023-01-01,08:35:03,2023-01-01 08:35:03,2,5,Lower Manhattan,45,3.0,Tea,Brewed herbal tea,Peppermint Lg
30+
29,2023-01-01,08:35:03,2023-01-01 08:35:03,1,5,Lower Manhattan,71,3.75,Bakery,Pastry,Chocolate Croissant
31+
30,2023-01-01,08:41:57,2023-01-01 08:41:57,2,8,Hells Kitchen,40,3.75,Coffee,Barista Espresso,Cappuccino
32+
31,2023-01-01,08:52:03,2023-01-01 08:52:03,1,8,Hells Kitchen,38,3.75,Coffee,Barista Espresso,Latte
33+
32,2023-01-01,08:52:32,2023-01-01 08:52:32,1,5,Lower Manhattan,26,3.0,Coffee,Organic brewed coffee,Brazilian Rg
34+
33,2023-01-01,08:54:33,2023-01-01 08:54:33,2,5,Lower Manhattan,27,3.5,Coffee,Organic brewed coffee,Brazilian Lg
35+
34,2023-01-01,08:56:27,2023-01-01 08:56:27,2,5,Lower Manhattan,29,2.5,Coffee,Gourmet brewed coffee,Columbian Medium Roast Rg
36+
35,2023-01-01,08:57:46,2023-01-01 08:57:46,1,8,Hells Kitchen,54,2.5,Tea,Brewed Chai tea,Morning Sunrise Chai Rg
37+
36,2023-01-01,08:58:55,2023-01-01 08:58:55,1,5,Lower Manhattan,26,3.0,Coffee,Organic brewed coffee,Brazilian Rg

‎macros/_macros.yml

+9
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,15 @@ macros:
8080
type: string
8181
description: |
8282
The target schema that the v1 artifact sources are in - does not have to be different to `old_schema`
83+
## UTILS ##
84+
- name: copy_models
85+
description: |
86+
A macro to help encapsulate model copying, specifically for the upload_model macro. Namely, this macro is helpful as the
87+
contract for the graph nodes is prone to change without warning. This macro makes handling for these changes easier and more DRY.
88+
arguments:
89+
- name: model
90+
type: Node
91+
description: A node object from the dbt graph
8392

8493
## UPLOAD INDIVIDUAL DATASETS ##
8594
- name: upload_exposures

‎macros/upload_individual_datasets/upload_models.sql

+4-8
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,7 @@
2424
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(15)) }}
2525
from values
2626
{% for model in models -%}
27-
{% set model_copy = model.copy() -%}
28-
{% do model_copy.pop('raw_code', None) %}
27+
{% set model_copy = dbt_artifacts.copy_model(model) -%}
2928
(
3029
'{{ invocation_id }}', {# command_invocation_id #}
3130
'{{ model_copy.unique_id }}', {# node_id #}
@@ -59,8 +58,7 @@
5958
{% if models != [] %}
6059
{% set model_values %}
6160
{% for model in models -%}
62-
{% set model_copy = model.copy() -%}
63-
{% do model_copy.pop('raw_code', None) %}
61+
{% set model_copy = dbt_artifacts.copy_model(model) -%}
6462
(
6563
'{{ invocation_id }}', {# command_invocation_id #}
6664
'{{ model_copy.unique_id }}', {# node_id #}
@@ -94,8 +92,7 @@
9492
{% if models != [] %}
9593
{% set model_values %}
9694
{% for model in models -%}
97-
{% set model_copy = model.copy() -%}
98-
{% do model_copy.pop('raw_code', None) %}
95+
{% set model_copy = dbt_artifacts.copy_model(model) -%}
9996
(
10097
'{{ invocation_id }}', {# command_invocation_id #}
10198
'{{ model_copy.unique_id }}', {# node_id #}
@@ -134,8 +131,7 @@
134131
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"
135132
from ( values
136133
{% for model in models -%}
137-
{% set model_copy = model.copy() -%}
138-
{% do model_copy.pop('raw_code', None) %}
134+
{% set model_copy = dbt_artifacts.copy_model(model) -%}
139135
(
140136
'{{ invocation_id }}', {# command_invocation_id #}
141137
'{{ model_copy.unique_id }}', {# node_id #}

‎macros/utils/copy_model.sql

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% macro copy_model(model) %}
2+
{% set model_copy = model.copy() %}
3+
{% do model_copy.pop('raw_code', None) %}
4+
{% if model_copy.config.begin %}
5+
{% set _ = model_copy.config.update({"begin": model_copy.config.begin.strftime("%Y-%m-%dT%H:%M:%S.%f")}) %}
6+
{% endif %}
7+
8+
{{ return(model_copy) }}
9+
{% endmacro %}

0 commit comments

Comments
 (0)