Skip to content

🐛 fix: Normalization date-time should handle empty strings "" #6379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 26 commits into from
Oct 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
535cf8c
add empty string normalization for postgres
annalvova05 Sep 22, 2021
789e81f
Merge remote-tracking branch 'origin/master' into alvova/5480-fix-emp…
annalvova05 Sep 22, 2021
196bc1e
add empty string normalization for destinations
annalvova05 Sep 22, 2021
d10f045
fix
annalvova05 Sep 22, 2021
aa7a221
fix
annalvova05 Sep 23, 2021
4856ef8
fix
annalvova05 Sep 23, 2021
2b86610
Merge remote-tracking branch 'origin/master' into alvova/5480-fix-emp…
annalvova05 Sep 23, 2021
a699acf
fix for snowflake
annalvova05 Sep 23, 2021
d46028b
Merge remote-tracking branch 'origin/master' into alvova/5480-fix-emp…
annalvova05 Sep 23, 2021
34dd6db
fix for mysql
annalvova05 Sep 23, 2021
b86942b
fix normalization for mysql
annalvova05 Sep 24, 2021
d38826e
upd doc
annalvova05 Sep 24, 2021
65f8c2b
upd doc
annalvova05 Sep 24, 2021
c0e8cb4
Update airbyte-integrations/bases/base-normalization/integration_test…
annalvova05 Oct 6, 2021
0a0c485
Update airbyte-integrations/bases/base-normalization/integration_test…
annalvova05 Oct 6, 2021
4d79ac7
Merge remote-tracking branch 'origin/master' into alvova/5480-fix-emp…
annalvova05 Oct 7, 2021
4ea412b
bump version
annalvova05 Oct 7, 2021
84e40e4
bump version
annalvova05 Oct 7, 2021
5d692a8
Merge remote-tracking branch 'origin/master' into alvova/5480-fix-emp…
annalvova05 Oct 7, 2021
81bcf4f
add datetime normalization for mssql
annalvova05 Oct 7, 2021
ce65d57
upd row count for mssql
annalvova05 Oct 8, 2021
9a260c6
Merge remote-tracking branch 'origin/master' into alvova/5480-fix-emp…
annalvova05 Oct 8, 2021
9e04bb0
upd
annalvova05 Oct 8, 2021
86b48e8
Merge remote-tracking branch 'origin/master' into alvova/5480-fix-emp…
annalvova05 Oct 8, 2021
c8e2529
bump version
annalvova05 Oct 8, 2021
4c3eb93
upd docs for 0.1.50 normalization version
annalvova05 Oct 8, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions airbyte-integrations/bases/base-normalization/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ RUN pip install cx_Oracle

COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte

# Install SSH Tunneling dependencies
# Install SSH Tunneling dependencies
RUN apt-get update && apt-get install -y jq sshpass
WORKDIR /airbyte
COPY entrypoint.sh .
Expand Down Expand Up @@ -63,5 +63,5 @@ WORKDIR /airbyte
ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh"
ENTRYPOINT ["/airbyte/entrypoint.sh"]

LABEL io.airbyte.version=0.1.51
LABEL io.airbyte.version=0.1.52
LABEL io.airbyte.name=airbyte/normalization
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,12 @@
{% macro sqlserver__cast_to_boolean(field) -%}
cast({{ field }} as bit)
{%- endmacro %}

{# empty_string_to_null ------------------------------------------------- #}
{% macro empty_string_to_null(field) -%}
{{ return(adapter.dispatch('empty_string_to_null')(field)) }}
{%- endmacro %}

{%- macro default__empty_string_to_null(field) -%}
nullif({{ field }}, '')
{%- endmacro %}
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@ select
cast(currency as
string
) as currency,
cast(date as
cast(nullif(date, '') as
date
) as date,
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
timestamp
) as timestamp_col,
cast(HKD_special___characters as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@ select
cast(currency as
string
) as currency,
cast(date as
cast(nullif(date, '') as
date
) as date,
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
timestamp
) as timestamp_col,
cast(HKD_special___characters as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast(date as {{ type_date() }}) as date,
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(empty_string_to_null('date') as {{ type_date() }}) as date,
cast(empty_string_to_null('timestamp_col') as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(HKD_special___characters as {{ dbt_utils.type_float() }}) as HKD_special___characters,
cast(HKD_special___characters_1 as {{ dbt_utils.type_string() }}) as HKD_special___characters_1,
cast(NZD as {{ dbt_utils.type_float() }}) as NZD,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast(date as {{ type_date() }}) as date,
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(empty_string_to_null('date') as {{ type_date() }}) as date,
cast(empty_string_to_null('timestamp_col') as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(HKD_special___characters as {{ dbt_utils.type_float() }}) as HKD_special___characters,
cast(HKD_special___characters_1 as {{ dbt_utils.type_string() }}) as HKD_special___characters_1,
cast(NZD as {{ dbt_utils.type_float() }}) as NZD,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ select
) as id,
cast(currency as
VARCHAR(max)) as currency,
try_parse("date" as date) as "date",
try_parse(timestamp_col as datetime) as timestamp_col,
try_parse(nullif("date", '''') as date) as "date",
try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col,
cast("HKD@spéçiäl & characters" as
float
) as "HKD@spéçiäl & characters",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ select
) as id,
cast(currency as
VARCHAR(max)) as currency,
try_parse("date" as date) as "date",
try_parse(timestamp_col as datetime) as timestamp_col,
try_parse(nullif("date", '''') as date) as "date",
try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col,
cast("HKD@spéçiäl & characters" as
float
) as "HKD@spéçiäl & characters",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
with table_row_counts as (
select distinct count(*) as row_count, 9 as expected_count
select distinct count(*) as row_count, 10 as expected_count
from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }}
union all
select distinct count(*) as row_count, 9 as expected_count
select distinct count(*) as row_count, 10 as expected_count
from {{ ref('exchange_rate') }}

union all

select distinct count(*) as row_count, 9 as expected_count
select distinct count(*) as row_count, 10 as expected_count
from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }}
union all
select distinct count(*) as row_count, 9 as expected_count
select distinct count(*) as row_count, 10 as expected_count
from {{ ref('dedup_exchange_rate_scd') }}
union all
select distinct count(*) as row_count, 5 as expected_count
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
try_parse({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }},
try_parse(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
try_parse({{ empty_string_to_null(adapter.quote('date')) }} as {{ type_date() }}) as {{ adapter.quote('date') }},
try_parse({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
try_parse({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }},
try_parse(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
try_parse({{ empty_string_to_null(adapter.quote('date')) }} as {{ type_date() }}) as {{ adapter.quote('date') }},
try_parse({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,11 @@ select
signed
) as id,
cast(currency as char) as currency,
cast(`date` as
date
) as `date`,
cast(timestamp_col as char) as timestamp_col,
case when `date` = '' then NULL
else cast(`date` as date)
end as `date`
,
cast(nullif(timestamp_col, '') as char) as timestamp_col,
cast(`HKD@spéçiäl & characters` as
float
) as `HKD@spéçiäl & characters`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,11 @@ select
signed
) as id,
cast(currency as char) as currency,
cast(`date` as
date
) as `date`,
cast(timestamp_col as char) as timestamp_col,
case when `date` = '' then NULL
else cast(`date` as date)
end as `date`
,
cast(nullif(timestamp_col, '') as char) as timestamp_col,
cast(`HKD@spéçiäl & characters` as
float
) as `HKD@spéçiäl & characters`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }},
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
case when {{ adapter.quote('date') }} = '' then NULL
else cast({{ adapter.quote('date') }} as date)
end as {{ adapter.quote('date') }}
,
cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }},
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
case when {{ adapter.quote('date') }} = '' then NULL
else cast({{ adapter.quote('date') }} as date)
end as {{ adapter.quote('date') }}
,
cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ select
numeric
) as id,
cast(currency as varchar2(4000)) as currency,
cast("DATE" as
cast(nullif("DATE", '') as
varchar2(4000)
) as "DATE",
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
varchar2(4000)
) as timestamp_col,
cast(hkd_special___characters as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ select
numeric
) as id,
cast(currency as varchar2(4000)) as currency,
cast("DATE" as
cast(nullif("DATE", '') as
varchar2(4000)
) as "DATE",
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
varchar2(4000)
) as timestamp_col,
cast(hkd_special___characters as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast({{ quote('DATE') }} as {{ type_date() }}) as {{ quote('DATE') }},
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(empty_string_to_null({{ quote('DATE') }}) as {{ type_date() }}) as {{ quote('DATE') }},
cast(empty_string_to_null('timestamp_col') as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(hkd_special___characters as {{ dbt_utils.type_float() }}) as hkd_special___characters,
cast(hkd_special___characters_1 as {{ dbt_utils.type_string() }}) as hkd_special___characters_1,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast({{ quote('DATE') }} as {{ type_date() }}) as {{ quote('DATE') }},
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(empty_string_to_null({{ quote('DATE') }}) as {{ type_date() }}) as {{ quote('DATE') }},
cast(empty_string_to_null('timestamp_col') as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(hkd_special___characters as {{ dbt_utils.type_float() }}) as hkd_special___characters,
cast(hkd_special___characters_1 as {{ dbt_utils.type_string() }}) as hkd_special___characters_1,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ select
cast(currency as
varchar
) as currency,
cast("date" as
cast(nullif("date", '') as
date
) as "date",
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
timestamp with time zone
) as timestamp_col,
cast("HKD@spéçiäl & characters" as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ select
cast(currency as
varchar
) as currency,
cast("date" as
cast(nullif("date", '') as
date
) as "date",
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
timestamp with time zone
) as timestamp_col,
cast("HKD@spéçiäl & characters" as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }},
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }},
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ empty_string_to_null(adapter.quote('date')) }} as {{ type_date() }}) as {{ adapter.quote('date') }},
cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }},
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }},
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ empty_string_to_null(adapter.quote('date')) }} as {{ type_date() }}) as {{ adapter.quote('date') }},
cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ select
bigint
) as id,
cast(currency as varchar) as currency,
cast(date as
cast(nullif(date, '') as
date
) as date,
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
timestamp with time zone
) as timestamp_col,
cast("hkd@spéçiäl & characters" as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ select
bigint
) as id,
cast(currency as varchar) as currency,
cast(date as
cast(nullif(date, '') as
date
) as date,
cast(timestamp_col as
cast(nullif(timestamp_col, '') as
timestamp with time zone
) as timestamp_col,
cast("hkd@spéçiäl & characters" as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast(date as {{ type_date() }}) as date,
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(empty_string_to_null('date') as {{ type_date() }}) as date,
cast(empty_string_to_null('timestamp_col') as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('hkd@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('hkd@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
select
cast(id as {{ dbt_utils.type_bigint() }}) as id,
cast(currency as {{ dbt_utils.type_string() }}) as currency,
cast(date as {{ type_date() }}) as date,
cast(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast(empty_string_to_null('date') as {{ type_date() }}) as date,
cast(empty_string_to_null('timestamp_col') as {{ type_timestamp_with_timezone() }}) as timestamp_col,
cast({{ adapter.quote('hkd@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('hkd@spéçiäl & characters') }},
cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters,
cast(nzd as {{ dbt_utils.type_float() }}) as nzd,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,15 @@ select
cast(CURRENCY as
varchar
) as CURRENCY,
cast(DATE as
cast(nullif(DATE, '') as
date
) as DATE,
case
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH')
when TIMESTAMP_COL = '' then NULL
else to_timestamp_tz(TIMESTAMP_COL)
end as TIMESTAMP_COL
,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,15 @@ select
cast(CURRENCY as
varchar
) as CURRENCY,
cast(DATE as
cast(nullif(DATE, '') as
date
) as DATE,
case
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH')
when TIMESTAMP_COL = '' then NULL
else to_timestamp_tz(TIMESTAMP_COL)
end as TIMESTAMP_COL
,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@
select
cast(ID as {{ dbt_utils.type_bigint() }}) as ID,
cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY,
cast(DATE as {{ type_date() }}) as DATE,
cast({{ empty_string_to_null('DATE') }} as {{ type_date() }}) as DATE,
case
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM')
when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH')
when TIMESTAMP_COL = '' then NULL
else to_timestamp_tz(TIMESTAMP_COL)
end as TIMESTAMP_COL
,
Expand Down
Loading