Skip to content

Commit ebe9e29

Browse files
thinhhathinhha
authored
Merge pull request #1556 from GoogleCloudPlatform/jccb/autodq-proposal
autodq parsing proposal
2 parents 04fdb60 + fee474c commit ebe9e29

File tree

3 files changed

+43
-67
lines changed

3 files changed

+43
-67
lines changed

‎modules/dataplex-autodq/README.md

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -149,10 +149,8 @@ module "dataplex-autodq" {
149149
data = {
150150
resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations"
151151
}
152-
incremental_field = "modified_date"
153-
data_quality_spec_file = {
154-
path = "config/data_quality_spec.yaml"
155-
}
152+
incremental_field = "modified_date"
153+
data_quality_spec_file = "config/data_quality_spec.yaml"
156154
}
157155
# tftest modules=1 resources=1 files=data_quality_spec inventory=datascan_dq.yaml
158156
```
@@ -232,11 +230,8 @@ module "dataplex-autodq" {
232230
data = {
233231
resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations"
234232
}
235-
incremental_field = "modified_date"
236-
data_quality_spec_file = {
237-
path = "config/data_quality_spec_camel_case.yaml"
238-
convert_camel_case = true
239-
}
233+
incremental_field = "modified_date"
234+
data_quality_spec_file = "config/data_quality_spec_camel_case.yaml"
240235
}
241236
# tftest modules=1 resources=1 files=data_quality_spec_camel_case inventory=datascan_dq.yaml
242237
```

‎modules/dataplex-autodq/main.tf

Lines changed: 38 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -15,71 +15,55 @@
1515
*/
1616

1717
locals {
18-
prefix = var.prefix == null ? "" : "${var.prefix}-"
19-
_parsed_data_quality_spec = try(yamldecode(file(var.data_quality_spec_file.path)), null)
20-
_convert_camel_case = local._parsed_data_quality_spec == null ? null : try(var.data_quality_spec_file.convert_camel_case, false)
21-
_converted_data_quality_spec = local._parsed_data_quality_spec == null ? null : {
22-
sampling_percent = try(local._convert_camel_case ? local._parsed_data_quality_spec.samplingPercent : local._parsed_data_quality_spec.sampling_percent, null)
23-
row_filter = try(local._convert_camel_case ? local._parsed_data_quality_spec.rowFilter : local._parsed_data_quality_spec.row_filter, null)
18+
prefix = var.prefix == null ? "" : "${var.prefix}-"
19+
_file_data_quality_spec_raw = var.data_quality_spec_file != null ? yamldecode(file(var.data_quality_spec_file)) : tomap({})
20+
_file_data_quality_spec = var.data_quality_spec_file == null ? null : {
21+
sampling_percent = try(local._file_data_quality_spec_raw.samplingPercent, local._file_data_quality_spec_raw.sampling_percent, null)
22+
row_filter = try(local._file_data_quality_spec_raw.rowFilter, local._file_data_quality_spec_raw.row_filter, null)
2423
rules = [
25-
for rule in try(local._parsed_data_quality_spec.rules, []) : {
24+
for rule in try(local._file_data_quality_spec_raw.rules, []) : {
2625
column = try(rule.column, null)
27-
ignore_null = try(local._convert_camel_case ? rule.ignoreNull : rule.ignore_null, null)
26+
ignore_null = try(rule.ignoreNull, rule.ignore_null, null)
2827
dimension = rule.dimension
2928
threshold = try(rule.threshold, null)
30-
non_null_expectation = try(local._convert_camel_case ? rule.nonNullExpectation : rule.non_null_expectation, null)
31-
range_expectation = local._convert_camel_case ? (
32-
can(rule.rangeExpectation) ?
33-
{
34-
min_value = try(rule.rangeExpectation.minValue, null)
35-
max_value = try(rule.rangeExpectation.maxValue, null)
36-
strict_min_enabled = try(rule.rangeExpectation.strictMinEnabled, null)
37-
strict_max_enabled = try(rule.rangeExpectation.strictMaxEnabled, null)
38-
} : null
39-
) : try(rule.range_expectation, null)
40-
regex_expectation = local._convert_camel_case ? (
41-
can(rule.regexExpectation) ?
42-
{
43-
regex = try(rule.regexExpectation.regex, null)
44-
} : null
45-
) : try(rule.regex_expectation, null)
46-
set_expectation = local._convert_camel_case ? (
47-
can(rule.setExpectation) ?
48-
{
49-
values = try(rule.setExpectation.values, null)
50-
} : null
51-
) : try(rule.set_expectation, null)
52-
uniqueness_expectation = try(local._convert_camel_case ? rule.uniquenessExpectation : rule.uniqueness_expectation, null)
53-
statistic_range_expectation = local._convert_camel_case ? (
54-
can(rule.statisticRangeExpectation) ?
55-
{
56-
statistic = rule.statisticRangeExpectation.statistic
57-
min_value = try(rule.statisticRangeExpectation.minValue, null)
58-
max_value = try(rule.statisticRangeExpectation.maxValue, null)
59-
strict_min_enabled = try(rule.statisticRangeExpectation.strictMinEnabled, null)
60-
strict_max_enabled = try(rule.statisticRangeExpectation.strictMaxEnabled, null)
61-
} : null
62-
) : try(rule.statistic_range_expectation, null)
63-
row_condition_expectation = local._convert_camel_case ? (
64-
can(rule.rowConditionExpectation) ?
65-
{
66-
sql_expression = try(rule.rowConditionExpectation.sqlExpression, null)
67-
} : null
68-
) : try(rule.row_condition_expectation, null)
69-
table_condition_expectation = local._convert_camel_case ? (
70-
can(rule.tableConditionExpectation) ?
71-
{
72-
sql_expression = try(rule.tableConditionExpectation.sqlExpression, null)
73-
} : null
74-
) : try(rule.table_condition_expectation, null)
29+
non_null_expectation = try(rule.nonNullExpectation, rule.non_null_expectation, null)
30+
range_expectation = can(rule.rangeExpectation) || can(rule.range_expectation) ? {
31+
min_value = try(rule.rangeExpectation.minValue, rule.range_expectation.min_value, null)
32+
max_value = try(rule.rangeExpectation.maxValue, rule.range_expectation.max_value, null)
33+
strict_min_enabled = try(rule.rangeExpectation.strictMinEnabled, rule.range_expectation.strict_min_enabled, null)
34+
strict_max_enabled = try(rule.rangeExpectation.strictMaxEnabled, rule.range_expectation.strict_max_enabled, null)
35+
} : null
36+
regex_expectation = can(rule.regexExpectation) || can(rule.regex_expectation) ? {
37+
regex = try(rule.regexExpectation.regex, rule.regex_expectation.regex, null)
38+
} : null
39+
set_expectation = can(rule.setExpectation) || can(rule.set_expectation) ? {
40+
values = try(rule.setExpectation.values, rule.set_expectation.values, null)
41+
} : null
42+
uniqueness_expectation = try(rule.uniquenessExpectation, rule.uniqueness_expectation, null)
43+
statistic_range_expectation = can(rule.statisticRangeExpectation) || can(rule.statistic_range_expectation) ? {
44+
statistic = try(rule.statisticRangeExpectation.statistic, rule.statistic_range_expectation.statistic)
45+
min_value = try(rule.statisticRangeExpectation.minValue, rule.statistic_range_expectation.min_value, null)
46+
max_value = try(rule.statisticRangeExpectation.maxValue, rule.statistic_range_expectation.max_value, null)
47+
strict_min_enabled = try(rule.statisticRangeExpectation.strictMinEnabled, rule.statistic_range_expectation.strict_min_enabled, null)
48+
strict_max_enabled = try(rule.statisticRangeExpectation.strictMaxEnabled, rule.statistic_range_expectation.strict_max_enabled, null)
49+
} : null
50+
row_condition_expectation = can(rule.rowConditionExpectation) || can(rule.row_condition_expectation) ? {
51+
sql_expression = try(rule.rowConditionExpectation.sqlExpression, rule.row_condition_expectation.sql_expression, null)
52+
} : null
53+
table_condition_expectation = can(rule.tableConditionExpectation) || can(rule.table_condition_expectation) ? {
54+
sql_expression = try(rule.tableConditionExpectation.sqlExpression, rule.table_condition_expectation.sql_expression, null)
55+
} : null
7556
}
7657
]
7758
}
7859
data_quality_spec = (
7960
var.data_quality_spec != null || var.data_quality_spec_file != null ?
80-
merge(var.data_quality_spec, local._converted_data_quality_spec) :
61+
merge(var.data_quality_spec, local._file_data_quality_spec) :
8162
null
8263
)
64+
# data_quality_spec = (
65+
# var.data_quality_spec
66+
# )
8367
}
8468

8569
resource "google_dataplex_datascan" "datascan" {

‎modules/dataplex-autodq/variables.tf

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,10 +80,7 @@ variable "data_quality_spec" {
8080
variable "data_quality_spec_file" {
8181
description = "Path to a YAML file containing DataQualityScan related setting. Set `convert_camel_case` to true if your YAML file uses camelCase instead of snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec."
8282
default = null
83-
type = object({
84-
path = string
85-
convert_camel_case = optional(bool, false)
86-
})
83+
type = string
8784
}
8885

8986
variable "description" {

0 commit comments

Comments
 (0)