Skip to content

Commit d38427a

Browse files
mayurinehatemayurinehate
authored
feat(ingest/snowflake): secure view lineage without owner permissions (datahub-project#12123)
1 parent d2ffeb0 commit d38427a

File tree

6 files changed

+116
-16
lines changed

6 files changed

+116
-16
lines changed

‎metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,19 @@ def show_views_for_database(
237237
LIMIT {limit} {from_clause};
238238
"""
239239

240+
@staticmethod
241+
def get_secure_view_definitions() -> str:
242+
# https://docs.snowflake.com/en/sql-reference/account-usage/views
243+
return """
244+
SELECT
245+
TABLE_CATALOG as "TABLE_CATALOG",
246+
TABLE_SCHEMA as "TABLE_SCHEMA",
247+
TABLE_NAME as "TABLE_NAME",
248+
VIEW_DEFINITION as "VIEW_DEFINITION"
249+
FROM SNOWFLAKE.ACCOUNT_USAGE.VIEWS
250+
WHERE IS_SECURE = 'YES' AND VIEW_DEFINITION !='' AND DELETED IS NULL
251+
"""
252+
240253
@staticmethod
241254
def columns_for_schema(
242255
schema_name: str,

‎metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,22 @@ def get_schemas_for_database(self, db_name: str) -> List[SnowflakeSchema]:
266266
snowflake_schemas.append(snowflake_schema)
267267
return snowflake_schemas
268268

269+
@serialized_lru_cache(maxsize=1)
270+
def get_secure_view_definitions(self) -> Dict[str, Dict[str, Dict[str, str]]]:
271+
secure_view_definitions: Dict[str, Dict[str, Dict[str, str]]] = defaultdict(
272+
lambda: defaultdict(lambda: defaultdict())
273+
)
274+
cur = self.connection.query(SnowflakeQuery.get_secure_view_definitions())
275+
for view in cur:
276+
db_name = view["TABLE_CATALOG"]
277+
schema_name = view["TABLE_SCHEMA"]
278+
view_name = view["TABLE_NAME"]
279+
secure_view_definitions[db_name][schema_name][view_name] = view[
280+
"VIEW_DEFINITION"
281+
]
282+
283+
return secure_view_definitions
284+
269285
@serialized_lru_cache(maxsize=1)
270286
def get_tables_for_database(
271287
self, db_name: str

‎metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,10 @@ def _process_schema(
424424
view_identifier = self.identifiers.get_dataset_identifier(
425425
view.name, schema_name, db_name
426426
)
427+
if view.is_secure and not view.view_definition:
428+
view.view_definition = self.fetch_secure_view_definition(
429+
view.name, schema_name, db_name
430+
)
427431
if view.view_definition:
428432
self.aggregator.add_view_definition(
429433
view_urn=self.identifiers.gen_dataset_urn(view_identifier),
@@ -449,6 +453,25 @@ def _process_schema(
449453
context=f"{db_name}.{schema_name}",
450454
)
451455

456+
def fetch_secure_view_definition(
457+
self, table_name: str, schema_name: str, db_name: str
458+
) -> Optional[str]:
459+
try:
460+
view_definitions = self.data_dictionary.get_secure_view_definitions()
461+
return view_definitions[db_name][schema_name][table_name]
462+
except Exception as e:
463+
if isinstance(e, SnowflakePermissionError):
464+
error_msg = (
465+
"Failed to get secure views definitions. Please check permissions."
466+
)
467+
else:
468+
error_msg = "Failed to get secure views definitions"
469+
self.structured_reporter.warning(
470+
error_msg,
471+
exc=e,
472+
)
473+
return None
474+
452475
def fetch_views_for_schema(
453476
self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str
454477
) -> List[SnowflakeView]:

‎metadata-ingestion/tests/integration/snowflake/common.py

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414
NUM_OPS = 10
1515
NUM_USAGE = 0
1616

17+
18+
def is_secure(view_idx):
19+
return view_idx == 1
20+
21+
1722
FROZEN_TIME = "2022-06-07 17:00:00"
1823
large_sql_query = """WITH object_access_history AS
1924
(
@@ -247,9 +252,25 @@ def default_query_results( # noqa: C901
247252
"name": f"VIEW_{view_idx}",
248253
"created_on": datetime(2021, 6, 8, 0, 0, 0, 0),
249254
"comment": "Comment for View",
250-
"text": f"create view view_{view_idx} as select * from table_{view_idx}",
255+
"is_secure": "true" if is_secure(view_idx) else "false",
256+
"text": (
257+
f"create view view_{view_idx} as select * from table_{view_idx}"
258+
if not is_secure(view_idx)
259+
else None
260+
),
261+
}
262+
for view_idx in range(1, num_views + 1)
263+
]
264+
elif query == SnowflakeQuery.get_secure_view_definitions():
265+
return [
266+
{
267+
"TABLE_CATALOG": "TEST_DB",
268+
"TABLE_SCHEMA": "TEST_SCHEMA",
269+
"TABLE_NAME": f"VIEW_{view_idx}",
270+
"VIEW_DEFINITION": f"create view view_{view_idx} as select * from table_{view_idx}",
251271
}
252272
for view_idx in range(1, num_views + 1)
273+
if is_secure(view_idx)
253274
]
254275
elif query == SnowflakeQuery.columns_for_schema("TEST_SCHEMA", "TEST_DB"):
255276
return [

‎metadata-ingestion/tests/integration/snowflake/snowflake_golden.json

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,9 @@
490490
"aspectName": "datasetProperties",
491491
"aspect": {
492492
"json": {
493-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
493+
"customProperties": {
494+
"CLUSTERING_KEY": "LINEAR(COL_1)"
495+
},
494496
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
495497
"name": "TABLE_1",
496498
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
@@ -789,7 +791,9 @@
789791
"aspectName": "datasetProperties",
790792
"aspect": {
791793
"json": {
792-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
794+
"customProperties": {
795+
"CLUSTERING_KEY": "LINEAR(COL_1)"
796+
},
793797
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
794798
"name": "TABLE_2",
795799
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
@@ -1088,7 +1092,9 @@
10881092
"aspectName": "datasetProperties",
10891093
"aspect": {
10901094
"json": {
1091-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
1095+
"customProperties": {
1096+
"CLUSTERING_KEY": "LINEAR(COL_1)"
1097+
},
10921098
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
10931099
"name": "TABLE_3",
10941100
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
@@ -1387,7 +1393,9 @@
13871393
"aspectName": "datasetProperties",
13881394
"aspect": {
13891395
"json": {
1390-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
1396+
"customProperties": {
1397+
"CLUSTERING_KEY": "LINEAR(COL_1)"
1398+
},
13911399
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
13921400
"name": "TABLE_4",
13931401
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
@@ -1686,7 +1694,9 @@
16861694
"aspectName": "datasetProperties",
16871695
"aspect": {
16881696
"json": {
1689-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
1697+
"customProperties": {
1698+
"CLUSTERING_KEY": "LINEAR(COL_1)"
1699+
},
16901700
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/",
16911701
"name": "TABLE_5",
16921702
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5",
@@ -1985,7 +1995,9 @@
19851995
"aspectName": "datasetProperties",
19861996
"aspect": {
19871997
"json": {
1988-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
1998+
"customProperties": {
1999+
"CLUSTERING_KEY": "LINEAR(COL_1)"
2000+
},
19892001
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
19902002
"name": "TABLE_6",
19912003
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
@@ -2284,7 +2296,9 @@
22842296
"aspectName": "datasetProperties",
22852297
"aspect": {
22862298
"json": {
2287-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
2299+
"customProperties": {
2300+
"CLUSTERING_KEY": "LINEAR(COL_1)"
2301+
},
22882302
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
22892303
"name": "TABLE_7",
22902304
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
@@ -2583,7 +2597,9 @@
25832597
"aspectName": "datasetProperties",
25842598
"aspect": {
25852599
"json": {
2586-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
2600+
"customProperties": {
2601+
"CLUSTERING_KEY": "LINEAR(COL_1)"
2602+
},
25872603
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
25882604
"name": "TABLE_8",
25892605
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
@@ -2882,7 +2898,9 @@
28822898
"aspectName": "datasetProperties",
28832899
"aspect": {
28842900
"json": {
2885-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
2901+
"customProperties": {
2902+
"CLUSTERING_KEY": "LINEAR(COL_1)"
2903+
},
28862904
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
28872905
"name": "TABLE_9",
28882906
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
@@ -3181,7 +3199,9 @@
31813199
"aspectName": "datasetProperties",
31823200
"aspect": {
31833201
"json": {
3184-
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
3202+
"customProperties": {
3203+
"CLUSTERING_KEY": "LINEAR(COL_1)"
3204+
},
31853205
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
31863206
"name": "TABLE_10",
31873207
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
@@ -3471,23 +3491,25 @@
34713491
"aspectName": "datasetProperties",
34723492
"aspect": {
34733493
"json": {
3474-
"customProperties": {},
3494+
"customProperties": {
3495+
"IS_SECURE": "true"
3496+
},
34753497
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/",
34763498
"name": "VIEW_1",
34773499
"qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1",
34783500
"description": "Comment for View",
34793501
"created": {
3480-
"time": 1623103200000
3502+
"time": 1623090600000
34813503
},
34823504
"lastModified": {
3483-
"time": 1623103200000
3505+
"time": 1623090600000
34843506
},
34853507
"tags": []
34863508
}
34873509
},
34883510
"systemMetadata": {
34893511
"lastObserved": 1615443388097,
3490-
"runId": "snowflake-2023_12_18-10_16_09",
3512+
"runId": "snowflake-2024_12_16-15_30_20-649nax",
34913513
"lastRunId": "no-run-id-provided"
34923514
}
34933515
},

‎metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -621,12 +621,17 @@
621621
"op": "add",
622622
"path": "/qualifiedName",
623623
"value": "TEST_DB.TEST_SCHEMA.VIEW_1"
624+
},
625+
{
626+
"op": "add",
627+
"path": "/customProperties/IS_SECURE",
628+
"value": "true"
624629
}
625630
]
626631
},
627632
"systemMetadata": {
628633
"lastObserved": 1654621200000,
629-
"runId": "snowflake-2022_06_07-17_00_00-ad3hnf",
634+
"runId": "snowflake-2022_06_07-17_00_00-ivthci",
630635
"lastRunId": "no-run-id-provided"
631636
}
632637
},

0 commit comments

Comments
 (0)