Skip to content

Commit d5d4810

Browse files
hsheth2sid-acryl
hsheth2
and
sid-acryl
authored
fix(ingest/powerbi): change default for use_powerbi_email (datahub-project#11742)
Co-authored-by: sid-acryl <[email protected]>
1 parent fa97f66 commit d5d4810

15 files changed

+621
-618
lines changed

‎docs/how/updating-datahub.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
3434
`datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true.
3535

3636
- #11701: The Fivetran `sources_to_database` field is deprecated in favor of setting directly within `sources_to_platform_instance.<key>.database`.
37+
- #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information.
3738

3839
### Breaking Changes
3940

‎metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,9 @@ class OwnershipMapping(ConfigModel):
251251
default=True, description="Whether ingest PowerBI user as Datahub Corpuser"
252252
)
253253
use_powerbi_email: bool = pydantic.Field(
254-
default=False,
254+
# TODO: Deprecate and remove this config, since the non-email format
255+
# doesn't make much sense.
256+
default=True,
255257
description="Use PowerBI User email to ingest as corpuser, default is powerbi user identifier",
256258
)
257259
remove_email_suffix: bool = pydantic.Field(

‎metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[
22
{
33
"entityType": "corpuser",
4-
"entityUrn": "urn:li:corpuser:users.[email protected]",
4+
"entityUrn": "urn:li:corpuser:[email protected]",
55
"changeType": "UPSERT",
66
"aspectName": "corpUserKey",
77
"aspect": {
@@ -17,7 +17,7 @@
1717
},
1818
{
1919
"entityType": "corpuser",
20-
"entityUrn": "urn:li:corpuser:users.[email protected]",
20+
"entityUrn": "urn:li:corpuser:[email protected]",
2121
"changeType": "UPSERT",
2222
"aspectName": "corpUserKey",
2323
"aspect": {
@@ -402,11 +402,11 @@
402402
"json": {
403403
"owners": [
404404
{
405-
"owner": "urn:li:corpuser:users.[email protected]",
405+
"owner": "urn:li:corpuser:[email protected]",
406406
"type": "NONE"
407407
},
408408
{
409-
"owner": "urn:li:corpuser:users.[email protected]",
409+
"owner": "urn:li:corpuser:[email protected]",
410410
"type": "NONE"
411411
}
412412
],
@@ -445,7 +445,7 @@
445445
},
446446
{
447447
"entityType": "corpuser",
448-
"entityUrn": "urn:li:corpuser:users.[email protected]",
448+
"entityUrn": "urn:li:corpuser:[email protected]",
449449
"changeType": "UPSERT",
450450
"aspectName": "status",
451451
"aspect": {
@@ -461,7 +461,7 @@
461461
},
462462
{
463463
"entityType": "corpuser",
464-
"entityUrn": "urn:li:corpuser:users.[email protected]",
464+
"entityUrn": "urn:li:corpuser:[email protected]",
465465
"changeType": "UPSERT",
466466
"aspectName": "status",
467467
"aspect": {

‎metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -875,7 +875,7 @@
875875
},
876876
{
877877
"entityType": "corpuser",
878-
"entityUrn": "urn:li:corpuser:users.[email protected]",
878+
"entityUrn": "urn:li:corpuser:[email protected]",
879879
"changeType": "UPSERT",
880880
"aspectName": "corpUserKey",
881881
"aspect": {
@@ -891,7 +891,7 @@
891891
},
892892
{
893893
"entityType": "corpuser",
894-
"entityUrn": "urn:li:corpuser:users.[email protected]",
894+
"entityUrn": "urn:li:corpuser:[email protected]",
895895
"changeType": "UPSERT",
896896
"aspectName": "corpUserKey",
897897
"aspect": {
@@ -1302,11 +1302,11 @@
13021302
"json": {
13031303
"owners": [
13041304
{
1305-
"owner": "urn:li:corpuser:users.[email protected]",
1305+
"owner": "urn:li:corpuser:[email protected]",
13061306
"type": "NONE"
13071307
},
13081308
{
1309-
"owner": "urn:li:corpuser:users.[email protected]",
1309+
"owner": "urn:li:corpuser:[email protected]",
13101310
"type": "NONE"
13111311
}
13121312
],
@@ -1922,7 +1922,7 @@
19221922
},
19231923
{
19241924
"entityType": "corpuser",
1925-
"entityUrn": "urn:li:corpuser:users.[email protected]",
1925+
"entityUrn": "urn:li:corpuser:[email protected]",
19261926
"changeType": "UPSERT",
19271927
"aspectName": "corpUserKey",
19281928
"aspect": {
@@ -1938,7 +1938,7 @@
19381938
},
19391939
{
19401940
"entityType": "corpuser",
1941-
"entityUrn": "urn:li:corpuser:users.[email protected]",
1941+
"entityUrn": "urn:li:corpuser:[email protected]",
19421942
"changeType": "UPSERT",
19431943
"aspectName": "corpUserKey",
19441944
"aspect": {
@@ -2074,11 +2074,11 @@
20742074
"json": {
20752075
"owners": [
20762076
{
2077-
"owner": "urn:li:corpuser:users.[email protected]",
2077+
"owner": "urn:li:corpuser:[email protected]",
20782078
"type": "NONE"
20792079
},
20802080
{
2081-
"owner": "urn:li:corpuser:users.[email protected]",
2081+
"owner": "urn:li:corpuser:[email protected]",
20822082
"type": "NONE"
20832083
}
20842084
],
@@ -2117,7 +2117,7 @@
21172117
},
21182118
{
21192119
"entityType": "corpuser",
2120-
"entityUrn": "urn:li:corpuser:users.[email protected]",
2120+
"entityUrn": "urn:li:corpuser:[email protected]",
21212121
"changeType": "UPSERT",
21222122
"aspectName": "status",
21232123
"aspect": {
@@ -2133,7 +2133,7 @@
21332133
},
21342134
{
21352135
"entityType": "corpuser",
2136-
"entityUrn": "urn:li:corpuser:users.[email protected]",
2136+
"entityUrn": "urn:li:corpuser:[email protected]",
21372137
"changeType": "UPSERT",
21382138
"aspectName": "status",
21392139
"aspect": {

‎metadata-ingestion/tests/integration/powerbi/golden_test_cll.json

Lines changed: 66 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,6 @@
1717
"lastRunId": "no-run-id-provided"
1818
}
1919
},
20-
{
21-
"entityType": "corpuser",
22-
"entityUrn": "urn:li:corpuser:[email protected]",
23-
"changeType": "UPSERT",
24-
"aspectName": "corpUserKey",
25-
"aspect": {
26-
"json": {
27-
"username": "[email protected]"
28-
}
29-
},
30-
"systemMetadata": {
31-
"lastObserved": 1643871600000,
32-
"runId": "powerbi-test",
33-
"lastRunId": "no-run-id-provided"
34-
}
35-
},
3620
{
3721
"entityType": "dataset",
3822
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
@@ -71,22 +55,6 @@
7155
"lastRunId": "no-run-id-provided"
7256
}
7357
},
74-
{
75-
"entityType": "corpuser",
76-
"entityUrn": "urn:li:corpuser:[email protected]",
77-
"changeType": "UPSERT",
78-
"aspectName": "corpUserKey",
79-
"aspect": {
80-
"json": {
81-
"username": "[email protected]"
82-
}
83-
},
84-
"systemMetadata": {
85-
"lastObserved": 1643871600000,
86-
"runId": "powerbi-test",
87-
"lastRunId": "no-run-id-provided"
88-
}
89-
},
9058
{
9159
"entityType": "dataset",
9260
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
@@ -992,6 +960,38 @@
992960
"lastRunId": "no-run-id-provided"
993961
}
994962
},
963+
{
964+
"entityType": "corpuser",
965+
"entityUrn": "urn:li:corpuser:[email protected]",
966+
"changeType": "UPSERT",
967+
"aspectName": "corpUserKey",
968+
"aspect": {
969+
"json": {
970+
"username": "[email protected]"
971+
}
972+
},
973+
"systemMetadata": {
974+
"lastObserved": 1643871600000,
975+
"runId": "powerbi-test",
976+
"lastRunId": "no-run-id-provided"
977+
}
978+
},
979+
{
980+
"entityType": "corpuser",
981+
"entityUrn": "urn:li:corpuser:[email protected]",
982+
"changeType": "UPSERT",
983+
"aspectName": "corpUserKey",
984+
"aspect": {
985+
"json": {
986+
"username": "[email protected]"
987+
}
988+
},
989+
"systemMetadata": {
990+
"lastObserved": 1643871600000,
991+
"runId": "powerbi-test",
992+
"lastRunId": "no-run-id-provided"
993+
}
994+
},
995995
{
996996
"entityType": "dashboard",
997997
"entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
@@ -1159,11 +1159,11 @@
11591159
"json": {
11601160
"owners": [
11611161
{
1162-
"owner": "urn:li:corpuser:users.[email protected]",
1162+
"owner": "urn:li:corpuser:[email protected]",
11631163
"type": "NONE"
11641164
},
11651165
{
1166-
"owner": "urn:li:corpuser:users.[email protected]",
1166+
"owner": "urn:li:corpuser:[email protected]",
11671167
"type": "NONE"
11681168
}
11691169
],
@@ -1200,38 +1200,6 @@
12001200
"lastRunId": "no-run-id-provided"
12011201
}
12021202
},
1203-
{
1204-
"entityType": "corpuser",
1205-
"entityUrn": "urn:li:corpuser:[email protected]",
1206-
"changeType": "UPSERT",
1207-
"aspectName": "status",
1208-
"aspect": {
1209-
"json": {
1210-
"removed": false
1211-
}
1212-
},
1213-
"systemMetadata": {
1214-
"lastObserved": 1643871600000,
1215-
"runId": "powerbi-test",
1216-
"lastRunId": "no-run-id-provided"
1217-
}
1218-
},
1219-
{
1220-
"entityType": "corpuser",
1221-
"entityUrn": "urn:li:corpuser:[email protected]",
1222-
"changeType": "UPSERT",
1223-
"aspectName": "status",
1224-
"aspect": {
1225-
"json": {
1226-
"removed": false
1227-
}
1228-
},
1229-
"systemMetadata": {
1230-
"lastObserved": 1643871600000,
1231-
"runId": "powerbi-test",
1232-
"lastRunId": "no-run-id-provided"
1233-
}
1234-
},
12351203
{
12361204
"entityType": "dataset",
12371205
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
@@ -1503,5 +1471,37 @@
15031471
"runId": "powerbi-test",
15041472
"lastRunId": "no-run-id-provided"
15051473
}
1474+
},
1475+
{
1476+
"entityType": "corpuser",
1477+
"entityUrn": "urn:li:corpuser:[email protected]",
1478+
"changeType": "UPSERT",
1479+
"aspectName": "status",
1480+
"aspect": {
1481+
"json": {
1482+
"removed": false
1483+
}
1484+
},
1485+
"systemMetadata": {
1486+
"lastObserved": 1643871600000,
1487+
"runId": "powerbi-test",
1488+
"lastRunId": "no-run-id-provided"
1489+
}
1490+
},
1491+
{
1492+
"entityType": "corpuser",
1493+
"entityUrn": "urn:li:corpuser:[email protected]",
1494+
"changeType": "UPSERT",
1495+
"aspectName": "status",
1496+
"aspect": {
1497+
"json": {
1498+
"removed": false
1499+
}
1500+
},
1501+
"systemMetadata": {
1502+
"lastObserved": 1643871600000,
1503+
"runId": "powerbi-test",
1504+
"lastRunId": "no-run-id-provided"
1505+
}
15061506
}
15071507
]

0 commit comments

Comments
 (0)