Skip to content

Commit f029414

Browse files
bnchrchbnchrch
authored
Add provinance to the registry entry (#37804)
## What Adds the generation time and root metadata file info to the registry entry ![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/PTsI7qAmiIMkhFQg04QF/4aa5ca8b-011d-4e6f-90a2-081feae7756c.png) Spun out of #32715 as a stack closes airbytehq/airbyte-internal-issues#7466
1 parent 2dd1140 commit f029414

23 files changed

+330
-160
lines changed

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py

+13-16
Original file line numberDiff line numberDiff line change
@@ -108,22 +108,18 @@ class GitInfo(BaseModel):
108108
class Config:
109109
extra = Extra.forbid
110110

111-
commit_sha: Optional[str] = Field(
112-
None,
113-
description="The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
114-
)
115-
commit_timestamp: Optional[datetime] = Field(
116-
None,
117-
description="The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
118-
)
119-
commit_author: Optional[str] = Field(
120-
None,
121-
description="The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
122-
)
123-
commit_author_email: Optional[str] = Field(
124-
None,
125-
description="The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
126-
)
111+
commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
112+
commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
113+
commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
114+
commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
115+
116+
117+
class SourceFileInfo(BaseModel):
118+
metadata_etag: Optional[str] = None
119+
metadata_file_path: Optional[str] = None
120+
metadata_bucket_name: Optional[str] = None
121+
metadata_last_modified: Optional[str] = None
122+
registry_entry_generated_at: Optional[str] = None
127123

128124

129125
class JobTypeResourceLimit(BaseModel):
@@ -147,6 +143,7 @@ class Config:
147143

148144
class GeneratedFields(BaseModel):
149145
git: Optional[GitInfo] = None
146+
source_file_info: Optional[SourceFileInfo] = None
150147

151148

152149
class ActorDefinitionResourceRequirements(BaseModel):

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py

+25-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
from __future__ import annotations
55

6-
from datetime import date
6+
from datetime import date, datetime
77
from typing import Any, Dict, List, Optional
88
from uuid import UUID
99

@@ -79,6 +79,24 @@ class Config:
7979
ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
8080

8181

82+
class GitInfo(BaseModel):
83+
class Config:
84+
extra = Extra.forbid
85+
86+
commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
87+
commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
88+
commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
89+
commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
90+
91+
92+
class SourceFileInfo(BaseModel):
93+
metadata_etag: Optional[str] = None
94+
metadata_file_path: Optional[str] = None
95+
metadata_bucket_name: Optional[str] = None
96+
metadata_last_modified: Optional[str] = None
97+
registry_entry_generated_at: Optional[str] = None
98+
99+
82100
class JobTypeResourceLimit(BaseModel):
83101
class Config:
84102
extra = Extra.forbid
@@ -91,6 +109,11 @@ class BreakingChangeScope(BaseModel):
91109
__root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")
92110

93111

112+
class GeneratedFields(BaseModel):
113+
git: Optional[GitInfo] = None
114+
source_file_info: Optional[SourceFileInfo] = None
115+
116+
94117
class ActorDefinitionResourceRequirements(BaseModel):
95118
class Config:
96119
extra = Extra.forbid
@@ -172,3 +195,4 @@ class Config:
172195
releases: Optional[ConnectorReleases] = None
173196
ab_internal: Optional[AirbyteInternal] = None
174197
supportsRefreshes: Optional[bool] = False
198+
generated: Optional[GeneratedFields] = None

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py

+25-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
from __future__ import annotations
55

6-
from datetime import date
6+
from datetime import date, datetime
77
from typing import Any, Dict, List, Optional
88
from uuid import UUID
99

@@ -75,6 +75,24 @@ class Config:
7575
ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
7676

7777

78+
class GitInfo(BaseModel):
79+
class Config:
80+
extra = Extra.forbid
81+
82+
commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
83+
commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
84+
commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
85+
commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
86+
87+
88+
class SourceFileInfo(BaseModel):
89+
metadata_etag: Optional[str] = None
90+
metadata_file_path: Optional[str] = None
91+
metadata_bucket_name: Optional[str] = None
92+
metadata_last_modified: Optional[str] = None
93+
registry_entry_generated_at: Optional[str] = None
94+
95+
7896
class JobTypeResourceLimit(BaseModel):
7997
class Config:
8098
extra = Extra.forbid
@@ -87,6 +105,11 @@ class BreakingChangeScope(BaseModel):
87105
__root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")
88106

89107

108+
class GeneratedFields(BaseModel):
109+
git: Optional[GitInfo] = None
110+
source_file_info: Optional[SourceFileInfo] = None
111+
112+
90113
class ActorDefinitionResourceRequirements(BaseModel):
91114
class Config:
92115
extra = Extra.forbid
@@ -164,3 +187,4 @@ class Config:
164187
)
165188
releases: Optional[ConnectorReleases] = None
166189
ab_internal: Optional[AirbyteInternal] = None
190+
generated: Optional[GeneratedFields] = None

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py

+26-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
from __future__ import annotations
55

6-
from datetime import date
6+
from datetime import date, datetime
77
from typing import Any, Dict, List, Optional
88
from uuid import UUID
99

@@ -79,6 +79,24 @@ class Config:
7979
ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
8080

8181

82+
class GitInfo(BaseModel):
83+
class Config:
84+
extra = Extra.forbid
85+
86+
commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
87+
commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
88+
commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
89+
commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
90+
91+
92+
class SourceFileInfo(BaseModel):
93+
metadata_etag: Optional[str] = None
94+
metadata_file_path: Optional[str] = None
95+
metadata_bucket_name: Optional[str] = None
96+
metadata_last_modified: Optional[str] = None
97+
registry_entry_generated_at: Optional[str] = None
98+
99+
82100
class SuggestedStreams(BaseModel):
83101
class Config:
84102
extra = Extra.allow
@@ -101,6 +119,11 @@ class BreakingChangeScope(BaseModel):
101119
__root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")
102120

103121

122+
class GeneratedFields(BaseModel):
123+
git: Optional[GitInfo] = None
124+
source_file_info: Optional[SourceFileInfo] = None
125+
126+
104127
class ActorDefinitionResourceRequirements(BaseModel):
105128
class Config:
106129
extra = Extra.forbid
@@ -178,6 +201,7 @@ class Config:
178201
)
179202
releases: Optional[ConnectorReleases] = None
180203
ab_internal: Optional[AirbyteInternal] = None
204+
generated: Optional[GeneratedFields] = None
181205

182206

183207
class ConnectorRegistryDestinationDefinition(BaseModel):
@@ -214,6 +238,7 @@ class Config:
214238
releases: Optional[ConnectorReleases] = None
215239
ab_internal: Optional[AirbyteInternal] = None
216240
supportsRefreshes: Optional[bool] = False
241+
generated: Optional[GeneratedFields] = None
217242

218243

219244
class ConnectorRegistryV0(BaseModel):

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/GeneratedFields.py

+13-16
Original file line numberDiff line numberDiff line change
@@ -13,23 +13,20 @@ class GitInfo(BaseModel):
1313
class Config:
1414
extra = Extra.forbid
1515

16-
commit_sha: Optional[str] = Field(
17-
None,
18-
description="The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
19-
)
20-
commit_timestamp: Optional[datetime] = Field(
21-
None,
22-
description="The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
23-
)
24-
commit_author: Optional[str] = Field(
25-
None,
26-
description="The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
27-
)
28-
commit_author_email: Optional[str] = Field(
29-
None,
30-
description="The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
31-
)
16+
commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
17+
commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
18+
commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
19+
commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
20+
21+
22+
class SourceFileInfo(BaseModel):
23+
metadata_etag: Optional[str] = None
24+
metadata_file_path: Optional[str] = None
25+
metadata_bucket_name: Optional[str] = None
26+
metadata_last_modified: Optional[str] = None
27+
registry_entry_generated_at: Optional[str] = None
3228

3329

3430
class GeneratedFields(BaseModel):
3531
git: Optional[GitInfo] = None
32+
source_file_info: Optional[SourceFileInfo] = None

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/GitInfo.py

+4-16
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,7 @@ class GitInfo(BaseModel):
1313
class Config:
1414
extra = Extra.forbid
1515

16-
commit_sha: Optional[str] = Field(
17-
None,
18-
description="The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
19-
)
20-
commit_timestamp: Optional[datetime] = Field(
21-
None,
22-
description="The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
23-
)
24-
commit_author: Optional[str] = Field(
25-
None,
26-
description="The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
27-
)
28-
commit_author_email: Optional[str] = Field(
29-
None,
30-
description="The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
31-
)
16+
commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
17+
commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
18+
commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
19+
commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SourceFileInfo.py

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# generated by datamodel-codegen:
2+
# filename: SourceFileInfo.yaml
3+
4+
from __future__ import annotations
5+
6+
from typing import Optional
7+
8+
from pydantic import BaseModel
9+
10+
11+
class SourceFileInfo(BaseModel):
12+
metadata_etag: Optional[str] = None
13+
metadata_file_path: Optional[str] = None
14+
metadata_bucket_name: Optional[str] = None
15+
metadata_last_modified: Optional[str] = None
16+
registry_entry_generated_at: Optional[str] = None

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,6 @@
1616
from .ReleaseStage import *
1717
from .RemoteRegistries import *
1818
from .ResourceRequirements import *
19+
from .SourceFileInfo import *
1920
from .SuggestedStreams import *
2021
from .SupportLevel import *

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,5 @@ properties:
7575
supportsRefreshes:
7676
type: boolean
7777
default: false
78+
generated:
79+
"$ref": GeneratedFields.yaml

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,5 @@ properties:
7474
"$ref": ConnectorReleases.yaml
7575
ab_internal:
7676
"$ref": AirbyteInternal.yaml
77+
generated:
78+
"$ref": GeneratedFields.yaml

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GeneratedFields.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,5 @@ type: object
77
properties:
88
git:
99
"$ref": GitInfo.yaml
10+
source_file_info:
11+
"$ref": SourceFileInfo.yaml

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,20 @@
22
"$schema": http://json-schema.org/draft-07/schema#
33
"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml
44
title: GitInfo
5-
description: Information about the author of the last commit that modified this file
5+
description: Information about the author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
66
type: object
77
additionalProperties: false
88
properties:
99
commit_sha:
1010
type: string
11-
description: The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
11+
description: The git commit sha of the last commit that modified this file.
1212
commit_timestamp:
1313
type: string
1414
format: date-time
15-
description: The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
15+
description: The git commit timestamp of the last commit that modified this file.
1616
commit_author:
1717
type: string
18-
description: The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
18+
description: The git commit author of the last commit that modified this file.
1919
commit_author_email:
2020
type: string
21-
description: The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
21+
description: The git commit author email of the last commit that modified this file.

‎airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SourceFileInfo.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
"$schema": http://json-schema.org/draft-07/schema#
3+
"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SourceFileInfo.yaml
4+
title: SourceFileInfo
5+
description: Information about the source file that generated the registry entry
6+
type: object
7+
properties:
8+
metadata_etag:
9+
type: string
10+
metadata_file_path:
11+
type: string
12+
metadata_bucket_name:
13+
type: string
14+
metadata_last_modified:
15+
type: string
16+
registry_entry_generated_at:
17+
type: string

‎airbyte-ci/connectors/metadata_service/lib/pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "metadata-service"
3-
version = "0.4.0"
3+
version = "0.5.0"
44
description = ""
55
authors = ["Ben Church <[email protected]>"]
66
readme = "README.md"

‎airbyte-ci/connectors/metadata_service/orchestrator/.env.template

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ NIGHTLY_REPORT_CHANNEL=""
55
# METADATA_CDN_BASE_URL="https://connectors.airbyte.com/files"
66
DOCKER_HUB_USERNAME=""
77
DOCKER_HUB_PASSWORD=""
8+
89
SLACK_TOKEN = ""
9-
PUBLISH_UPDATE_CHANNEL="#ben-test"
10+
SLACK_NOTIFICATIONS_DISABLED = "true"
11+
PUBLISH_UPDATE_CHANNEL="#test-ci-slack-intergrations"
1012
# SENTRY_DSN=""
1113
# SENTRY_ENVIRONMENT="dev"
1214
# SENTRY_TRACES_SAMPLE_RATE=1.0

‎airbyte-ci/connectors/metadata_service/orchestrator/README.md

+9
Original file line numberDiff line numberDiff line change
@@ -169,3 +169,12 @@ open http://localhost:3000
169169
```
170170

171171
And run the `generate_registry` job
172+
173+
## Additional Notes
174+
175+
### Testing Slack Notifications
176+
You will need to add the following environment variables to your `.env` file:
177+
178+
- `SLACK_TOKEN`: Set to an OAuth token for the [Connector Ops Dagster Bot](https://airbytehq-team.slack.com/apps/A05K845HBE0-connector-ops-dagster-bot?settings=1)
179+
- `PUBLISH_UPDATE_CHANNEL`: Set to `#test-ci-slack-intergrations`
180+
- `SLACK_NOTIFICATIONS_DISABLED`: Set to `False`

‎airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py

+4
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,10 @@ def metadata_definitions(context: OpExecutionContext) -> List[LatestMetadataEntr
130130
metadata_entry = LatestMetadataEntry(
131131
metadata_definition=metadata_def,
132132
icon_url=icon_url,
133+
last_modified=blob.last_modified,
134+
etag=blob.etag,
135+
file_path=blob.name,
136+
bucket_name=blob.bucket.name,
133137
)
134138
metadata_entries.append(metadata_entry)
135139

0 commit comments

Comments
 (0)