airbyte-ci: re-enable connector dependency upload on publish (#36962)

Author: alafanechere

airbyte-ci/connectors/pipelines/README.md

@@ -302,7 +302,7 @@ flowchart TD
 | `--code-tests-only`                                     | True     | False         | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. |
 | `--concurrent-cat`                                      | False    | False         | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits.                                                                                              |
 | `--<step-id>.<extra-parameter>=<extra-parameter-value>` | True     |               | You can pass extra parameters for specific test steps. More details in the extra parameters section below                                                                                                |
-| `--ci-requirements`                                     | False    |               |                                                                                                                                                                                                          | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. 
+| `--ci-requirements`                                     | False    |               |                                                                                                                                                                                                          | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use.
 
 Note:
 
@@ -649,6 +649,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 4.7.2   | [#36962](https://github.com/airbytehq/airbyte/pull/36962)  | Re-enable connector dependencies upload on publish.                                                                        |
 | 4.7.1   | [#36961](https://github.com/airbytehq/airbyte/pull/36961)  | Temporarily disable python connectors dependencies upload until we find a schema the data team can work with.              |
 | 4.7.0   | [#36892](https://github.com/airbytehq/airbyte/pull/36892)  | Upload Python connectors dependencies list to GCS on publish.                                                              |
 | 4.6.5   | [#36722](https://github.com/airbytehq/airbyte/pull/36527)  | Fix incorrect pipeline names                                                                                               |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py

@@ -84,7 +84,7 @@ class ConnectorDependenciesMetadata(BaseModel):
     connector_repository: str
     connector_version: str
     connector_definition_id: str
-    dependencies: Dict[str, str]
+    dependencies: List[Dict[str, str]]
     generation_time: datetime = datetime.utcnow()
 
 
@@ -100,7 +100,9 @@ async def _run(self, built_containers_per_platform: Dict[Platform, Container]) -
         ], "This step can only run for Python connectors."
         built_container = built_containers_per_platform[LOCAL_BUILD_PLATFORM]
         pip_freeze_output = await built_container.with_exec(["pip", "freeze"], skip_entrypoint=True).stdout()
-        dependencies = {line.split("==")[0]: line.split("==")[1] for line in pip_freeze_output.splitlines() if "==" in line}
+        dependencies = [
+            {"package_name": line.split("==")[0], "version": line.split("==")[1]} for line in pip_freeze_output.splitlines() if "==" in line
+        ]
         connector_technical_name = self.context.connector.technical_name
         connector_version = self.context.metadata["dockerImageTag"]
         dependencies_metadata = ConnectorDependenciesMetadata(
@@ -365,11 +367,9 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport:
             if build_connector_results.status is not StepStatus.SUCCESS:
                 return create_connector_report(results)
 
-            # Temporarily disabling it until we find the correct schema that the data team can ingest
-
-            # if context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]:
-            #     upload_dependencies_step = await UploadDependenciesToMetadataService(context).run(build_connector_results.output)
-            #     results.append(upload_dependencies_step)
+            if context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]:
+                upload_dependencies_step = await UploadDependenciesToMetadataService(context).run(build_connector_results.output)
+                results.append(upload_dependencies_step)
 
             built_connector_platform_variants = list(build_connector_results.output.values())
             push_connector_image_results = await PushConnectorImageToRegistry(context).run(built_connector_platform_variants)

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.7.1"
+version = "4.7.2"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]