feat(airbyte-ci): include components.py in manifest-only build (#44879)

Author: ChristoGrab

airbyte-ci/connectors/connector_ops/README.md

@@ -37,6 +37,7 @@ poetry run pytest
 ```
 
 ## Changelog
+- 0.9.0: Add components path attribute for manifest-only connectors.
 - 0.8.1: Gradle dependency discovery logic supports the Bulk CDK.
 - 0.8.0: Add a `sbom_url` property to `Connector`
 - 0.7.0: Added required reviewers for manifest-only connector changes/additions.

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -60,6 +60,7 @@ def download_catalog(catalog_url):
 
 OSS_CATALOG = download_catalog(OSS_CATALOG_URL)
 MANIFEST_FILE_NAME = "manifest.yaml"
+COMPONENTS_FILE_NAME = "components.py"
 DOCKERFILE_FILE_NAME = "Dockerfile"
 PYPROJECT_FILE_NAME = "pyproject.toml"
 ICON_FILE_NAME = "icon.svg"
@@ -347,6 +348,11 @@ def manifest_path(self) -> Path:
 
         return self._manifest_low_code_path
 
+    @property
+    def manifest_only_components_path(self) -> Path:
+        """Return the path to the components.py file of a manifest-only connector."""
+        return self.code_directory / COMPONENTS_FILE_NAME
+
     @property
     def has_dockerfile(self) -> bool:
         return self.dockerfile_file_path.is_file()

airbyte-ci/connectors/connector_ops/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "connector_ops"
-version = "0.8.1"
+version = "0.9.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-ci/connectors/connector_ops/tests/test_required_reviewer_checks.py

@@ -23,7 +23,7 @@ def mock_diffed_branched(mocker):
 
 @pytest.fixture
 def pokeapi_metadata_path():
-    return "airbyte-integrations/connectors/source-pokeapi/metadata.yaml"
+    return "airbyte-integrations/connectors/source-zoho-crm/metadata.yaml"
 
 
 @pytest.fixture
@@ -37,7 +37,7 @@ def not_tracked_change_expected_team(tmp_path, pokeapi_metadata_path):
     backup_path = tmp_path / "non_strategic_acceptance_test_config.backup"
     shutil.copyfile(pokeapi_metadata_path, backup_path)
     with open(pokeapi_metadata_path, "a") as metadata_file:
-        metadata_file.write("not_tracked")
+        metadata_file.write("\nnot_tracked: true\n")
     yield expected_teams
     shutil.copyfile(backup_path, pokeapi_metadata_path)
 

airbyte-ci/connectors/pipelines/README.md

@@ -843,6 +843,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------|
+| 4.35.0  | [#44879](https://github.com/airbytehq/airbyte/pull/44879)  | Mount `components.py` when building manifest-only connector image                                                            |
 | 4.34.2  | [#44786](https://github.com/airbytehq/airbyte/pull/44786)  | Pre-emptively skip archived connectors when searching for modified files                                                     |
 | 4.34.1  | [#44557](https://github.com/airbytehq/airbyte/pull/44557)  | Conditionally propagate parameters in manifest-only migration                                                                |
 | 4.34.0  | [#44551](https://github.com/airbytehq/airbyte/pull/44551)  | `connectors publish` do not push the `latest` tag when the current version is a release candidate.                           |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py

@@ -9,7 +9,7 @@
 from pipelines.airbyte_ci.connectors.build_image.steps import build_customization
 from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.consts import MANIFEST_FILE_PATH
+from pipelines.consts import COMPONENTS_FILE_PATH, MANIFEST_FILE_PATH
 from pipelines.models.steps import StepResult
 from pydash.objects import get  # type: ignore
 
@@ -43,11 +43,20 @@ async def _build_from_base_image(self, platform: Platform) -> Container:
         """
         self.logger.info(f"Building connector from base image in metadata for {platform}")
 
+        # Mount manifest file
         base_container = self._get_base_container(platform).with_file(
             f"source_declarative_manifest/{MANIFEST_FILE_PATH}",
             (await self.context.get_connector_dir(include=[MANIFEST_FILE_PATH])).file(MANIFEST_FILE_PATH),
         )
 
+        # Mount components file if it exists
+        components_file = self.context.connector.manifest_only_components_path
+        if components_file.exists():
+            base_container = base_container.with_file(
+                f"source_declarative_manifest/{COMPONENTS_FILE_PATH}",
+                (await self.context.get_connector_dir(include=[COMPONENTS_FILE_PATH])).file(COMPONENTS_FILE_PATH),
+            )
+
         connector_container = build_customization.apply_airbyte_entrypoint(base_container, self.context.connector)
         return connector_container
 

airbyte-ci/connectors/pipelines/pipelines/consts.py

@@ -10,6 +10,7 @@
 
 PYPROJECT_TOML_FILE_PATH = "pyproject.toml"
 MANIFEST_FILE_PATH = "manifest.yaml"
+COMPONENTS_FILE_PATH = "components.py"
 LICENSE_SHORT_FILE_PATH = "LICENSE_SHORT"
 CONNECTOR_TESTING_REQUIREMENTS = [
     "pip==21.3.1",

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.34.2"
+version = "4.35.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py

@@ -35,10 +35,22 @@ def test_context_with_connector_with_base_image(self, test_context):
         }
         return test_context
 
+    @pytest.fixture
+    def mock_connector_directory(self, mocker):
+        mock_components_file = mocker.Mock()
+        mock_connector_dir = mocker.Mock()
+        mock_connector_dir.file.return_value = mock_components_file
+        return mock_connector_dir, mock_components_file
+
+    def _assert_file_not_handled(self, container_mock, file_path):
+        """Assert that a specified file_path was not handled by the container_mock"""
+        assert not any(file_path in call.args[0] for call in container_mock.with_file.call_args_list)
+
     async def test__run_using_base_image_with_mocks(self, mocker, test_context_with_connector_with_base_image, all_platforms):
         container_built_from_base = mock_container()
         container_built_from_base.with_label.return_value = container_built_from_base
 
+        mocker.patch.object(Path, "exists", return_value=True)  # Mock Path.exists() to always return True
         mocker.patch.object(
             manifest_only_connectors.BuildConnectorImages,
             "_build_from_base_image",
@@ -59,3 +71,39 @@ async def test__run_using_base_image_with_mocks(self, mocker, test_context_with_
         assert step_result.status is StepStatus.SUCCESS
         for platform in all_platforms:
             assert step_result.output[platform] == container_built_from_base
+
+    @pytest.mark.parametrize("components_file_exists", [True, False])
+    async def test__run_using_base_image_with_components_file(
+        self, mocker, all_platforms, test_context_with_connector_with_base_image, mock_connector_directory, components_file_exists
+    ):
+        mock_connector_dir, mock_components_file = mock_connector_directory
+        container_built_from_base = mock_container()
+
+        container_built_from_base.with_label.return_value = container_built_from_base
+        container_built_from_base.with_file.return_value = container_built_from_base
+
+        test_context_with_connector_with_base_image.get_connector_dir = mocker.AsyncMock(return_value=mock_connector_dir)
+        test_context_with_connector_with_base_image.connector.manifest_only_components_path.exists = mocker.Mock(
+            return_value=components_file_exists
+        )
+
+        mocker.patch.object(
+            manifest_only_connectors.BuildConnectorImages,
+            "_get_base_container",
+            return_value=container_built_from_base,
+        )
+
+        mocker.patch.object(
+            build_customization,
+            "apply_airbyte_entrypoint",
+            return_value=container_built_from_base,
+        )
+
+        step = manifest_only_connectors.BuildConnectorImages(test_context_with_connector_with_base_image)
+
+        await step._build_connector(all_platforms[0], container_built_from_base)
+        if components_file_exists:
+            container_built_from_base.with_file.assert_any_call("source_declarative_manifest/components.py", mock_components_file)
+            mock_connector_dir.file.assert_any_call("components.py")
+        else:
+            self._assert_file_not_handled(container_built_from_base, "source_declarative_manifest/components.py")

airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py

@@ -183,8 +183,7 @@ async def test_check_path_in_workdir(dagger_client):
         .with_workdir(str(connector.code_directory))
     )
     assert await utils.check_path_in_workdir(container, "metadata.yaml")
-    assert await utils.check_path_in_workdir(container, "pyproject.toml")
-    assert await utils.check_path_in_workdir(container, "poetry.lock")
+    assert await utils.check_path_in_workdir(container, "manifest.yaml")
     assert await utils.check_path_in_workdir(container, "not_existing_file") is False
 
 

airbyte-ci/connectors/pipelines/tests/utils.py

@@ -19,6 +19,9 @@ def with_label(self, *args, **kwargs):
     async def with_exec(self, *args, **kwargs):
         return self
 
+    def with_file(self, *args, **kwargs):
+        return self
+
 
 def pick_a_random_connector(
     language: ConnectorLanguage = None,
@@ -55,4 +58,5 @@ def mock_container():
     container_mock = AsyncMock(MockContainerClass)
     container_mock.with_label.return_value = container_mock
     container_mock.with_exec.return_value = container_mock
+    container_mock.with_file.return_value = container_mock
     return container_mock