connectors-ci: fix unhashable type 'set' (#29064)

Author: alafanechere

airbyte-ci/connectors/pipelines/README.md

@@ -380,6 +380,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 
 | Version | PR                                                        | Description                                                                                  |
 | ------- | --------------------------------------------------------- | -------------------------------------------------------------------------------------------- |
+| 0.4.4   | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set.                                             |
 | 0.4.3   | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors.                                             |
 | 0.4.2   | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`.                                 |
 | 0.4.1   | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands.                           |

airbyte-ci/connectors/pipelines/pipelines/bases.py

@@ -38,7 +38,7 @@
 
 @dataclass(frozen=True)
 class ConnectorWithModifiedFiles(Connector):
-    modified_files: Set[Path] = field(default_factory=list)
+    modified_files: Set[Path] = field(default_factory=frozenset)
 
     @property
     def has_metadata_change(self) -> bool:

airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py

@@ -78,4 +78,4 @@ async def run_test(airbyte_ci_package_path: str) -> bool:
         except dagger.ExecError as e:
             logger.error("Tests failed")
             logger.error(e.stderr)
-            return False
+            sys.exit(1)

airbyte-ci/connectors/pipelines/pipelines/utils.py

@@ -15,7 +15,7 @@
 from glob import glob
 from io import TextIOWrapper
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, FrozenSet, List, Optional, Set, Tuple, Union
 
 import anyio
 import asyncer
@@ -359,13 +359,13 @@ def get_modified_connectors(modified_files: Set[Path], all_connectors: Set[Conne
     return modified_connectors
 
 
-def get_connector_modified_files(connector: Connector, all_modified_files: Set[Path]) -> Set[Path]:
+def get_connector_modified_files(connector: Connector, all_modified_files: Set[Path]) -> FrozenSet[Path]:
     connector_modified_files = set()
     for modified_file in all_modified_files:
         modified_file_path = Path(modified_file)
         if modified_file_path.is_relative_to(connector.code_directory):
             connector_modified_files.add(modified_file)
-    return connector_modified_files
+    return frozenset(connector_modified_files)
 
 
 def get_modified_metadata_files(modified_files: Set[Union[str, Path]]) -> Set[Path]:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "0.4.3"
+version = "0.4.4"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-ci/connectors/pipelines/tests/conftest.py

@@ -15,12 +15,12 @@
 from tests.utils import ALL_CONNECTORS
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 def anyio_backend():
     return "asyncio"
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="module")
 async def dagger_client():
     async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
         yield client

airbyte-ci/connectors/pipelines/tests/test_gradle.py

@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+from pathlib import Path
+
+import pytest
+from pipelines import bases, gradle
+
+pytestmark = [
+    pytest.mark.anyio,
+]
+
+
+class TestGradleTask:
+    class DummyStep(gradle.GradleTask):
+        gradle_task_name = "dummyTask"
+
+        async def _run(self) -> bases.StepResult:
+            return bases.StepResult(self, bases.StepStatus.SUCCESS)
+
+    @pytest.fixture
+    def test_context(self, mocker, dagger_client):
+        return mocker.Mock(
+            secrets_to_mask=[],
+            dagger_client=dagger_client,
+            connector=bases.ConnectorWithModifiedFiles(
+                "source-postgres", frozenset({Path("airbyte-integrations/connectors/source-postgres/metadata.yaml")})
+            ),
+        )
+
+    async def test_build_include(self, test_context):
+        step = self.DummyStep(test_context)
+        assert step.build_include

airbyte-ci/connectors/pipelines/tests/test_utils.py

@@ -140,3 +140,28 @@ def test_get_modified_connectors_with_dependency_scanning(all_connectors, enable
     else:
         assert not_modified_java_connector not in modified_connectors
     assert modified_java_connector in modified_connectors
+
+
+def test_get_connector_modified_files():
+    connector = pick_a_random_connector()
+    other_connector = pick_a_random_connector(other_picked_connectors=[connector])
+
+    all_modified_files = {
+        connector.code_directory / "setup.py",
+        other_connector.code_directory / "README.md",
+    }
+
+    result = utils.get_connector_modified_files(connector, all_modified_files)
+    assert result == frozenset({connector.code_directory / "setup.py"})
+
+
+def test_no_modified_files_in_connector_directory():
+    connector = pick_a_random_connector()
+    other_connector = pick_a_random_connector(other_picked_connectors=[connector])
+
+    all_modified_files = {
+        other_connector.code_directory / "README.md",
+    }
+
+    result = utils.get_connector_modified_files(connector, all_modified_files)
+    assert result == frozenset()