airbyte-ci: ignore third party connectors on publish (#44025)

Author: alafanechere

airbyte-ci/connectors/pipelines/README.md

@@ -790,6 +790,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only
 
 | Version | PR                                                         | Description                                                                                                                  |
 |---------| ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------|
+| 4.32.4  | [#44025](https://github.com/airbytehq/airbyte/pull/44025)  | Ignore third party connectors on `publish`.                                                                                  |
 | 4.32.3  | [#44118](https://github.com/airbytehq/airbyte/pull/44118)  | Improve error handling in live tests.                                                                                        |
 | 4.32.2  | [#43970](https://github.com/airbytehq/airbyte/pull/43970)  | Make `connectors publish` early exit if no connectors are selected.                                                          |
 | 4.32.1  | [#41642](https://github.com/airbytehq/airbyte/pull/41642)  | Avoid transient publish failures by increasing `POETRY_REQUESTS_TIMEOUT` and setting retries on `PublishToPythonRegistry`.   |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py

@@ -1,6 +1,9 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
 import asyncclick as click
 from pipelines import main_logger
@@ -15,6 +18,23 @@
 from pipelines.helpers.utils import fail_if_missing_docker_hub_creds
 from pipelines.models.secrets import Secret
 
+if TYPE_CHECKING:
+    from typing import Iterable, List
+
+    from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles
+
+
+def filter_out_third_party_connectors(
+    selected_connectors_with_modified_files: Iterable[ConnectorWithModifiedFiles],
+) -> List[ConnectorWithModifiedFiles]:
+    filtered_connectors = []
+    for connector in selected_connectors_with_modified_files:
+        if connector.is_third_party:
+            main_logger.info(f"Skipping third party connector {connector.technical_name} from the list of connectors")
+        else:
+            filtered_connectors.append(connector)
+    return filtered_connectors
+
 
 @click.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.")
 @click_ci_requirements_option()
@@ -90,6 +110,9 @@ async def publish(
     python_registry_check_url: str,
 ) -> bool:
 
+    ctx.obj["selected_connectors_with_modified_files"] = filter_out_third_party_connectors(
+        ctx.obj["selected_connectors_with_modified_files"]
+    )
     if not ctx.obj["selected_connectors_with_modified_files"]:
         return True
 

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.32.3"
+version = "4.32.4"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py

@@ -34,7 +34,12 @@ def test_get_selected_connectors_by_name_no_file_modification():
         modified_files=set(),
     )
 
-    assert len(selected_connectors) == 1
+    has_strict_encrypt_variant = any("-strict-encrypt" in c.technical_name for c in selected_connectors)
+    if has_strict_encrypt_variant:
+        assert len(selected_connectors) == 2
+    else:
+        assert len(selected_connectors) == 1
+
     assert isinstance(selected_connectors[0], ConnectorWithModifiedFiles)
     assert selected_connectors[0].technical_name == connector.technical_name
     assert not selected_connectors[0].modified_files
@@ -138,7 +143,7 @@ def test_get_selected_connectors_with_modified_and_language():
         assert len(selected_connectors) == 2
     else:
         assert len(selected_connectors) == 1
-    assert selected_connectors[0].technical_name == second_modified_connector.technical_name
+        assert selected_connectors[0].technical_name == second_modified_connector.technical_name
 
 
 def test_get_selected_connectors_with_modified_and_support_level():
@@ -159,7 +164,7 @@ def test_get_selected_connectors_with_modified_and_support_level():
         assert len(selected_connectors) == 2
     else:
         assert len(selected_connectors) == 1
-    assert selected_connectors[0].technical_name == second_modified_connector.technical_name
+        assert selected_connectors[0].technical_name == second_modified_connector.technical_name
 
 
 def test_get_selected_connectors_with_modified_and_metadata_only():
@@ -180,12 +185,16 @@ def test_get_selected_connectors_with_modified_and_metadata_only():
         modified_files=modified_files,
     )
 
-    assert len(selected_connectors) == 1
-    assert selected_connectors[0].technical_name == second_modified_connector.technical_name
-    assert selected_connectors[0].modified_files == {
-        second_modified_connector.code_directory / METADATA_FILE_NAME,
-        second_modified_connector.code_directory / "setup.py",
-    }
+    has_strict_encrypt_variant = any("-strict-encrypt" in c.technical_name for c in selected_connectors)
+    if has_strict_encrypt_variant:
+        assert len(selected_connectors) == 2
+    else:
+        assert len(selected_connectors) == 1
+        assert selected_connectors[0].technical_name == second_modified_connector.technical_name
+        assert selected_connectors[0].modified_files == {
+            second_modified_connector.code_directory / METADATA_FILE_NAME,
+            second_modified_connector.code_directory / "setup.py",
+        }
 
 
 def test_get_selected_connectors_with_metadata_only():
@@ -205,13 +214,16 @@ def test_get_selected_connectors_with_metadata_only():
         metadata_query=None,
         modified_files=modified_files,
     )
-
-    assert len(selected_connectors) == 1
-    assert selected_connectors[0].technical_name == second_modified_connector.technical_name
-    assert selected_connectors[0].modified_files == {
-        second_modified_connector.code_directory / METADATA_FILE_NAME,
-        second_modified_connector.code_directory / "setup.py",
-    }
+    has_strict_encrypt_variant = any("-strict-encrypt" in c.technical_name for c in selected_connectors)
+    if has_strict_encrypt_variant:
+        assert len(selected_connectors) == 2
+    else:
+        assert len(selected_connectors) == 1
+        assert selected_connectors[0].technical_name == second_modified_connector.technical_name
+        assert selected_connectors[0].modified_files == {
+            second_modified_connector.code_directory / METADATA_FILE_NAME,
+            second_modified_connector.code_directory / "setup.py",
+        }
 
 
 def test_get_selected_connectors_with_metadata_query():
@@ -306,19 +318,6 @@ def click_context_obj(in_memory_secret_store):
     "command, command_args",
     [
         (connectors_test_command.test, []),
-        (
-            connectors_publish_command.publish,
-            [
-                "--spec-cache-gcs-credentials",
-                '{"foo": "bar"}',
-                "--spec-cache-bucket-name",
-                "test",
-                "--metadata-service-gcs-credentials",
-                '{"foo": "bar"}',
-                "--metadata-service-bucket-name",
-                "test",
-            ],
-        ),
         (connectors_build_command.build, []),
     ],
 )