feat(connectors-qa): support manifest-only connectors (#43377)

Author: natikgadzhi

airbyte-ci/connectors/connectors_qa/README.md

@@ -108,6 +108,10 @@ poe lint
 
 ## Changelog
 
+### 1.6.0
+
+Added `manifest-only` connectors support — they will run basic assets and metadata checks.
+
 ### 1.5.1
 
 Bumped dependencies.

airbyte-ci/connectors/connectors_qa/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "connectors-qa"
-version = "1.5.1"
+version = "1.6.0"
 description = "A package to run QA checks on Airbyte connectors, generate reports and documentation."
 authors = ["Airbyte <[email protected]>"]
 readme = "README.md"

airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py

@@ -1,6 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
-
 import os
 from datetime import datetime, timedelta
 
@@ -40,8 +39,11 @@ class CheckConnectorLanguageTag(MetadataCheck):
 
     PYTHON_LANGUAGE_TAG = "language:python"
     JAVA_LANGUAGE_TAG = "language:java"
+    MANIFEST_ONLY_LANGUAGE_TAG = "language:manifest-only"
 
     def get_expected_language_tag(self, connector: Connector) -> str:
+        if (connector.code_directory / "manifest.yaml").exists():
+            return self.MANIFEST_ONLY_LANGUAGE_TAG
         if (connector.code_directory / consts.SETUP_PY_FILE_NAME).exists() or (
             connector.code_directory / consts.PYPROJECT_FILE_NAME
         ).exists():
@@ -145,7 +147,6 @@ class ValidateBreakingChangesDeadlines(MetadataCheck):
     minimum_days_until_deadline = 7
 
     def _run(self, connector: Connector) -> CheckResult:
-
         # fetch the current branch version of the connector first.
         # we'll try and see if there are any breaking changes associated
         # with it next.

airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py

@@ -1,6 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
-
 import semver
 import toml
 from connector_ops.utils import Connector, ConnectorLanguage  # type: ignore
@@ -63,6 +62,24 @@ def _run(self, connector: Connector) -> CheckResult:
         return self.create_check_result(connector=connector, passed=True, message="PyPi publishing is enabled")
 
 
+class CheckManifestOnlyConnectorBaseImage(PackagingCheck):
+    name = "Manifest-only connectors must use `source-declarative-manifest` as their base image"
+    description = "Manifest-only connectors must use `airbyte/source-declarative-manifest` as their base image."
+    applies_to_connector_languages = [ConnectorLanguage.MANIFEST_ONLY]
+
+    def _run(self, connector: Connector) -> CheckResult:
+        base_image = get(connector.metadata, "connectorBuildOptions.baseImage")
+        base_image_name = base_image.split(":")[0] if base_image else None
+
+        if base_image_name != "docker.io/airbyte/source-declarative-manifest":
+            return self.create_check_result(
+                connector=connector,
+                passed=False,
+                message=f"A manifest-only connector must use `source-declarative-manifest` base image. Replace the base image in {consts.METADATA_FILE_NAME} file",
+            )
+        return self.create_check_result(connector=connector, passed=True, message="Connector uses source-declarative-manifest base image")
+
+
 class CheckConnectorLicense(PackagingCheck):
     name = "Connectors must be licensed under MIT or Elv2"
     description = f"Connectors must be licensed under the MIT or Elv2 license. This is to ensure that all connectors are licensed under a permissive license. More details in our [License FAQ]({consts.LICENSE_FAQ_URL})."
@@ -218,4 +235,5 @@ def _run(self, connector: Connector) -> CheckResult:
     CheckVersionFollowsSemver(),
     CheckConnectorVersionMatchInPyproject(),
     CheckPublishToPyPiIsEnabled(),
+    CheckManifestOnlyConnectorBaseImage(),
 ]

airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py

@@ -122,6 +122,7 @@ class CheckConnectorUsesPythonBaseImage(SecurityCheck):
     applies_to_connector_languages = [
         ConnectorLanguage.PYTHON,
         ConnectorLanguage.LOW_CODE,
+        ConnectorLanguage.MANIFEST_ONLY,
     ]
 
     def _run(self, connector: Connector) -> CheckResult:

airbyte-ci/connectors/connectors_qa/src/connectors_qa/consts.py

@@ -13,6 +13,7 @@
 LOW_CODE_MANIFEST_FILE_NAME = "manifest.yaml"
 METADATA_DOCUMENTATION_URL = "https://docs.airbyte.com/connector-development/connector-metadata-file"
 METADATA_FILE_NAME = "metadata.yaml"
+MANIFEST_FILE_NAME = "manifest.yaml"
 POETRY_LOCK_FILE_NAME = "poetry.lock"
 PYPROJECT_FILE_NAME = "pyproject.toml"
 SEMVER_FOR_CONNECTORS_DOC_URL = "https://docs.airbyte.com/contributing-to-airbyte/#semantic-versioning-for-connectors"

airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py

@@ -17,6 +17,7 @@
     ConnectorLanguage.JAVA,
     ConnectorLanguage.LOW_CODE,
     ConnectorLanguage.PYTHON,
+    ConnectorLanguage.MANIFEST_ONLY,
 ]
 
 ALL_TYPES = ["source", "destination"]
@@ -62,7 +63,6 @@ def __repr__(self) -> str:
 
 
 class Check(ABC):
-
     requires_metadata: bool = True
     runs_on_released_connectors: bool = True
 

airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import os
 
@@ -9,7 +10,6 @@
 
 
 class TestValidateMetadata:
-
     def test_fail_when_deserialization_fails(self, mocker, tmp_path):
         # Arrange
         mocker.patch.object(metadata, "validate_and_load", return_value=(None, "error"))
@@ -38,6 +38,19 @@ def test_pass_when_metadata_file_is_valid(self, mocker, tmp_path):
         assert result.status == CheckStatus.PASSED
         assert result.message == "Metadata file valid."
 
+    def test_checks_apply_to_manifest_only_connectors(self, mocker, tmp_path):
+        # Arrange
+        connector = mocker.MagicMock(metadata={"tags": ["language:manifest-only"]}, code_directory=tmp_path)
+        code_directory = tmp_path
+        (code_directory / consts.MANIFEST_FILE_NAME).touch()
+
+        # Act
+        result = metadata.CheckConnectorLanguageTag()._run(connector)
+
+        # Assert
+        assert result.status == CheckStatus.PASSED
+        assert result.message == "Language tag language:manifest-only is present in the metadata file"
+
 
 class TestCheckConnectorLanguageTag:
     def test_fail_when_no_language_tags(self, mocker):
@@ -118,7 +131,6 @@ def test_pass_when_java(self, mocker, tmp_path):
 
 
 class TestCheckConnectorCDKTag:
-
     def test_fail_when_no_cdk_tags(self, mocker):
         # Arrange
         connector = mocker.MagicMock(metadata={"tags": []})
@@ -160,7 +172,7 @@ def test_fail_when_python_tag_on_low_code_connector(self, mocker, tmp_path):
         connector = mocker.MagicMock(technical_name="source-test", metadata={"tags": ["cdk:python"]}, code_directory=tmp_path)
         code_directory = tmp_path
         (code_directory / "source_test").mkdir()
-        (code_directory / "source_test"/ consts.LOW_CODE_MANIFEST_FILE_NAME).touch()
+        (code_directory / "source_test" / consts.LOW_CODE_MANIFEST_FILE_NAME).touch()
 
         # Act
         result = metadata.CheckConnectorCDKTag()._run(connector)

airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 from connectors_qa import consts
 from connectors_qa.checks import packaging
@@ -57,6 +58,47 @@ def test_pass_when_poetry_is_used(self, tmp_path, mocker):
         assert result.message == "Poetry is used for dependency management"
 
 
+class TestCheckManifestOnlyConnectorBaseImage:
+    def test_pass_with_source_declarative_manifest(self, mocker, tmp_path):
+        connector = mocker.MagicMock(
+            code_directory=tmp_path,
+            metadata={"connectorBuildOptions": {"baseImage": "docker.io/airbyte/source-declarative-manifest:4.3.0@SHA"}},
+        )
+
+        # Act
+        result = packaging.CheckManifestOnlyConnectorBaseImage()._run(connector)
+
+        # Assert
+        assert result.status == CheckStatus.PASSED
+        assert "Connector uses source-declarative-manifest base image" in result.message
+
+    def test_fail_with_different_image(self, mocker, tmp_path):
+        connector = mocker.MagicMock(
+            code_directory=tmp_path,
+            metadata={"connectorBuildOptions": {"baseImage": "docker.io/airbyte/connector-base-image:2.0.0@SHA"}},
+        )
+
+        # Act
+        result = packaging.CheckManifestOnlyConnectorBaseImage()._run(connector)
+
+        # Assert
+        assert result.status == CheckStatus.FAILED
+        assert "A manifest-only connector must use `source-declarative-manifest` base image" in result.message
+
+    def test_fail_with_missing_image(self, mocker, tmp_path):
+        connector = mocker.MagicMock(
+            code_directory=tmp_path,
+            metadata={"connectorBuildOptions": {}},
+        )
+
+        # Act
+        result = packaging.CheckManifestOnlyConnectorBaseImage()._run(connector)
+
+        # Assert
+        assert result.status == CheckStatus.FAILED
+        assert "A manifest-only connector must use `source-declarative-manifest` base image" in result.message
+
+
 class TestCheckPublishToPyPiIsEnabled:
     def test_fail_if_publish_to_pypi_is_not_enabled(self, mocker):
         # Arrange

docs/contributing-to-airbyte/resources/qa-checks.md

@@ -11,7 +11,7 @@ They are by no mean replacing the need for a manual review of the connector code
 ### Breaking changes must be accompanied by a migration guide
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -21,7 +21,7 @@ This document should contain a section for each breaking change, in order of the
 ### Connectors must have user facing documentation
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -30,7 +30,7 @@ The user facing connector documentation should be stored under `./docs/integrati
 ### Connectors must have a changelog entry for each version
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -41,7 +41,7 @@ Each new version of a connector must have a changelog entry defined in the user
 ### Connectors must have valid metadata.yaml file
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -50,7 +50,7 @@ Connectors must have a `metadata.yaml` file at the root of their directory. This
 ### Connector must have a language tag in metadata
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -68,7 +68,7 @@ Python connectors must have a CDK tag in their metadata. It must be set in the `
 ### Breaking change deadline should be a week in the future
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -77,7 +77,7 @@ If the connector version has a breaking change, the deadline field must be set t
 ### Certified source connector must have a value filled out for maxSecondsBetweenMessages in metadata
 
 _Applies to the following connector types: source_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with certified support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -97,7 +97,7 @@ Connectors must use [Poetry](https://python-poetry.org/) for dependency manageme
 ### Connectors must be licensed under MIT or Elv2
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -115,7 +115,7 @@ Connectors license in metadata.yaml and pyproject.toml file must match. This is
 ### Connector version must follow Semantic Versioning
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -139,12 +139,21 @@ _Applies to connector with any Airbyte usage level_
 
 Python connectors must have [PyPi](https://pypi.org/) publishing enabled in their `metadata.yaml` file. This is declared by setting `remoteRegistries.pypi.enabled` to `true` in metadata.yaml. This is to ensure that all connectors can be published to PyPi and can be used in `PyAirbyte`.
 
+### Manifest-only connectors must use `source-declarative-manifest` as their base image
+
+_Applies to the following connector types: source, destination_
+_Applies to the following connector languages: manifest-only_
+_Applies to connector with any support level_
+_Applies to connector with any Airbyte usage level_
+
+Manifest-only connectors must use `airbyte/source-declarative-manifest` as their base image.
+
 ## 💼 Assets
 
 ### Connectors must have an icon
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -155,7 +164,7 @@ Each connector must have an icon available in at the root of the connector code
 ### Connectors must use HTTPS only
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -164,7 +173,7 @@ Connectors must use HTTPS only when making requests to external services.
 ### Python connectors must not use a Dockerfile and must declare their base image in metadata.yaml file
 
 _Applies to the following connector types: source, destination_
-_Applies to the following connector languages: python, low-code_
+_Applies to the following connector languages: python, low-code, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with any Airbyte usage level_
 
@@ -176,7 +185,7 @@ This is to ensure that all connectors use a base image which is maintained and h
 ### Medium to High Use Connectors must enable acceptance tests
 
 _Applies to the following connector types: source_
-_Applies to the following connector languages: java, low-code, python_
+_Applies to the following connector languages: java, low-code, python, manifest-only_
 _Applies to connector with any support level_
 _Applies to connector with medium, high Airbyte usage level_