ci-connector-ops: Check test strictness level on GA source connectors

.github/workflows/connector_ops_ci.yml

@@ -0,0 +1,23 @@
+name: Connector Ops CI
+
+on:
+  pull_request:
+    paths:
+      - "airbyte-integrations/connectors/source-**"
+jobs:
+  test-strictness-level:
+    name: "Check test strictness level"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Install ci-connector-ops package
+        run: pip install --quiet -e ./tools/ci_connector_ops
+      - name: Check test strictness level
+        run: check-test-strictness-level

tools/ci_connector_ops/ci_connector_ops/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#

tools/ci_connector_ops/ci_connector_ops/check_test_strictness_level.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import sys
+from typing import List
+
+from ci_connector_ops import utils
+
+RELEASE_STAGE_TO_STRICTNESS_LEVEL_MAPPING = {"generally_available": "high"}
+
+
+def find_connectors_with_bad_strictness_level() -> List[str]:
+    """Check if changed connectors have the expected SAT test strictness level according to their release stage.
+    1. Identify changed connectors
+    2. Retrieve their release stage from the catalog
+    3. Parse their acceptance test config file
+    4. Check if the test strictness level matches the strictness level expected for their release stage.
+
+    Returns:
+        List[str]: List of changed connector names that are not matching test strictness level expectations.
+    """
+    connectors_with_bad_strictness_level = []
+    changed_connector_names = utils.get_changed_connector_names()
+    for connector_name in changed_connector_names:
+        connector_release_stage = utils.get_connector_release_stage(connector_name)
+        expected_test_strictness_level = RELEASE_STAGE_TO_STRICTNESS_LEVEL_MAPPING.get(connector_release_stage)
+        _, acceptance_test_config = utils.get_acceptance_test_config(connector_name)
+        can_check_strictness_level = all(
+            [item is not None for item in [connector_release_stage, expected_test_strictness_level, acceptance_test_config]]
+        )
+        if can_check_strictness_level:
+            try:
+                assert acceptance_test_config.get("test_strictness_level") == expected_test_strictness_level
+            except AssertionError:
+                connectors_with_bad_strictness_level.append(connector_name)
+    return connectors_with_bad_strictness_level
+
+
+def main():
+    connectors_with_bad_strictness_level = find_connectors_with_bad_strictness_level()
+    if connectors_with_bad_strictness_level:
+        logging.error(
+            f"The following GA connectors must enable high test strictness level: {connectors_with_bad_strictness_level}. Please check this documentation for details: https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/#strictness-level"
+        )
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

tools/ci_connector_ops/ci_connector_ops/utils.py

@@ -0,0 +1,105 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+import logging
+from typing import Dict, Optional, Set, Tuple
+
+import git
+import requests
+import yaml
+
+AIRBYTE_REPO = git.Repo(".")
+OSS_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/oss_catalog.json"
+CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
+SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-"
+ACCEPTANCE_TEST_CONFIG_FILE_NAME = "acceptance-test-config.yml"
+AIRBYTE_DOCKER_REPO = "airbyte"
+SOURCE_DEFINITIONS_FILE_PATH = "airbyte-config/init/src/main/resources/seed/source_definitions.yaml"
+DESTINATION_DEFINITIONS_FILE_PATH = "airbyte-config/init/src/main/resources/seed/destination_definitions.yaml"
+DEFINITIONS_FILE_PATH = {"source": SOURCE_DEFINITIONS_FILE_PATH, "destination": DESTINATION_DEFINITIONS_FILE_PATH}
+
+
+def download_catalog(catalog_url):
+    response = requests.get(catalog_url)
+    return response.json()
+
+
+OSS_CATALOG = download_catalog(OSS_CATALOG_URL)
+
+
+def read_definitions(definitions_file_path: str) -> Dict:
+    with open(definitions_file_path) as definitions_file:
+        return yaml.safe_load(definitions_file)
+
+
+def get_changed_connector_names() -> Set[str]:
+    """Retrieve a list of connector names that were changed in the current branch (compared to master).
+
+    Returns:
+        Set[str]: Set of connector names e.g ["source-pokeapi"]
+    """
+    changed_source_connector_files = {
+        file_path
+        for file_path in AIRBYTE_REPO.git.diff("--name-only", "origin/master...").split("\n")
+        if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX)
+    }
+
+    def get_connector_name_from_path(path):
+        return path.split("/")[2]
+
+    return {get_connector_name_from_path(changed_file) for changed_file in changed_source_connector_files}
+
+
+def get_connector_definition(connector_name: str) -> Optional[Dict]:
+    """Find a connector definition from the catalog.
+
+    Args:
+        connector_name (str): The connector name. E.G. 'source-pokeapi'
+
+    Raises:
+        Exception: Raised if the definition type (source/destination) could not be determined from connector name.
+
+    Returns:
+        Optional[Dict]: The definition if the connector was found in the catalo. Returns None otherwise.
+    """
+    try:
+        definition_type = connector_name.split("-")[0]
+        assert definition_type in ["source", "destination"]
+    except AssertionError:
+        raise Exception(f"Could not determine the definition type for {connector_name}.")
+    definitions = read_definitions(DEFINITIONS_FILE_PATH[definition_type])
+    for definition in definitions:
+        if definition["dockerRepository"].replace(f"{AIRBYTE_DOCKER_REPO}/", "") == connector_name:
+            return definition
+
+
+def get_connector_release_stage(connector_name: str) -> Optional[str]:
+    """Retrieve the connector release stage (E.G. alpha/beta/generally_available).
+
+    Args:
+        connector_name (str): The connector name. E.G. 'source-pokeapi'
+
+    Returns:
+        Optional[str]: The connector release stage if it was defined. Returns None otherwise.
+    """
+    definition = get_connector_definition(connector_name)
+    return definition.get("releaseStage")
+
+
+def get_acceptance_test_config(connector_name: str) -> Tuple[str, Dict]:
+    """Retrieve the acceptance test config file path and its content as dict.
+
+    Args:
+        connector_name (str): The connector name. E.G. 'source-pokeapi'
+
+
+    Returns:
+        Tuple(str, Dict): The acceptance test config file path and its content as dict.
+    """
+    acceptance_test_config_path = f"{CONNECTOR_PATH_PREFIX}/{connector_name}/{ACCEPTANCE_TEST_CONFIG_FILE_NAME}"
+    try:
+        with open(acceptance_test_config_path) as acceptance_test_config_file:
+            return acceptance_test_config_path, yaml.safe_load(acceptance_test_config_file)
+    except FileNotFoundError:
+        logging.warning(f"No {ACCEPTANCE_TEST_CONFIG_FILE_NAME} file found for {connector_name}")
+        return None, None

tools/ci_connector_ops/setup.py

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+from setuptools import find_packages, setup
+
+MAIN_REQUIREMENTS = ["requests", "PyYAML~=6.0", "GitPython~=3.1.29"]
+
+
+setup(
+    version="0.1.0",
+    name="ci_connector_ops",
+    description="Packaged maintained by the connector operations team to perform CI for connectors",
+    author="Airbyte",
+    author_email="[email protected]",
+    packages=find_packages(),
+    install_requires=MAIN_REQUIREMENTS,
+    python_requires=">=3.9",
+    entry_points={
+        "console_scripts": [
+            "check-test-strictness-level = ci_connector_ops.check_test_strictness_level:main",
+        ],
+    },
+)