more robustness

Author: alafanechere

tools/ci_connector_ops/ci_connector_ops/utils.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 import logging
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, Optional, Set, Tuple
 
 import git
 import requests
@@ -14,6 +14,9 @@
 SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-"
 ACCEPTANCE_TEST_CONFIG_FILE_NAME = "acceptance-test-config.yml"
 AIRBYTE_DOCKER_REPO = "airbyte"
+SOURCE_DEFINITIONS_FILE_PATH = "airbyte-config/init/src/main/resources/seed/source_definitions.yaml"
+DESTINATION_DEFINITIONS_FILE_PATH = "airbyte-config/init/src/main/resources/seed/destination_definitions.yaml"
+DEFINITIONS_FILE_PATH = {"source": SOURCE_DEFINITIONS_FILE_PATH, "destination": DESTINATION_DEFINITIONS_FILE_PATH}
 
 
 def download_catalog(catalog_url):
@@ -24,28 +27,34 @@ def download_catalog(catalog_url):
 OSS_CATALOG = download_catalog(OSS_CATALOG_URL)
 
 
-def get_changed_connector_names() -> List[str]:
+def read_definitions(definitions_file_path: str) -> Dict:
+    with open(definitions_file_path) as definitions_file:
+        return yaml.safe_load(definitions_file)
+
+
+def get_changed_connector_names() -> Set[str]:
     """Retrieve a list of connector names that were changed in the current branch (compared to master).
 
     Returns:
-        List[str]: List of connector names e.g ["source-pokeapi"]
+        Set[str]: Set of connector names e.g ["source-pokeapi"]
     """
-    head_commit = AIRBYTE_REPO.head.commit
-    master_diffs = head_commit.diff(AIRBYTE_REPO.remotes.origin.refs.master)
-    changed_source_connector_files = [diff.b_path for diff in master_diffs if diff.b_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX)]
+    changed_source_connector_files = {
+        file_path
+        for file_path in AIRBYTE_REPO.git.diff("--name-only", "origin/master...").split("\n")
+        if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX)
+    }
 
     def get_connector_name_from_path(path):
         return path.split("/")[2]
 
-    return [get_connector_name_from_path(changed_file) for changed_file in changed_source_connector_files]
+    return {get_connector_name_from_path(changed_file) for changed_file in changed_source_connector_files}
 
 
-def get_connector_definition(connector_name: str, catalog: Dict = OSS_CATALOG) -> Optional[Dict]:
+def get_connector_definition(connector_name: str) -> Optional[Dict]:
     """Find a connector definition from the catalog.
 
     Args:
         connector_name (str): The connector name. E.G. 'source-pokeapi'
-        catalog (Dict, optional): The connector catalog. Defaults to OSS_CATALOG.
 
     Raises:
         Exception: Raised if the definition type (source/destination) could not be determined from connector name.
@@ -58,22 +67,22 @@ def get_connector_definition(connector_name: str, catalog: Dict = OSS_CATALOG) -
         assert definition_type in ["source", "destination"]
     except AssertionError:
         raise Exception(f"Could not determine the definition type for {connector_name}.")
-    for definition in catalog[definition_type + "s"]:
+    definitions = read_definitions(DEFINITIONS_FILE_PATH[definition_type])
+    for definition in definitions:
         if definition["dockerRepository"].replace(f"{AIRBYTE_DOCKER_REPO}/", "") == connector_name:
             return definition
 
 
-def get_connector_release_stage(connector_name: str, catalog: Dict = OSS_CATALOG) -> Optional[str]:
+def get_connector_release_stage(connector_name: str) -> Optional[str]:
     """Retrieve the connector release stage (E.G. alpha/beta/generally_available).
 
     Args:
         connector_name (str): The connector name. E.G. 'source-pokeapi'
-        catalog (Dict, optional): The connector catalog. Defaults to OSS_CATALOG.
 
     Returns:
         Optional[str]: The connector release stage if it was defined. Returns None otherwise.
     """
-    definition = get_connector_definition(connector_name, catalog)
+    definition = get_connector_definition(connector_name)
     return definition.get("releaseStage")