Version connector build statuses (#22029)

* Refactor the job log json to include the docker_version

* Output to versioned folder

* Handle the case where people call the action without connector prefixed

* Retrieve status of each connector

* Use build report statuses in the QA Engine

* Cast build status as an enum

Author: bnchrch

airbyte-integrations/bases/source-acceptance-test/README.md

@@ -1,10 +1,10 @@
 # Source Acceptance Tests
-This package gathers multiple test suites to assess the sanity of any Airbyte **source** connector. 
+This package gathers multiple test suites to assess the sanity of any Airbyte **source** connector.
 It is shipped as a [pytest](https://docs.pytest.org/en/7.1.x/) plugin and relies on pytest to discover, configure and execute tests.
 Test-specific documentation can be found [here](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/)).
 
 ## Running the acceptance tests on a source connector:
-1. `cd` into your connector project (e.g. `airbyte-integrations/connectors/source-pokeapi`) 
+1. `cd` into your connector project (e.g. `airbyte-integrations/connectors/source-pokeapi`)
 2. Edit `acceptance-test-config.yml` according to your need. Please refer to our [Source Acceptance Test Reference](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/) if you need details about the available options.
 3. Build the connector docker image ( e.g.: `docker build . -t airbyte/source-pokeapi:dev`)
 4. Use one of the following ways to run tests (**from your connector project directory**)
@@ -33,9 +33,9 @@ _Note: this will use the latest docker image for source-acceptance-test and will
   * When running or `./acceptance-test-docker.sh` in a connector project
 * When running `/test` command on a GitHub pull request.
 * When running `/publish` command on a GitHub pull request.
-* When running ` integration-test` GitHub action that is creating the [connector builds summary](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/builds.md).
+* When running ` integration-test` GitHub action that is creating the JSON files linked to from [connector builds summary](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/builds.md).
 
-## Developing on the SAT 
+## Developing on the SAT
 You may want to iterate on the SAT project: adding new tests, fixing a bug etc.
 These iterations are more conveniently achieved by remaining in the current directory.
 

airbyte-integrations/builds.md

@@ -30,6 +30,7 @@
 DESTINATION_DEFINITIONS_YAML = f"{CONNECTOR_DEFINITIONS_DIR}/destination_definitions.yaml"
 CONNECTORS_ROOT_PATH = "./airbyte-integrations/connectors"
 RELEVANT_BASE_MODULES = ["base-normalization", "source-acceptance-test"]
+CONNECTOR_BUILD_OUTPUT_URL = "https://dnsgjos7lj2fu.cloudfront.net/tests/history/connectors"
 
 # Global vars
 TESTED_SOURCE = []
@@ -42,7 +43,7 @@
 
 
 def get_status_page(connector) -> str:
-    response = requests.get(f"https://dnsgjos7lj2fu.cloudfront.net/tests/summary/{connector}/index.html")
+    response = requests.get(f"{CONNECTOR_BUILD_OUTPUT_URL}/{connector}/index.html")
     if response.status_code == 200:
         return response.text
 

tools/bin/build_report.py

@@ -9,7 +9,7 @@
 
 
 def check_documentation_file_exists(connector: Connector) -> bool:
-    """Check if a markdown file with connector documentation is available 
+    """Check if a markdown file with connector documentation is available
     in docs/integrations/<connector-type>s/<connector-name>.md
 
     Args:
@@ -38,7 +38,7 @@ def check_documentation_follows_guidelines(connector: Connector) -> bool:
         follows_guidelines = False
 
     expected_sections = [
-        "## Prerequisites", 
+        "## Prerequisites",
         "## Setup guide",
         "## Supported sync modes",
         "## Supported streams",
@@ -112,7 +112,7 @@ def check_connector_has_no_critical_vulnerabilities(connector: Connector) -> boo
 QA_CHECKS = [
     check_documentation_file_exists,
     # Disabling the following check because it's likely to not pass on a lot of connectors.
-    # check_documentation_follows_guidelines, 
+    # check_documentation_follows_guidelines,
     check_changelog_entry_is_updated,
     check_connector_icon_is_available,
     check_connector_https_url_only,

tools/ci_connector_ops/ci_connector_ops/qa_checks.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-
+CONNECTOR_BUILD_OUTPUT_URL = "https://dnsgjos7lj2fu.cloudfront.net/tests/history/connectors"
 CLOUD_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/cloud_catalog.json"
 OSS_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/oss_catalog.json"
 

tools/ci_connector_ops/ci_connector_ops/qa_engine/constants.py

@@ -6,8 +6,8 @@
 import pandas as pd
 
 def get_enriched_catalog(
-    oss_catalog: pd.DataFrame, 
-    cloud_catalog: pd.DataFrame, 
+    oss_catalog: pd.DataFrame,
+    cloud_catalog: pd.DataFrame,
     adoption_metrics_per_connector_version: pd.DataFrame) -> pd.DataFrame:
     """Merge OSS and Cloud catalog in a single dataframe on their definition id.
     Transformations:
@@ -35,7 +35,7 @@ def get_enriched_catalog(
         indicator=True,
         suffixes=("", "_cloud"),
     )
-    
+
     enriched_catalog.columns = enriched_catalog.columns.str.replace(
         "(?<=[a-z])(?=[A-Z])", "_", regex=True
     ).str.lower() # column names to snake case

tools/ci_connector_ops/ci_connector_ops/qa_engine/enrichments.py

@@ -5,14 +5,57 @@
 import os
 from importlib.resources import files
 import json
+import logging
+
+from .constants import CONNECTOR_BUILD_OUTPUT_URL
 
 from google.oauth2 import service_account
 import requests
 import pandas as pd
+from typing import Optional
+
+from enum import Enum
+
+LOGGER = logging.getLogger(__name__)
+
+class BUILD_STATUSES(str, Enum):
+    SUCCESS = "success"
+    FAILURE = "failure"
+    NOT_FOUND = None
+
+    @classmethod
+    def from_string(cls, string_value: Optional[str]) -> "BUILD_STATUSES":
+        if string_value is None:
+            return BUILD_STATUSES.NOT_FOUND
+
+        return BUILD_STATUSES[string_value.upper()]
+
+
+
+def get_connector_build_output_url(connector_technical_name: str, connector_version: str) -> str:
+    return f"{CONNECTOR_BUILD_OUTPUT_URL}/{connector_technical_name}/{connector_version}.json"
+
+def fetch_latest_build_status_for_connector_version(connector_technical_name: str, connector_version: str) ->BUILD_STATUSES:
+    """Fetch the latest build status for a given connector version."""
+    connector_build_output_url = get_connector_build_output_url(connector_technical_name, connector_version)
+    connector_build_output_response = requests.get(connector_build_output_url)
+
+    # if the connector returned successfully, return the outcome
+    if connector_build_output_response.status_code == 200:
+        connector_build_output = connector_build_output_response.json()
+        outcome = connector_build_output.get("outcome")
+
+        try:
+            return BUILD_STATUSES.from_string(outcome)
+        except KeyError:
+            LOGGER.error(f"Error: Unexpected build status value: {outcome} for connector {connector_technical_name}:{connector_version}")
+            return BUILD_STATUSES.NOT_FOUND
 
+    else:
+        return BUILD_STATUSES.NOT_FOUND
 
 def fetch_remote_catalog(catalog_url: str) -> pd.DataFrame:
-    """Fetch a combined remote catalog and return a single DataFrame 
+    """Fetch a combined remote catalog and return a single DataFrame
     with sources and destinations defined by the connector_type column.
 
     Args:

tools/ci_connector_ops/ci_connector_ops/qa_engine/inputs.py

@@ -3,17 +3,17 @@
 #
 
 
-from .constants import CLOUD_CATALOG_URL, GCS_QA_REPORT_PATH, OSS_CATALOG_URL
-from . import enrichments, inputs, validations, outputs
+from .constants import CLOUD_CATALOG_URL, OSS_CATALOG_URL
+from . import enrichments, inputs, validations
 
 
 def main():
     oss_catalog = inputs.fetch_remote_catalog(OSS_CATALOG_URL)
     cloud_catalog = inputs.fetch_remote_catalog(CLOUD_CATALOG_URL)
     adoption_metrics_per_connector_version = inputs.fetch_adoption_metrics_per_connector_version()
     enriched_catalog = enrichments.get_enriched_catalog(
-        oss_catalog, 
-        cloud_catalog, 
+        oss_catalog,
+        cloud_catalog,
         adoption_metrics_per_connector_version
     )
     validations.get_qa_report(enriched_catalog, len(oss_catalog))

tools/ci_connector_ops/ci_connector_ops/qa_engine/main.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-
 from datetime import datetime
 from typing import Iterable
 
@@ -11,6 +10,7 @@
 
 from .constants import INAPPROPRIATE_FOR_CLOUD_USE_CONNECTORS
 from .models import ConnectorQAReport, QAReport
+from .inputs import fetch_latest_build_status_for_connector_version, BUILD_STATUSES
 
 TRUTHY_COLUMNS_TO_BE_ELIGIBLE = [
   "documentation_is_available",
@@ -32,26 +32,31 @@ def is_eligible_for_promotion_to_cloud(connector_qa_data: pd.Series) -> bool:
   if connector_qa_data["is_on_cloud"]:
     return False
   return all([
-    connector_qa_data[col] 
+    connector_qa_data[col]
     for col in TRUTHY_COLUMNS_TO_BE_ELIGIBLE
   ])
 
+def latest_build_is_successful(connector_qa_data: pd.Series) -> bool:
+    connector_technical_name = connector_qa_data["connector_technical_name"]
+    connector_version = connector_qa_data["connector_version"]
+    latest_build_status = fetch_latest_build_status_for_connector_version(connector_technical_name, connector_version)
+    return latest_build_status == BUILD_STATUSES.SUCCESS
 
 def get_qa_report(enriched_catalog: pd.DataFrame, oss_catalog_length: int) -> pd.DataFrame:
     """Perform validation steps on top of the enriched catalog.
     Adds the following columns:
       - documentation_is_available:
         GET the documentation URL and expect a 200 status code.
-      - is_appropriate_for_cloud_use: 
+      - is_appropriate_for_cloud_use:
         Determined from an hardcoded list of definition ids inappropriate for cloud use.
       - latest_build_is_successful:
         Check if the latest build for the current connector version is successful.
       - number_of_connections:
         Get the number of connections using this connector version from our datawarehouse.
       - number_of_users:
-        Get the number of users using this connector version from our datawarehouse.   
+        Get the number of users using this connector version from our datawarehouse.
       - sync_success_rate:
-        Get the sync success rate of the connections with this connector version from our datawarehouse.    
+        Get the sync success rate of the connections with this connector version from our datawarehouse.
     Args:
         enriched_catalog (pd.DataFrame): The enriched catalog.
         oss_catalog_length (pd.DataFrame): The length of the OSS catalog, for sanity check.
@@ -62,12 +67,8 @@ def get_qa_report(enriched_catalog: pd.DataFrame, oss_catalog_length: int) -> pd
     qa_report = enriched_catalog.copy(deep=True)
     qa_report["documentation_is_available"] = qa_report.documentation_url.apply(url_is_reachable)
     qa_report["is_appropriate_for_cloud_use"] = qa_report.connector_definition_id.apply(is_appropriate_for_cloud_use)
-    
-    # TODO YET TO IMPLEMENT VALIDATIONS
-    qa_report["latest_build_is_successful"] = False # TODO, tracked in https://github.com/airbytehq/airbyte/issues/21720
 
-    qa_report["is_eligible_for_promotion_to_cloud"] = qa_report.apply(is_eligible_for_promotion_to_cloud, axis="columns")
-    qa_report["report_generation_datetime"] = datetime.utcnow()
+    qa_report["latest_build_is_successful"] = qa_report.apply(latest_build_is_successful, axis="columns")
 
     qa_report["is_eligible_for_promotion_to_cloud"] = qa_report.apply(is_eligible_for_promotion_to_cloud, axis="columns")
     qa_report["report_generation_datetime"] = datetime.utcnow()

tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py

@@ -36,9 +36,6 @@ def download_catalog(catalog_url):
 OSS_CATALOG = download_catalog(OSS_CATALOG_URL)
 
 
-
-
-
 class ConnectorInvalidNameError(Exception):
     pass
 
@@ -108,15 +105,15 @@ def icon_path(self) -> Path:
     @property
     def code_directory(self) -> Path:
         return Path(f"./airbyte-integrations/connectors/{self.technical_name}")
-    
+
     @property
     def version(self) -> str:
         with open(self.code_directory / "Dockerfile") as f:
             for line in f:
                 if "io.airbyte.version" in line:
                     return line.split("=")[1].strip()
         raise ConnectorVersionNotFound("""
-            Could not find the connector version from its Dockerfile. 
+            Could not find the connector version from its Dockerfile.
             The io.airbyte.version tag is missing.
             """)
 

tools/ci_connector_ops/ci_connector_ops/utils.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
@@ -21,7 +20,6 @@
     "pytest-mock~=3.10.0",
 ]
 
-
 setup(
     version="0.1.10",
     name="ci_connector_ops",

tools/ci_connector_ops/setup.py

@@ -7,6 +7,8 @@
 
 import pandas as pd
 import pytest
+from unittest.mock import MagicMock, call
+import requests
 
 from ci_connector_ops.qa_engine import inputs, constants
 
@@ -56,3 +58,86 @@ def test_fetch_adoption_metrics_per_connector_version(mocker):
         project_id=expected_project_id,
         credentials=inputs.service_account.Credentials.from_service_account_info.return_value
     )
+
+@pytest.mark.parametrize("connector_name, connector_version, mocked_json_payload, mocked_status_code, expected_status", [
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        {
+            "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+            "outcome": "success",
+            "docker_version": "0.1.5",
+            "timestamp": "1674872401",
+            "connector": "connectors/source-pokeapi"
+        },
+        200,
+        inputs.BUILD_STATUSES.SUCCESS
+    ),
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        {
+            "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+            "outcome": "failure",
+            "docker_version": "0.1.5",
+            "timestamp": "1674872401",
+            "connector": "connectors/source-pokeapi"
+        },
+        200,
+        inputs.BUILD_STATUSES.FAILURE
+    ),
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        None,
+        404,
+        inputs.BUILD_STATUSES.NOT_FOUND
+    ),
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        {
+            "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+            "docker_version": "0.1.5",
+            "timestamp": "1674872401",
+            "connector": "connectors/source-pokeapi"
+        },
+        200,
+        inputs.BUILD_STATUSES.NOT_FOUND
+    ),
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        None,
+        404,
+        inputs.BUILD_STATUSES.NOT_FOUND
+    ),
+])
+def test_fetch_latest_build_status_for_connector_version(mocker, connector_name, connector_version, mocked_json_payload, mocked_status_code, expected_status):
+    # Mock the api call to get the latest build status for a connector version
+    mock_response = MagicMock()
+    mock_response.json.return_value = mocked_json_payload
+    mock_response.status_code = mocked_status_code
+    mock_get = mocker.patch.object(requests, 'get', return_value=mock_response)
+
+    assert inputs.fetch_latest_build_status_for_connector_version(connector_name, connector_version) == expected_status
+    assert mock_get.call_args == call(f"{constants.CONNECTOR_BUILD_OUTPUT_URL}/{connector_name}/{connector_version}.json")
+
+def test_fetch_latest_build_status_for_connector_version_invalid_status(mocker, caplog):
+    connector_name = "connectors/source-pokeapi"
+    connector_version = "0.1.5"
+    mocked_json_payload = {
+        "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+        "outcome": "unknown_outcome_123",
+        "docker_version": "0.1.5",
+        "timestamp": "1674872401",
+        "connector": "connectors/source-pokeapi"
+    }
+    # Mock the api call to get the latest build status for a connector version
+    mock_response = MagicMock()
+    mock_response.json.return_value = mocked_json_payload
+    mock_response.status_code = 200
+    mocker.patch.object(requests, 'get', return_value=mock_response)
+
+    assert inputs.fetch_latest_build_status_for_connector_version(connector_name, connector_version) == inputs.BUILD_STATUSES.NOT_FOUND
+    assert 'Error: Unexpected build status value: unknown_outcome_123 for connector connectors/source-pokeapi:0.1.5' in caplog.text