Use build report statuses in the QA Engine

Author: bnchrch

tools/bin/build_report.py

@@ -42,7 +42,8 @@
 
 
 def get_status_page(connector) -> str:
-    response = requests.get(f"https://dnsgjos7lj2fu.cloudfront.net/tests/summary/{connector}/index.html")
+    # TODO (ben): ensure this is still working as expected
+    response = requests.get(f"https://dnsgjos7lj2fu.cloudfront.net/tests/summary/connectors/{connector}/index.html")
     if response.status_code == 200:
         return response.text
 

tools/ci_connector_ops/ci_connector_ops/qa_engine/enrichments.py

@@ -6,8 +6,8 @@
 import pandas as pd
 
 def get_enriched_catalog(
-    oss_catalog: pd.DataFrame, 
-    cloud_catalog: pd.DataFrame, 
+    oss_catalog: pd.DataFrame,
+    cloud_catalog: pd.DataFrame,
     adoption_metrics_per_connector_version: pd.DataFrame) -> pd.DataFrame:
     """Merge OSS and Cloud catalog in a single dataframe on their definition id.
     Transformations:
@@ -35,7 +35,7 @@ def get_enriched_catalog(
         indicator=True,
         suffixes=("", "_cloud"),
     )
-    
+
     enriched_catalog.columns = enriched_catalog.columns.str.replace(
         "(?<=[a-z])(?=[A-Z])", "_", regex=True
     ).str.lower() # column names to snake case

tools/ci_connector_ops/ci_connector_ops/qa_engine/validations.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-
 from datetime import datetime
 from typing import Iterable
 
@@ -43,7 +42,6 @@ def latest_build_is_successful(connector_qa_data: pd.Series) -> bool:
     latest_build_status = fetch_latest_build_status_for_connector_version(connector_technical_name, connector_version)
     return latest_build_status == "success"
 
-
 def get_qa_report(enriched_catalog: pd.DataFrame, oss_catalog_length: int) -> pd.DataFrame:
     """Perform validation steps on top of the enriched catalog.
     Adds the following columns:
@@ -70,10 +68,7 @@ def get_qa_report(enriched_catalog: pd.DataFrame, oss_catalog_length: int) -> pd
     qa_report["documentation_is_available"] = qa_report.documentation_url.apply(url_is_reachable)
     qa_report["is_appropriate_for_cloud_use"] = qa_report.connector_definition_id.apply(is_appropriate_for_cloud_use)
 
-    qa_report["latest_build_is_successful"] = qa_report.apply(latest_build_is_successful)
-
-    qa_report["is_eligible_for_promotion_to_cloud"] = qa_report.apply(is_eligible_for_promotion_to_cloud, axis="columns")
-    qa_report["report_generation_datetime"] = datetime.utcnow()
+    qa_report["latest_build_is_successful"] = qa_report.apply(latest_build_is_successful, axis="columns")
 
     qa_report["is_eligible_for_promotion_to_cloud"] = qa_report.apply(is_eligible_for_promotion_to_cloud, axis="columns")
     qa_report["report_generation_datetime"] = datetime.utcnow()

tools/ci_connector_ops/setup.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
@@ -21,7 +20,6 @@
     "pytest-mock~=3.10.0",
 ]
 
-
 setup(
     version="0.1.9",
     name="ci_connector_ops",

tools/ci_connector_ops/tests/test_qa_engine/test_inputs.py

@@ -7,6 +7,8 @@
 
 import pandas as pd
 import pytest
+from unittest.mock import MagicMock, call
+import requests
 
 from ci_connector_ops.qa_engine import inputs, constants
 
@@ -56,3 +58,48 @@ def test_fetch_adoption_metrics_per_connector_version(mocker):
         project_id=expected_project_id,
         credentials=inputs.service_account.Credentials.from_service_account_info.return_value
     )
+
+@pytest.mark.parametrize("connector_name, connector_version, mocked_json_payload, mocked_status_code, expected_status", [
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        {
+            "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+            "outcome": "success",
+            "docker_version": "0.1.5",
+            "timestamp": "1674872401",
+            "connector": "connectors/source-pokeapi"
+        },
+        200,
+        "success"
+    ),
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        {
+            "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+            "outcome": "failed",
+            "docker_version": "0.1.5",
+            "timestamp": "1674872401",
+            "connector": "connectors/source-pokeapi"
+        },
+        200,
+        "failed"
+    ),
+    (
+        "connectors/source-pokeapi",
+        "0.1.5",
+        None,
+        404,
+        None
+    ),
+])
+def test_fetch_latest_build_status_for_connector_version_success(mocker, connector_name, connector_version, mocked_json_payload, mocked_status_code, expected_status):
+    # Mock the api call to get the latest build status for a connector version
+    mock_response = MagicMock()
+    mock_response.json.return_value = mocked_json_payload
+    mock_response.status_code = mocked_status_code
+    mock_get = mocker.patch.object(requests, 'get', return_value=mock_response)
+
+    assert inputs.fetch_latest_build_status_for_connector_version(connector_name, connector_version) == expected_status
+    assert mock_get.call_args == call(f"{constants.CONNECTOR_BUILD_OUTPUT_URL}/{connector_name}/{connector_version}.json")

tools/ci_connector_ops/tests/test_qa_engine/test_validations.py

@@ -5,14 +5,16 @@
 
 import pandas as pd
 import pytest
+from unittest.mock import MagicMock, call
+import requests
 
 from ci_connector_ops.qa_engine import enrichments, inputs, models, validations
 
 @pytest.fixture
 def enriched_catalog(oss_catalog, cloud_catalog, adoption_metrics_per_connector_version) -> pd.DataFrame:
     return enrichments.get_enriched_catalog(
-        oss_catalog, 
-        cloud_catalog, 
+        oss_catalog,
+        cloud_catalog,
         adoption_metrics_per_connector_version
     )
 
@@ -108,3 +110,53 @@ def test_get_connectors_eligible_for_cloud(qa_report: pd.DataFrame):
     qa_report["is_eligible_for_promotion_to_cloud"] = False
     connectors_eligible_for_cloud = list(validations.get_connectors_eligible_for_cloud(qa_report))
     assert len(connectors_eligible_for_cloud) == 0
+
+@pytest.mark.parametrize("connector_qa_data, build_file_payload, build_file_status, expected_is_successful", [
+    (
+        pd.Series({
+            "connector_version": "0.1.0",
+            "connector_technical_name": "connectors/source-pokeapi",
+        }),
+        {
+            "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+            "outcome": "success",
+            "docker_version": "0.1.5",
+            "timestamp": "1674872401",
+            "connector": "connectors/source-pokeapi"
+        },
+        200,
+        True
+    ),
+    (
+        pd.Series({
+            "connector_version": "0.1.0",
+            "connector_technical_name": "connectors/source-pokeapi",
+        }),
+        {
+            "link": "https://github.com/airbytehq/airbyte/actions/runs/4029659593",
+            "outcome": "failed",
+            "docker_version": "0.1.5",
+            "timestamp": "1674872401",
+            "connector": "connectors/source-pokeapi"
+        },
+        200,
+        False
+    ),
+    (
+        pd.Series({
+            "connector_version": "0.1.0",
+            "connector_technical_name": "connectors/source-pokeapi",
+        }),
+        None,
+        404,
+        False
+    ),
+])
+def test_latest_build_is_successful(mocker, connector_qa_data: pd.Series, build_file_payload: object, build_file_status: int, expected_is_successful: bool):
+    # Mock the api call to get the latest build status for a connector version
+    mock_response = MagicMock()
+    mock_response.json.return_value = build_file_payload
+    mock_response.status_code = build_file_status
+    mocker.patch.object(requests, 'get', return_value=mock_response)
+
+    assert validations.latest_build_is_successful(connector_qa_data) == expected_is_successful