airbyte-ci: do no run QA checks on publish - only MetadataValidation (airbytehq#35437)

Co-authored-by: Ella Rohm-Ensing <[email protected]>

Author: 2 people

airbyte-ci/connectors/pipelines/README.md

@@ -644,6 +644,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 4.3.1   | [#35437](https://github.com/airbytehq/airbyte/pull/35437)  | Do not run QA checks on publish, just MetadataValidation.                                                             |
 | 4.3.0   | [#35438](https://github.com/airbytehq/airbyte/pull/35438)  | Optionally disable telemetry with environment variable.                                                                    |
 | 4.2.4   | [#35325](https://github.com/airbytehq/airbyte/pull/35325)  | Use `connectors_qa` for QA checks and remove redundant checks.                                                             |
 | 4.2.3   | [#35322](https://github.com/airbytehq/airbyte/pull/35322)  | Declare `connectors_qa` as an internal package for testing.                                                                |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py

@@ -13,8 +13,7 @@
 from pipelines.airbyte_ci.connectors.build_image import steps
 from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport
-from pipelines.airbyte_ci.connectors.test.steps.common import QaChecks
-from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload
+from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation
 from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry, PythonRegistryPublishContext
 from pipelines.dagger.actions.remote_storage import upload_to_gcs
 from pipelines.dagger.actions.system import docker
@@ -274,11 +273,11 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport:
 
             results = []
 
-            qa_check_results = await QaChecks(context).run()
-            results.append(qa_check_results)
+            metadata_validation_results = await MetadataValidation(context).run()
+            results.append(metadata_validation_results)
 
-            # Exit early if the qa checks do not pass
-            if qa_check_results.status is not StepStatus.SUCCESS:
+            # Exit early if the metadata file is invalid.
+            if metadata_validation_results.status is not StepStatus.SUCCESS:
                 return create_connector_report(results)
 
             check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run()

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.3.0"
+version = "4.3.1"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-ci/connectors/pipelines/tests/test_publish.py

@@ -147,7 +147,7 @@ def test_parse_spec_output_no_spec(self, publish_context):
 
 
 STEPS_TO_PATCH = [
-    (publish_pipeline, "QaChecks"),
+    (publish_pipeline, "MetadataValidation"),
     (publish_pipeline, "MetadataUpload"),
     (publish_pipeline, "CheckConnectorImageDoesNotExist"),
     (publish_pipeline, "UploadSpecToCache"),
@@ -159,29 +159,29 @@ def test_parse_spec_output_no_spec(self, publish_context):
 
 
 @pytest.mark.parametrize("pre_release", [True, False])
-async def test_run_connector_publish_pipeline_when_failed_qa_checks(mocker, pre_release):
-    """We validate that no other steps are called if the qa checks step fails."""
+async def test_run_connector_publish_pipeline_when_failed_validation(mocker, pre_release):
+    """We validate that no other steps are called if the metadata validation step fails."""
     for module, to_mock in STEPS_TO_PATCH:
         mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock())
 
-    run_qa_checks = publish_pipeline.QaChecks.return_value.run
-    run_qa_checks.return_value = mocker.Mock(status=StepStatus.FAILURE)
+    run_metadata_validation = publish_pipeline.MetadataValidation.return_value.run
+    run_metadata_validation.return_value = mocker.Mock(status=StepStatus.FAILURE)
 
     context = mocker.MagicMock(pre_release=pre_release)
     semaphore = anyio.Semaphore(1)
     report = await publish_pipeline.run_connector_publish_pipeline(context, semaphore)
-    run_qa_checks.assert_called_once()
+    run_metadata_validation.assert_called_once()
 
     # Check that nothing else is called
     for module, to_mock in STEPS_TO_PATCH:
-        if to_mock != "QaChecks":
+        if to_mock != "MetadataValidation":
             getattr(module, to_mock).return_value.run.assert_not_called()
 
     assert (
         report.steps_results
         == context.report.steps_results
         == [
-            run_qa_checks.return_value,
+            run_metadata_validation.return_value,
         ]
     )
 
@@ -200,8 +200,8 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker
     for module, to_mock in STEPS_TO_PATCH:
         mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock())
 
-    run_qa_checks = publish_pipeline.QaChecks.return_value.run
-    run_qa_checks.return_value = mocker.Mock(status=StepStatus.SUCCESS)
+    run_metadata_validation = publish_pipeline.MetadataValidation.return_value.run
+    run_metadata_validation.return_value = mocker.Mock(status=StepStatus.SUCCESS)
 
     # ensure spec always succeeds
     run_upload_spec_to_cache = publish_pipeline.UploadSpecToCache.return_value.run
@@ -214,12 +214,12 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker
 
     semaphore = anyio.Semaphore(1)
     report = await publish_pipeline.run_connector_publish_pipeline(publish_context, semaphore)
-    run_qa_checks.assert_called_once()
+    run_metadata_validation.assert_called_once()
     run_check_connector_image_does_not_exist.assert_called_once()
 
     # Check that nothing else is called
     for module, to_mock in STEPS_TO_PATCH:
-        if to_mock not in ["QaChecks", "MetadataUpload", "CheckConnectorImageDoesNotExist", "UploadSpecToCache"]:
+        if to_mock not in ["MetadataValidation", "MetadataUpload", "CheckConnectorImageDoesNotExist", "UploadSpecToCache"]:
             getattr(module, to_mock).return_value.run.assert_not_called()
 
     if check_image_exists_status is StepStatus.SKIPPED:
@@ -228,7 +228,7 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker
             report.steps_results
             == publish_context.report.steps_results
             == [
-                run_qa_checks.return_value,
+                run_metadata_validation.return_value,
                 run_check_connector_image_does_not_exist.return_value,
                 run_upload_spec_to_cache.return_value,
                 run_metadata_upload.return_value,
@@ -241,7 +241,7 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker
             report.steps_results
             == publish_context.report.steps_results
             == [
-                run_qa_checks.return_value,
+                run_metadata_validation.return_value,
                 run_check_connector_image_does_not_exist.return_value,
             ]
         )
@@ -268,10 +268,12 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist(
     upload_to_spec_cache_step_status,
     metadata_upload_step_status,
 ):
-    """We check that the full pipeline is executed as expected when the connector image does not exist and the qa checks passed."""
+    """We check that the full pipeline is executed as expected when the connector image does not exist and the metadata validation passed."""
     for module, to_mock in STEPS_TO_PATCH:
         mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock())
-    publish_pipeline.QaChecks.return_value.run.return_value = mocker.Mock(name="qa_checks_result", status=StepStatus.SUCCESS)
+    publish_pipeline.MetadataValidation.return_value.run.return_value = mocker.Mock(
+        name="metadata_validation_result", status=StepStatus.SUCCESS
+    )
     publish_pipeline.CheckConnectorImageDoesNotExist.return_value.run.return_value = mocker.Mock(
         name="check_connector_image_does_not_exist_result", status=StepStatus.SUCCESS
     )
@@ -306,7 +308,7 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist(
     report = await publish_pipeline.run_connector_publish_pipeline(context, semaphore)
 
     steps_to_run = [
-        publish_pipeline.QaChecks.return_value.run,
+        publish_pipeline.MetadataValidation.return_value.run,
         publish_pipeline.CheckConnectorImageDoesNotExist.return_value.run,
         publish_pipeline.steps.run_connector_build,
         publish_pipeline.PushConnectorImageToRegistry.return_value.run,
@@ -364,7 +366,7 @@ async def test_run_connector_python_registry_publish_pipeline(
     )
 
     for step in [
-        publish_pipeline.QaChecks,
+        publish_pipeline.MetadataValidation,
         publish_pipeline.CheckConnectorImageDoesNotExist,
         publish_pipeline.UploadSpecToCache,
         publish_pipeline.MetadataUpload,