airbyte-ci: run live tests on PRs

clnoll · clnoll · commit dd250a46bcd4 · 2024-07-30T14:57:04.000-04:00
diff --git a/.github/actions/run-airbyte-ci/action.yml b/.github/actions/run-airbyte-ci/action.yml
@@ -10,6 +10,9 @@ inputs:
   github_token:
     description: "GitHub token"
     required: false
+  github_maintenance_token:
+    description: "GitHub maintenance token"
+    required: false
   dagger_cloud_token:
     description: "Dagger Cloud token"
     required: false
@@ -121,6 +124,7 @@ runs:
         CI_GIT_REPO_URL: ${{ inputs.git_repo_url }}
         CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }}
         CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }}
+        CI_GITHUB_MAINTENANCE_TOKEN: ${{ inputs.github_maintenance_token }}
         CI_JOB_KEY: ${{ inputs.ci_job_key }}
         CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }}
         DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}"
diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml
@@ -126,10 +126,12 @@ jobs:
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcp_integration_tester_credentials: ${{ secrets.GCLOUD_INTEGRATION_TESTER }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           git_branch: ${{ github.head_ref }}
           git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
           github_token: ${{ env.PAT }}
+          github_maintenance_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -773,6 +773,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.27.0  | [#42574](https://github.com/airbytehq/airbyte/pull/42574)  | Live tests: run from connectors test pipeline for connectors with sandbox connections                                        |
 | 4.26.0  | [#42849](https://github.com/airbytehq/airbyte/pull/42849)  | Send publish failures messages to `#connector-publish-failures`                                                              |
 | 4.25.4  | [#42463](https://github.com/airbytehq/airbyte/pull/42463) | Add validation before live test runs                                                                                         |
 | 4.25.3  | [#42437](https://github.com/airbytehq/airbyte/pull/42437)  | Ugrade-cdk: Update to work with Python connectors using poetry                                                               |
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py
@@ -50,6 +50,7 @@ def __init__(
         ci_gcp_credentials: Optional[Secret] = None,
         ci_git_user: Optional[str] = None,
         ci_github_access_token: Optional[Secret] = None,
+        ci_github_maintenance_token: Optional[Secret] = None,
         connector_acceptance_test_image: str = DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE,
         gha_workflow_run_url: Optional[str] = None,
         dagger_logs_url: Optional[str] = None,
@@ -134,6 +135,7 @@ def __init__(
             ci_gcp_credentials=ci_gcp_credentials,
             ci_git_user=ci_git_user,
             ci_github_access_token=ci_github_access_token,
+            ci_github_maintenance_token=ci_github_maintenance_token,
             run_step_options=run_step_options,
             enable_report_auto_open=enable_report_auto_open,
             secret_stores=secret_stores,
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py
@@ -26,9 +26,6 @@
 GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
 GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
 REGRESSION_TEST_MANUAL_APPROVAL_CONTEXT = "Regression Test Results Reviewed and Approved"
-TESTS_SKIPPED_BY_DEFAULT = [
-    CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,
-]
 
 
 @click.command(
@@ -120,7 +117,6 @@ async def test(
         raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.")
     if not only_steps:
         skip_steps = list(skip_steps)
-        skip_steps += TESTS_SKIPPED_BY_DEFAULT
     if ctx.obj["is_ci"]:
         fail_if_missing_docker_hub_creds(ctx)
 
@@ -150,6 +146,7 @@ async def test(
             git_repo_url=ctx.obj["git_repo_url"],
             ci_git_user=ctx.obj["ci_git_user"],
             ci_github_access_token=ctx.obj["ci_github_access_token"],
+            ci_github_maintenance_token=ctx.obj["ci_github_maintenance_token"],
             ci_report_bucket=ctx.obj["ci_report_bucket_name"],
             report_output_prefix=ctx.obj["report_output_prefix"],
             gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py
@@ -19,6 +19,7 @@
     "unitTests": CONNECTOR_TEST_STEP_ID.UNIT,
     "integrationTests": CONNECTOR_TEST_STEP_ID.INTEGRATION,
     "acceptanceTests": CONNECTOR_TEST_STEP_ID.ACCEPTANCE,
+    "liveTests": CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,
 }
 
 
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py
@@ -13,7 +13,7 @@
 from functools import cached_property
 from pathlib import Path
 from textwrap import dedent
-from typing import ClassVar, List, Optional, Set
+from typing import Any, ClassVar, Dict, List, Optional, Set
 
 import requests  # type: ignore
 import semver
@@ -445,7 +445,7 @@ async def _run(self, current_acceptance_tests_result: StepResult) -> StepResult:
 
 
 class LiveTestSuite(Enum):
-    ALL = "all"
+    ALL = "live"
     REGRESSION = "regression"
     VALIDATION = "validation"
 
@@ -557,13 +557,10 @@ def __init__(self, context: ConnectorContext) -> None:
         self.connector_image = context.docker_image.split(":")[0]
         options = self.context.run_step_options.step_params.get(CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS, {})
 
-        self.connection_id = self.context.run_step_options.get_item_or_default(options, "connection-id", None)
-        self.pr_url = self.context.run_step_options.get_item_or_default(options, "pr-url", None)
-
-        if not self.connection_id and self.pr_url:
-            raise ValueError("`connection-id` and `pr-url` are required to run live tests.")
-
         self.test_suite = self.context.run_step_options.get_item_or_default(options, "test-suite", LiveTestSuite.ALL.value)
+        self.connection_id = self._get_connection_id(options)
+        self.pr_url = self._get_pr_url(options)
+
         self.test_dir = self.test_suite_to_dir[LiveTestSuite(self.test_suite)]
         self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", None)
         self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev")
@@ -573,7 +570,30 @@ def __init__(self, context: ConnectorContext) -> None:
         self.connection_subset = self.context.run_step_options.get_item_or_default(options, "connection-subset", "sandboxes")
         self.run_id = os.getenv("GITHUB_RUN_ID") or str(int(time.time()))
 
+    def _get_connection_id(self, options: Dict[str, List[Any]]) -> Optional[str]:
+        if self.context.is_pr:
+            connection_id = self._get_connection_from_connection_options()
+            self.logger.info(
+                f"Context is {self.context.ci_context}; got connection_id={connection_id} from metadata.yaml liveTests connectionOptions."
+            )
+        else:
+            connection_id = self.context.run_step_options.get_item_or_default(options, "connection-id", None)
+            self.logger.info(f"Context is {self.context.ci_context}; got connection_id={connection_id} from input options.")
+        return connection_id
+
+    def _get_pr_url(self, options: Dict[str, List[Any]]) -> Optional[str]:
+        if self.context.is_pr:
+            pull_request = self.context.pull_request.url if self.context.pull_request else None
+            self.logger.info(f"Context is {self.context.ci_context}; got pull_request={pull_request} from context.")
+        else:
+            pull_request = self.context.run_step_options.get_item_or_default(options, "pr-url", None)
+            self.logger.info(f"Context is {self.context.ci_context}; got pull_request={pull_request} from input options.")
+        return pull_request
+
     def _validate_job_can_run(self) -> None:
+        assert self.connection_id, "`connection-id` is required to run live tests."
+        assert self.pr_url, "`pr_url` is required to run live tests."
+
         connector_type = self.context.connector.metadata.get("connectorType")
         connector_subtype = self.context.connector.metadata.get("connectorSubtype")
         assert connector_type == "source", f"Live tests can only run against source connectors, got `connectorType={connector_type}`."
@@ -582,6 +602,24 @@ def _validate_job_can_run(self) -> None:
                 self.connection_subset == "sandboxes"
             ), f"Live tests for database sources may only be run against sandbox connections, got `connection_subset={self.connection_subset}`."
 
+        if self.context.is_pr:
+            connection_id_is_valid = False
+            for test_suite in self.context.connector.metadata.get("connectorTestSuitesOptions", []):
+                if test_suite["suite"] == "liveTests":
+                    assert self.connection_id in [option["connectionId"] for option in test_suite.get("connectionOptions", [])]
+                    connection_id_is_valid = True
+                    break
+            assert connection_id_is_valid, f"Connection ID {self.connection_id} is not a valid sandbox connection ID."
+
+    def _get_connection_from_connection_options(self) -> Optional[str]:
+        for test_suite in self.context.connector.metadata.get("connectorTestSuitesOptions", []):
+            if test_suite["suite"] == "liveTests":
+                for option in test_suite.get("connectionOptions", []):
+                    connection_id = option["connectionId"]
+                    connection_name = option["connectionName"]
+                    self.logger.info(f"Using connection {connection_name}; connectionId={connection_id}")
+                    return connection_id
+
     async def _run(self, connector_under_test_container: Container) -> StepResult:
         """Run the regression test suite.
 
@@ -676,16 +714,17 @@ async def _build_test_container(self, target_container_id: str) -> Container:
                     ]
                 )
                 .with_secret_variable(
-                    "CI_GITHUB_ACCESS_TOKEN",
+                    "CI_GITHUB_MAINTENANCE_TOKEN",
                     self.context.dagger_client.set_secret(
-                        "CI_GITHUB_ACCESS_TOKEN", self.context.ci_github_access_token.value if self.context.ci_github_access_token else ""
+                        "CI_GITHUB_MAINTENANCE_TOKEN",
+                        self.context.ci_github_maintenance_token.value if self.context.ci_github_maintenance_token else "",
                     ),
                 )
                 .with_exec(
                     [
                         "/bin/sh",
                         "-c",
-                        f"poetry config http-basic.airbyte-platform-internal-source {self.github_user} $CI_GITHUB_ACCESS_TOKEN",
+                        f"poetry config http-basic.airbyte-platform-internal-source {self.github_user} $CI_GITHUB_MAINTENANCE_TOKEN",
                     ]
                 )
                 # Add GCP credentials from the environment and point google to their location (also required for connection-retriever)
diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py
@@ -164,6 +164,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool:
 @click.option("--pull-request-number", envvar="PULL_REQUEST_NUMBER", type=int)
 @click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str)
 @click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str, callback=wrap_in_secret)
+@click.option("--ci-github-maintenance-token", envvar="CI_GITHUB_MAINTENANCE_TOKEN", type=str, callback=wrap_in_secret)
 @click.option("--ci-report-bucket-name", envvar="CI_REPORT_BUCKET_NAME", type=str)
 @click.option("--ci-artifact-bucket-name", envvar="CI_ARTIFACT_BUCKET_NAME", type=str)
 @click.option(
diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py
@@ -83,6 +83,7 @@ def __init__(
         ci_gcp_credentials: Optional[Secret] = None,
         ci_git_user: Optional[str] = None,
         ci_github_access_token: Optional[Secret] = None,
+        ci_github_maintenance_token: Optional[Secret] = None,
         run_step_options: RunStepOptions = RunStepOptions(),
         enable_report_auto_open: bool = True,
         secret_stores: Dict[str, SecretStore] | None = None,
@@ -129,6 +130,7 @@ def __init__(
         self.ci_report_bucket = ci_report_bucket
         self.ci_git_user = ci_git_user
         self.ci_github_access_token = ci_github_access_token
+        self.ci_github_maintenance_token = ci_github_maintenance_token
         self.started_at = None
         self.stopped_at = None
         self.secrets_to_mask = []
diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.26.0"
+version = "4.27.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@`
`19`	`19`	`"unitTests": CONNECTOR_TEST_STEP_ID.UNIT,`
`20`	`20`	`"integrationTests": CONNECTOR_TEST_STEP_ID.INTEGRATION,`
`21`	`21`	`"acceptanceTests": CONNECTOR_TEST_STEP_ID.ACCEPTANCE,`
	`22`	`+ "liveTests": CONNECTOR_TEST_STEP_ID.CONNECTOR_LIVE_TESTS,`
`22`	`23`	`}`
`23`	`24`
`24`	`25`