|
8 | 8 | import os |
9 | 9 | from abc import ABC, abstractmethod |
10 | 10 | from functools import cached_property |
11 | | -from typing import Any, ClassVar, List, Optional |
| 11 | +from typing import ClassVar, List, Optional |
12 | 12 |
|
13 | 13 | import requests # type: ignore |
14 | 14 | import semver |
15 | 15 | import yaml # type: ignore |
16 | | -from connector_ops.utils import Connector # type: ignore |
17 | 16 | from dagger import Container, Directory |
18 | 17 | from pipelines import hacks |
19 | 18 | from pipelines.airbyte_ci.connectors.context import ConnectorContext |
20 | | -from pipelines.consts import CIContext |
| 19 | +from pipelines.airbyte_ci.steps.docker import SimpleDockerStep |
| 20 | +from pipelines.consts import INTERNAL_TOOL_PATHS, CIContext |
21 | 21 | from pipelines.dagger.actions import secrets |
22 | | -from pipelines.dagger.containers import internal_tools |
23 | 22 | from pipelines.helpers.utils import METADATA_FILE_NAME |
24 | | -from pipelines.models.steps import STEP_PARAMS, Step, StepResult, StepStatus |
| 23 | +from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus |
25 | 24 |
|
26 | 25 |
|
27 | 26 | class VersionCheck(Step, ABC): |
@@ -119,71 +118,37 @@ def validate(self) -> StepResult: |
119 | 118 | return self.success_result |
120 | 119 |
|
121 | 120 |
|
122 | | -class VersionFollowsSemverCheck(VersionCheck): |
123 | | - context: ConnectorContext |
124 | | - title = "Connector version semver check" |
125 | | - |
126 | | - @property |
127 | | - def failure_message(self) -> str: |
128 | | - return f"The dockerImageTag in {METADATA_FILE_NAME} is not following semantic versioning or was decremented. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" |
129 | | - |
130 | | - def validate(self) -> StepResult: |
131 | | - try: |
132 | | - if not self.current_connector_version >= self.master_connector_version: |
133 | | - return self.failure_result |
134 | | - except ValueError: |
135 | | - return self.failure_result |
136 | | - return self.success_result |
137 | | - |
138 | | - |
139 | | -class QaChecks(Step): |
140 | | - """A step to run QA checks for a connector.""" |
141 | | - |
142 | | - context: ConnectorContext |
143 | | - title = "QA checks" |
144 | | - |
145 | | - async def _run(self) -> StepResult: |
146 | | - """Run QA checks on a connector. |
147 | | -
|
148 | | - The QA checks are defined in this module: |
149 | | - https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connector_ops/connector_ops/qa_checks.py |
150 | | -
|
151 | | - Args: |
152 | | - context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. |
153 | | - Returns: |
154 | | - StepResult: Failure or success of the QA checks with stdout and stderr. |
155 | | - """ |
156 | | - connector_ops = await internal_tools.with_connector_ops(self.context) |
157 | | - include = [ |
158 | | - str(self.context.connector.code_directory), |
159 | | - str(self.context.connector.documentation_file_path), |
160 | | - str(self.context.connector.migration_guide_file_path), |
161 | | - str(self.context.connector.icon_path), |
162 | | - ] |
163 | | - if ( |
164 | | - self.context.connector.technical_name.endswith("strict-encrypt") |
165 | | - or self.context.connector.technical_name == "source-file-secure" |
166 | | - ): |
167 | | - original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", "")) |
168 | | - include += [ |
169 | | - str(original_connector.code_directory), |
170 | | - str(original_connector.documentation_file_path), |
171 | | - str(original_connector.icon_path), |
172 | | - str(original_connector.migration_guide_file_path), |
173 | | - ] |
174 | | - |
175 | | - filtered_repo = self.context.get_repo_dir( |
176 | | - include=include, |
| 121 | +class QaChecks(SimpleDockerStep): |
| 122 | + """A step to run QA checks for a connectors. |
| 123 | + More details in https://github.com/airbytehq/airbyte/blob/main/airbyte-ci/connectors/connectors_qa/README.md |
| 124 | + """ |
| 125 | + |
| 126 | + def __init__(self, context: ConnectorContext) -> None: |
| 127 | + super().__init__( |
| 128 | + title=f"Run QA checks for {context.connector.technical_name}", |
| 129 | + context=context, |
| 130 | + paths_to_mount=[ |
| 131 | + MountPath(context.connector.code_directory), |
| 132 | + # These paths are optional |
| 133 | + # But their absence might make the QA check fail |
| 134 | + MountPath(context.connector.documentation_file_path, optional=True), |
| 135 | + MountPath(context.connector.migration_guide_file_path, optional=True), |
| 136 | + MountPath(context.connector.icon_path, optional=True), |
| 137 | + ], |
| 138 | + internal_tools=[ |
| 139 | + MountPath(INTERNAL_TOOL_PATHS.CONNECTORS_QA.value), |
| 140 | + ], |
| 141 | + secrets={ |
| 142 | + k: v |
| 143 | + for k, v in { |
| 144 | + "DOCKER_HUB_USERNAME": context.docker_hub_username_secret, |
| 145 | + "DOCKER_HUB_PASSWORD": context.docker_hub_password_secret, |
| 146 | + }.items() |
| 147 | + if v |
| 148 | + }, |
| 149 | + command=["connectors-qa", "run", f"--name={context.connector.technical_name}"], |
177 | 150 | ) |
178 | 151 |
|
179 | | - qa_checks = ( |
180 | | - connector_ops.with_mounted_directory("/airbyte", filtered_repo) |
181 | | - .with_workdir("/airbyte") |
182 | | - .with_exec(["run-qa-checks", f"connectors/{self.context.connector.technical_name}"]) |
183 | | - ) |
184 | | - |
185 | | - return await self.get_step_result(qa_checks) |
186 | | - |
187 | 152 |
|
188 | 153 | class AcceptanceTests(Step): |
189 | 154 | """A step to run acceptance tests for a connector if it has an acceptance test config file.""" |
@@ -318,58 +283,3 @@ async def _build_connector_acceptance_test(self, connector_under_test_container: |
318 | 283 | ) |
319 | 284 |
|
320 | 285 | return cat_container.with_unix_socket("/var/run/docker.sock", self.context.dagger_client.host().unix_socket("/var/run/docker.sock")) |
321 | | - |
322 | | - |
323 | | -class CheckBaseImageIsUsed(Step): |
324 | | - context: ConnectorContext |
325 | | - title = "Check our base image is used" |
326 | | - |
327 | | - async def _run(self, *args: Any, **kwargs: Any) -> StepResult: |
328 | | - is_certified = self.context.connector.metadata.get("supportLevel") == "certified" |
329 | | - if not is_certified: |
330 | | - return self.skip("Connector is not certified, it does not require the use of our base image.") |
331 | | - |
332 | | - is_using_base_image = self.context.connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is not None |
333 | | - migration_hint = f"Please run 'airbyte-ci connectors --name={self.context.connector.technical_name} migrate_to_base_image <PR NUMBER>' and commit the changes." |
334 | | - if not is_using_base_image: |
335 | | - return StepResult( |
336 | | - step=self, |
337 | | - status=StepStatus.FAILURE, |
338 | | - stdout=f"Connector is certified but does not use our base image. {migration_hint}", |
339 | | - ) |
340 | | - has_dockerfile = "Dockerfile" in await (await self.context.get_connector_dir(include=["Dockerfile"])).entries() |
341 | | - if has_dockerfile: |
342 | | - return StepResult( |
343 | | - step=self, |
344 | | - status=StepStatus.FAILURE, |
345 | | - stdout=f"Connector is certified but is still using a Dockerfile. {migration_hint}", |
346 | | - ) |
347 | | - return StepResult(step=self, status=StepStatus.SUCCESS, stdout="Connector is certified and uses our base image.") |
348 | | - |
349 | | - |
350 | | -class CheckPythonRegistryPublishConfiguration(Step): |
351 | | - context: ConnectorContext |
352 | | - title = "Check connector is published to python registry if it's a certified python connector" |
353 | | - |
354 | | - async def _run(self, *args: Any, **kwargs: Any) -> StepResult: |
355 | | - is_python_registry_published = self.context.connector.metadata.get("remoteRegistries", {}).get("pypi", {}).get("enabled", False) |
356 | | - if is_python_registry_published: |
357 | | - return StepResult(step=self, status=StepStatus.SUCCESS, stdout="Connector is published to PyPI.") |
358 | | - |
359 | | - tags = self.context.connector.metadata.get("tags", []) |
360 | | - is_python_registry_compatible = ("language:python" in tags or "language:low-code" in tags) and "language:java" not in tags |
361 | | - is_certified = self.context.connector.metadata.get("supportLevel") == "certified" |
362 | | - is_source = self.context.connector.metadata.get("connectorType") == "source" |
363 | | - if not is_source or not is_certified or not is_python_registry_compatible: |
364 | | - return self.skip( |
365 | | - "Connector is not a certified python source connector, it does not require to be published to python registry." |
366 | | - ) |
367 | | - |
368 | | - migration_hint = "Check the airbyte-ci readme under https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/pipelines#python-registry-publishing for how to configure publishing." |
369 | | - if not is_python_registry_published: |
370 | | - return StepResult( |
371 | | - step=self, |
372 | | - status=StepStatus.FAILURE, |
373 | | - stdout=f"Connector is a certified python source but publication to PyPI is not enabled. {migration_hint}", |
374 | | - ) |
375 | | - return StepResult(step=self, status=StepStatus.SUCCESS, stdout="Connector is a certified python source and is published to PyPI.") |
0 commit comments