Skip to content

Commit ab794c1

Browse files
alafanecherealafanechere
committed
airbyte-ci: make airbyte-ci test able to run any poetry run command
1 parent bfe9d8b commit ab794c1

File tree

4 files changed

+30
-16
lines changed

4 files changed

+30
-16
lines changed

‎.github/workflows/airbyte-ci-tests.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
7474
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
7575
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
76-
subcommand: "test airbyte-ci/connectors/connector_ops"
76+
subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'"
7777
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
7878
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
7979

@@ -88,7 +88,7 @@ jobs:
8888
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
8989
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
9090
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
91-
subcommand: "test airbyte-ci/connectors/pipelines"
91+
subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests'"
9292
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
9393
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
9494

@@ -103,7 +103,7 @@ jobs:
103103
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
104104
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
105105
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
106-
subcommand: "test airbyte-ci/connectors/base_images"
106+
subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'"
107107
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
108108
tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
109109

@@ -112,7 +112,7 @@ jobs:
112112
if: steps.changes.outputs.metadata_lib_any_changed == 'true'
113113
uses: ./.github/actions/run-dagger-pipeline
114114
with:
115-
subcommand: "test airbyte-ci/connectors/metadata_service/lib/"
115+
subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'"
116116
context: "pull_request"
117117
github_token: ${{ secrets.GITHUB_TOKEN }}
118118
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
@@ -125,7 +125,7 @@ jobs:
125125
if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
126126
uses: ./.github/actions/run-dagger-pipeline
127127
with:
128-
subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/"
128+
subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'"
129129
context: "pull_request"
130130
github_token: ${{ secrets.GITHUB_TOKEN }}
131131
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}

‎airbyte-ci/connectors/pipelines/README.md

+5-3
Original file line numberDiff line numberDiff line change
@@ -506,17 +506,19 @@ This command runs the Python tests for a airbyte-ci poetry package.
506506

507507
| Option | Required | Default | Mapped environment variable | Description |
508508
| ------------------ | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------------ |
509-
| `--test-directory` | False | tests | | The path to the directory on which pytest should discover tests, relative to the poetry package. |
509+
| `-c/--poetry-run-command` | True | None | | The command to run with `poetry run` |
510510

511511
#### Example
512512

513-
`airbyte-ci test airbyte-ci/connectors/pipelines --test-directory=tests`
514-
`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests`
513+
`airbyte-ci test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests'`
514+
`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test--poetry-run-command='pytest tests/unit_tests'`
515515

516516
## Changelog
517517

518518
| Version | PR | Description |
519519
| ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
520+
| 2.13.0 | [#33784](https://github.com/airbytehq/airbyte/pull/33784) | Make `airbyte-ci test` able to run any poetry command |
521+
| 2.12.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Add upgrade CDK command |
520522
| 2.11.0 | [#32188](https://github.com/airbytehq/airbyte/pull/32188) | Add -x option to connector test to allow for skipping steps |
521523
| 2.10.12 | [#33419](https://github.com/airbytehq/airbyte/pull/33419) | Make ClickPipelineContext handle dagger logging. |
522524
| 2.10.11 | [#33497](https://github.com/airbytehq/airbyte/pull/33497) | Consider nested .gitignore rules in format. |

‎airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py

+19-7
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
#
44

55
import logging
6+
from pathlib import Path
7+
from typing import List
68

79
import asyncclick as click
810
from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
@@ -13,7 +15,13 @@
1315

1416
@click.command()
1517
@click.argument("poetry_package_path")
16-
@click.option("--test-directory", default="tests", help="The directory containing the tests to run.")
18+
@click.option(
19+
"-c",
20+
"--poetry-run-command",
21+
multiple=True,
22+
help="The poetry run command to run.",
23+
required=True,
24+
)
1725
@click_merge_args_into_context_obj
1826
@pass_pipeline_context
1927
@click_ignore_unused_kwargs
@@ -24,7 +32,10 @@ async def test(pipeline_context: ClickPipelineContext):
2432
pipeline_context (ClickPipelineContext): The context object.
2533
"""
2634
poetry_package_path = pipeline_context.params["poetry_package_path"]
27-
test_directory = pipeline_context.params["test_directory"]
35+
if not Path(f"{poetry_package_path}/pyproject.toml").exists():
36+
raise click.UsageError(f"Could not find pyproject.toml in {poetry_package_path}")
37+
38+
commands_to_run: List[str] = pipeline_context.params["poetry_run_command"]
2839

2940
logger = logging.getLogger(f"{poetry_package_path}.tests")
3041
logger.info(f"Running tests for {poetry_package_path}")
@@ -47,7 +58,7 @@ async def test(pipeline_context: ClickPipelineContext):
4758

4859
pipeline_name = f"Unit tests for {poetry_package_path}"
4960
dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name)
50-
pytest_container = await (
61+
test_container = await (
5162
dagger_client.container()
5263
.from_("python:3.10.12")
5364
.with_env_variable("PIPX_BIN_DIR", "/usr/local/bin")
@@ -73,10 +84,11 @@ async def test(pipeline_context: ClickPipelineContext):
7384
),
7485
)
7586
.with_workdir(f"/airbyte/{poetry_package_path}")
76-
.with_exec(["poetry", "install"])
87+
.with_exec(["poetry", "install", "--with=dev"])
7788
.with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock"))
7889
.with_env_variable("CI", str(pipeline_context.params["is_ci"]))
79-
.with_exec(["poetry", "run", "pytest", test_directory])
90+
.with_workdir(f"/airbyte/{poetry_package_path}")
8091
)
81-
82-
await pytest_container
92+
for command in commands_to_run:
93+
test_container = test_container.with_exec(["poetry", "run", *command.split(" ")])
94+
await test_container

‎airbyte-ci/connectors/pipelines/pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "pipelines"
7-
version = "2.12.0"
7+
version = "2.13.0"
88
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
99
authors = ["Airbyte <[email protected]>"]
1010

0 commit comments

Comments
 (0)