Skip to content

Commit 4ef8db6

Browse files
alafanecherealafanechere
authored
community-ci: declare "early CI" (#37690)
1 parent a7102c9 commit 4ef8db6

File tree

5 files changed

+81
-12
lines changed

5 files changed

+81
-12
lines changed

‎.github/workflows/community_ci.yml

+56-6
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,13 @@ jobs:
4242

4343
# This will sync the .github folder of the main repo with the fork
4444
# This allows us to use up to date actions from the main repo
45-
- name: Pull .github folder from main repository
45+
- name: Pull .github folder and internal packages from main repository
4646
id: pull_github_folder
4747
run: |
4848
git remote add main https://github.com/airbytehq/airbyte.git
4949
git fetch main ${MAIN_BRANCH_NAME}
5050
git checkout main/${MAIN_BRANCH_NAME} -- .github
51+
git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
5152
5253
- name: Run airbyte-ci format check all
5354
# This path refers to the fork .github folder.
@@ -58,8 +59,56 @@ jobs:
5859
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
5960
subcommand: "format check all"
6061
is_fork: "true"
61-
connectors_test:
62-
name: Run connectors tests on fork
62+
connectors_early_ci:
63+
name: Run connectors early CI on fork
64+
if: github.event.pull_request.head.repo.fork == true
65+
environment: community-ci-auto
66+
runs-on: community-tooling-test-small
67+
timeout-minutes: 10
68+
env:
69+
MAIN_BRANCH_NAME: "master"
70+
permissions:
71+
statuses: write
72+
steps:
73+
# This checkouts a fork which can contain untrusted code
74+
# It's deemed safe as the static checks are not executing any checked out code
75+
- name: Checkout fork
76+
uses: actions/checkout@v4
77+
with:
78+
repository: ${{ github.event.pull_request.head.repo.full_name }}
79+
ref: ${{ github.event.pull_request.head.sha }}
80+
fetch-depth: 1
81+
82+
# This will sync the .github folder of the main repo with the fork
83+
# This allows us to use up to date actions and internal packages logic from the main repo
84+
# It will also prevent forks from changing CI logic
85+
- name: Pull .github folder from main repository
86+
id: pull_github_folder
87+
run: |
88+
git remote add main https://github.com/airbytehq/airbyte.git
89+
git fetch main ${MAIN_BRANCH_NAME}
90+
git checkout main/${MAIN_BRANCH_NAME} -- .github
91+
git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
92+
93+
- name: Run airbyte-ci static checks and version increment checks on modified connectors
94+
# This path refers to the fork .github folder.
95+
# We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
96+
uses: ./.github/actions/run-airbyte-ci
97+
with:
98+
context: "pull_request"
99+
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
100+
subcommand: "connectors --modified test --only-step=qa_checks --only-step=version_inc_check --global-status-check-context='Connectors early CI checks' --global-status-check-description='Running early CI checks on connectors'"
101+
is_fork: "true"
102+
- name: Upload pipeline reports
103+
id: upload-artifact
104+
uses: actions/upload-artifact@v4
105+
with:
106+
name: early-ci-pipeline-reports
107+
path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html
108+
retention-days: 7
109+
110+
connectors_full_ci:
111+
name: Run all connectors tests on fork
63112
if: github.event.pull_request.head.repo.fork == true
64113
# Deployment of jobs on the community-ci environment requires manual approval
65114
# This is something we set up in the GitHub environment settings:
@@ -83,14 +132,15 @@ jobs:
83132
ref: ${{ github.event.pull_request.head.sha }}
84133
fetch-depth: 1
85134

86-
# This will sync the .github folder of the main repo with the fork
87-
# This allows us to use up to date actions from the main repo
88-
- name: Pull .github folder from main repository
135+
# This will sync the .github folder of the main repo with the fork
136+
# This allows us to use up to date actions and internal packages logic from the main repo
137+
# It will also prevent forks from changing CI logic - name: Pull .github folder from main repository
89138
id: pull_github_folder
90139
run: |
91140
git remote add main https://github.com/airbytehq/airbyte.git
92141
git fetch main ${MAIN_BRANCH_NAME}
93142
git checkout main/${MAIN_BRANCH_NAME} -- .github
143+
git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
94144
95145
- name: Run airbyte-ci connectors test
96146
# This path refers to the fork .github folder.

‎airbyte-ci/connectors/pipelines/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
676676

677677
| Version | PR | Description |
678678
|---------| ---------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------|
679+
| 4.12.0 | [#37690](https://github.com/airbytehq/airbyte/pull/37690) | Pass custom CI status name in `connectors test` |
679680
| 4.11.0 | [#37641](https://github.com/airbytehq/airbyte/pull/37641) | Updates to run regression tests in GitHub Actions. |
680681
| 4.10.5 | [#37641](https://github.com/airbytehq/airbyte/pull/37641) | Reintroduce changes from 4.10.0 with a fix. |
681682
| 4.10.4 | [#37641](https://github.com/airbytehq/airbyte/pull/37641) | Temporarily revert changes from version 4.10.0 |

‎airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py

+21-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
from pipelines.helpers.utils import fail_if_missing_docker_hub_creds
2222
from pipelines.models.steps import STEP_PARAMS
2323

24+
GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
25+
GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
26+
2427

2528
@click.command(
2629
cls=DaggerPipelineCommand,
@@ -67,6 +70,18 @@
6770
type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]),
6871
help="Only run specific step by name. Can be used multiple times to keep multiple steps.",
6972
)
73+
@click.option(
74+
"--global-status-check-context",
75+
"global_status_check_context",
76+
help="The context of the global status check which will be sent to GitHub status API.",
77+
default=GITHUB_GLOBAL_CONTEXT_FOR_TESTS,
78+
)
79+
@click.option(
80+
"--global-status-check-description",
81+
"global_status_check_description",
82+
help="The description of the global status check which will be sent to GitHub status API.",
83+
default=GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS,
84+
)
7085
@click.argument(
7186
"extra_params", nargs=-1, type=click.UNPROCESSED, callback=argument_parsing.build_extra_params_mapping(CONNECTOR_TEST_STEP_ID)
7287
)
@@ -78,13 +93,18 @@ async def test(
7893
concurrent_cat: bool,
7994
skip_steps: List[str],
8095
only_steps: List[str],
96+
global_status_check_context: str,
97+
global_status_check_description: str,
8198
extra_params: Dict[CONNECTOR_TEST_STEP_ID, STEP_PARAMS],
8299
) -> bool:
83100
"""Runs a test pipeline for the selected connectors.
84101
85102
Args:
86103
ctx (click.Context): The click context.
87104
"""
105+
ctx.obj["global_status_check_context"] = global_status_check_context
106+
ctx.obj["global_status_check_description"] = global_status_check_description
107+
88108
if only_steps and skip_steps:
89109
raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.")
90110
if not only_steps:
@@ -109,7 +129,7 @@ async def test(
109129

110130
connectors_tests_contexts = [
111131
ConnectorContext(
112-
pipeline_name=f"Testing connector {connector.technical_name}",
132+
pipeline_name=f"{global_status_check_context} on {connector.technical_name}",
113133
connector=connector,
114134
is_local=ctx.obj["is_local"],
115135
git_branch=ctx.obj["git_branch"],

‎airbyte-ci/connectors/pipelines/pipelines/helpers/github.py

+2-4
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919
from github import Github, PullRequest
2020

2121
AIRBYTE_GITHUB_REPO = "airbytehq/airbyte"
22-
GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
23-
GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
2422

2523

2624
def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> None:
@@ -104,8 +102,8 @@ def update_global_commit_status_check_for_tests(click_context: dict, github_stat
104102
click_context["git_revision"],
105103
github_state,
106104
click_context["gha_workflow_run_url"],
107-
GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS,
108-
GITHUB_GLOBAL_CONTEXT_FOR_TESTS,
105+
click_context["global_status_check_description"],
106+
click_context["global_status_check_context"],
109107
should_send=click_context.get("ci_context") == CIContext.PULL_REQUEST,
110108
logger=logger,
111109
)

‎airbyte-ci/connectors/pipelines/pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "pipelines"
7-
version = "4.11.0"
7+
version = "4.12.0"
88
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
99
authors = ["Airbyte <[email protected]>"]
1010

0 commit comments

Comments
 (0)