Skip to content

Commit e395c7b

Browse files
JgerardopineJgerardopine
authored
Merge branch 'master' into jgpines/paypal-trans-silver-cert
2 parents 97ee14c + 58de015 commit e395c7b

File tree

1,908 files changed

+34825
-23939
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,908 files changed

+34825
-23939
lines changed

‎.bumpversion.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.50.44
2+
current_version = 0.50.47
33
commit = False
44
tag = False
55
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

‎.github/CODEOWNERS

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
# CDK and Connector Acceptance Tests
1010
/airbyte-cdk/python @airbytehq/connector-extensibility
1111
/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
12-
/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-operations @lazebnyi @oustynova
12+
/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-extensibility @lazebnyi @oustynova
1313

1414
# Protocol related items
1515
/docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
@@ -62,7 +62,7 @@ airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations
6262
/airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations
6363

6464
# Build customization file change
65-
/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-operations
65+
/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-extensibility
6666

6767
# airbyte-ci
68-
/airbyte-ci @airbytehq/connector-operations
68+
/airbyte-ci @airbytehq/connector-extensibility

‎.github/actions/airbyte-ci-requirements/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ runs:
8484
id: get-dagger-version
8585
shell: bash
8686
run: |
87-
dagger_version=$(airbyte-ci ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
87+
dagger_version=$(airbyte-ci --disable-update-check ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
8888
echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT"
8989
9090
- name: Get runner name

‎.github/actions/get-dagger-engine-image/action.yml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: "Get Dagger Engine Image"
2+
description: "Pulls the Dagger Engine Image or load from cache"
3+
4+
inputs:
5+
dagger_engine_image:
6+
description: "Image name of the Dagger Engine"
7+
required: true
8+
path_to_dagger_engine_image_cache:
9+
description: "Path to the Dagger Engine image cache"
10+
required: false
11+
default: "/home/runner/dagger-engine-image-cache"
12+
13+
runs:
14+
using: "composite"
15+
steps:
16+
- name: Create local image cache directory
17+
id: create-dagger-engine-image-cache-dir
18+
shell: bash
19+
run: mkdir -p ${{ inputs.path_to_dagger_engine_image_cache }}
20+
21+
- name: Restore dagger engine image cache
22+
id: dagger-engine-image-cache-restore
23+
uses: actions/cache/restore@v4
24+
with:
25+
path: ${{ inputs.path_to_dagger_engine_image_cache }}
26+
key: ${{ inputs.dagger_engine_image }}
27+
28+
# If no GitHub Action cache hit, pull the image and save it locally as tar to the cache directory
29+
- name: Pull dagger engine image
30+
id: pull-dagger-engine-image
31+
if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
32+
shell: bash
33+
run: |
34+
set -x
35+
docker pull ${{ inputs.dagger_engine_image }}
36+
docker save -o ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar ${{ inputs.dagger_engine_image }}
37+
38+
# If no GitHub Action cache hit, save the path to the image cache directory to the Github Action cache
39+
- name: Save dagger engine image cache
40+
id: dagger-engine-image-cache-save
41+
if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true'
42+
uses: actions/cache/save@v4
43+
with:
44+
path: ${{ inputs.path_to_dagger_engine_image_cache }}
45+
key: ${{ inputs.dagger_engine_image }}
46+
47+
# If GitHub Action cache hit, load the image tar restored from the cache
48+
- name: Load dagger engine image from cache
49+
if: steps.dagger-engine-image-cache-restore.outputs.cache-hit == 'true'
50+
shell: bash
51+
run: |
52+
set -x
53+
docker load -i ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar

‎.github/actions/install-airbyte-ci/action.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: "Install Airbyte CI"
2+
description: "Install Airbyte CI from source or from a binary according to changed files. Pulls the Dagger Engine image according to the dagger version used in airbyte-ci."
3+
4+
inputs:
5+
airbyte_ci_binary_url:
6+
description: "URL to airbyte-ci binary"
7+
required: false
8+
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
9+
path_to_airbyte_ci_source:
10+
description: "Path to airbyte-ci source"
11+
required: false
12+
default: airbyte-ci/connectors/pipelines
13+
runs:
14+
using: "composite"
15+
steps:
16+
- name: Get changed files
17+
uses: tj-actions/changed-files@v39
18+
id: changes
19+
with:
20+
files_yaml: |
21+
pipelines:
22+
- '${{ inputs.path_to_airbyte_ci_source }}/**'
23+
24+
- name: Determine how Airbyte CI should be installed
25+
shell: bash
26+
id: determine-install-mode
27+
run: |
28+
if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
29+
echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
30+
echo "install-mode=source" >> $GITHUB_OUTPUT
31+
else
32+
echo "install-mode=binary" >> $GITHUB_OUTPUT
33+
fi
34+
35+
- name: Install Airbyte CI from binary
36+
id: install-airbyte-ci-binary
37+
if: steps.determine-install-mode.outputs.install-mode == 'binary'
38+
shell: bash
39+
run: |
40+
curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
41+
sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
42+
sudo chmod +x /usr/local/bin/airbyte-ci
43+
44+
- name: Install Python 3.10
45+
id: install-python-3-10
46+
uses: actions/setup-python@v4
47+
if: steps.determine-install-mode.outputs.install-mode == 'source'
48+
with:
49+
python-version: "3.10"
50+
token: ${{ inputs.github_token }}
51+
52+
- name: Install Airbyte CI from source
53+
id: install-airbyte-ci-source
54+
if: steps.determine-install-mode.outputs.install-mode == 'source'
55+
shell: bash
56+
run: |
57+
pip install --upgrade pip
58+
pip install pipx
59+
pipx ensurepath
60+
pipx install ${{ inputs.path_to_airbyte_ci_source }}
61+
62+
- name: Get dagger engine image name
63+
id: get-dagger-engine-image-name
64+
shell: bash
65+
run: |
66+
dagger_engine_image=$(airbyte-ci --ci-requirements | tail -n 1 | jq -r '.dagger_engine_image')
67+
echo "dagger_engine_image=${dagger_engine_image}" >> "$GITHUB_OUTPUT"
68+
69+
- name: Get dagger engine image
70+
id: get-dagger-engine-image
71+
uses: ./.github/actions/get-dagger-engine-image
72+
with:
73+
dagger_engine_image: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}
74+
75+
outputs:
76+
install_mode:
77+
description: "Whether Airbyte CI was installed from source or from a binary"
78+
value: ${{ steps.determine-install-mode.outputs.install-mode }}
79+
dagger_engine_image_name:
80+
description: "Dagger engine image name"
81+
value: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }}

‎.github/actions/run-dagger-pipeline/action.yml renamed to ‎.github/actions/run-airbyte-ci/action.yml

Lines changed: 30 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,6 @@ inputs:
1919
docker_hub_password:
2020
description: "Dockerhub password"
2121
required: true
22-
docker_registry_mirror_url:
23-
description: "Docker registry mirror URL (not including http or https)"
24-
required: false
25-
# Do not use http or https here
26-
default: "ci-dockerhub-registry.airbyte.com"
2722
options:
2823
description: "Options for the subcommand"
2924
required: false
@@ -77,17 +72,24 @@ inputs:
7772
s3_build_cache_secret_key:
7873
description: "Gradle S3 Build Cache AWS secret key"
7974
required: false
80-
tailscale_auth_key:
81-
description: "Tailscale auth key"
8275
airbyte_ci_binary_url:
8376
description: "URL to airbyte-ci binary"
8477
required: false
8578
default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
79+
python_registry_token:
80+
description: "Python registry API token to publish python package"
81+
required: false
8682

8783
runs:
8884
using: "composite"
8985
steps:
86+
- name: Get start timestamp
87+
id: get-start-timestamp
88+
shell: bash
89+
run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
90+
9091
- name: Check if PR is from a fork
92+
id: check-if-pr-is-from-fork
9193
if: github.event_name == 'pull_request'
9294
shell: bash
9395
run: |
@@ -96,65 +98,24 @@ runs:
9698
exit 78
9799
fi
98100
99-
- name: Get changed files
100-
uses: tj-actions/changed-files@v39
101-
id: changes
102-
with:
103-
files_yaml: |
104-
pipelines:
105-
- 'airbyte-ci/connectors/pipelines/**'
106-
107-
- name: Determine how Airbyte CI should be installed
108-
shell: bash
109-
id: determine-install-mode
110-
run: |
111-
if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
112-
echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
113-
echo "install-mode=dev" >> $GITHUB_OUTPUT
114-
else
115-
echo "install-mode=production" >> $GITHUB_OUTPUT
116-
fi
117-
118101
- name: Docker login
119-
uses: docker/login-action@v1
102+
id: docker-login
103+
uses: docker/login-action@v3
120104
with:
121105
username: ${{ inputs.docker_hub_username }}
122106
password: ${{ inputs.docker_hub_password }}
123107

124-
- name: Get start timestamp
125-
id: get-start-timestamp
126-
shell: bash
127-
run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
128-
129-
- name: Install airbyte-ci binary
108+
- name: Install Airbyte CI
130109
id: install-airbyte-ci
131-
if: steps.determine-install-mode.outputs.install-mode == 'production'
132-
shell: bash
133-
run: |
134-
curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
135-
sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
136-
sudo chmod +x /usr/local/bin/airbyte-ci
137-
138-
- name: Install Python 3.10
139-
uses: actions/setup-python@v4
140-
if: steps.determine-install-mode.outputs.install-mode == 'dev'
110+
uses: ./.github/actions/install-airbyte-ci
141111
with:
142-
python-version: "3.10"
143-
token: ${{ inputs.github_token }}
144-
145-
- name: Install ci-connector-ops package
146-
if: steps.determine-install-mode.outputs.install-mode == 'dev'
147-
shell: bash
148-
run: |
149-
pip install pipx
150-
pipx ensurepath
151-
pipx install airbyte-ci/connectors/pipelines/
112+
airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url }}
152113

153114
- name: Run airbyte-ci
115+
id: run-airbyte-ci
154116
shell: bash
155117
run: |
156-
export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
157-
airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
118+
airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
158119
env:
159120
CI_CONTEXT: "${{ inputs.context }}"
160121
CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }}
@@ -163,22 +124,29 @@ runs:
163124
CI_JOB_KEY: ${{ inputs.ci_job_key }}
164125
CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
165126
CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }}
127+
CI: "True"
166128
DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}"
129+
DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }}
130+
DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }}
167131
GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }}
168132
GCS_CREDENTIALS: ${{ inputs.gcs_credentials }}
169133
METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }}
170134
METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }}
171135
PRODUCTION: ${{ inputs.production }}
172136
PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
137+
PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
138+
PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }}
139+
PYTHON_REGISTRY_CHECK_URL: ${{ inputs.python_registry_check_url }}
140+
S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
141+
S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
173142
SENTRY_DSN: ${{ inputs.sentry_dsn }}
174143
SENTRY_ENVIRONMENT: ${{ steps.determine-install-mode.outputs.install-mode }}
175144
SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
176145
SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
177146
SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
178-
DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }}
179-
DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }}
180-
S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
181-
S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
182-
CI: "True"
183-
TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }}
184-
DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }}
147+
# give the Dagger Engine more time to push cache data to Dagger Cloud
148+
- name: Stop Engine
149+
id: stop-engine
150+
if: always()
151+
shell: bash
152+
run: docker stop --time 300 $(docker ps --filter name="dagger-engine-*" -q)

0 commit comments

Comments
 (0)