Skip to content

Commit 5de126d

Browse files
PhlairPhlair
committed
Merge branch 'master' into george/fb-marketing
2 parents 3cc9060 + c570225 commit 5de126d

File tree

151 files changed

+2589
-756
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

151 files changed

+2589
-756
lines changed

‎.bumpversion.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.36.5-alpha
2+
current_version = 0.36.6-alpha
33
commit = False
44
tag = False
55
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

‎.env

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111

1212
### SHARED ###
13-
VERSION=0.36.5-alpha
13+
VERSION=0.36.6-alpha
1414

1515
# When using the airbyte-db via default docker image
1616
CONFIG_ROOT=/data

‎.github/workflows/build-connector-command.yml

+260
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
name: Bump, Build, Test Connectors [EXPERIMENTAL]
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
repo:
6+
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
7+
required: false
8+
default: "airbytehq/airbyte"
9+
gitref:
10+
description: "The git ref to check out from the specified repository."
11+
required: false
12+
default: master
13+
connector:
14+
description: "Airbyte Connector"
15+
required: true
16+
bump-version:
17+
description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually"
18+
required: false
19+
default: "false"
20+
run-tests:
21+
description: "Should run tests"
22+
required: false
23+
default: "true"
24+
comment-id:
25+
description: "The comment-id of the slash command. Used to update the comment with the status."
26+
required: false
27+
28+
jobs:
29+
find_valid_pat:
30+
name: "Find a PAT with room for actions"
31+
timeout-minutes: 10
32+
runs-on: ubuntu-latest
33+
outputs:
34+
pat: ${{ steps.variables.outputs.pat }}
35+
steps:
36+
- name: Checkout Airbyte
37+
uses: actions/checkout@v2
38+
- name: Check PAT rate limits
39+
id: variables
40+
run: |
41+
./tools/bin/find_non_rate_limited_PAT \
42+
${{ secrets.AIRBYTEIO_PAT }} \
43+
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
44+
${{ secrets.SUPERTOPHER_PAT }} \
45+
${{ secrets.DAVINCHIA_PAT }}
46+
## Gradle Build
47+
# In case of self-hosted EC2 errors, remove this block.
48+
start-bump-build-test-connector-runner:
49+
name: Start Build EC2 Runner
50+
runs-on: ubuntu-latest
51+
needs: find_valid_pat
52+
outputs:
53+
label: ${{ steps.start-ec2-runner.outputs.label }}
54+
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
55+
steps:
56+
- name: Checkout Airbyte
57+
uses: actions/checkout@v2
58+
with:
59+
repository: ${{ github.event.inputs.repo }}
60+
ref: ${{ github.event.inputs.gitref }}
61+
- name: Start AWS Runner
62+
id: start-ec2-runner
63+
uses: ./.github/actions/start-aws-runner
64+
with:
65+
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
66+
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
67+
github-token: ${{ needs.find_valid_pat.outputs.pat }}
68+
# 80 gb disk
69+
ec2-image-id: ami-0d648081937c75a73
70+
bump-build-test-connector:
71+
name: Bump, Build, Test Connector
72+
needs: start-bump-build-test-connector-runner
73+
runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
74+
environment: more-secrets
75+
steps:
76+
############################
77+
## SET UP ##
78+
############################
79+
- name: Set up Cloud SDK
80+
uses: google-github-actions/setup-gcloud@v0
81+
with:
82+
service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
83+
export_default_credentials: true
84+
- name: Search for valid connector name format
85+
id: regex
86+
uses: AsasInnab/regex-action@v1
87+
with:
88+
regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$"
89+
regex_flags: "i" # required to be set for this plugin
90+
search_string: ${{ github.event.inputs.connector }}
91+
- name: Validate input workflow format
92+
if: steps.regex.outputs.first_match != github.event.inputs.connector
93+
run: echo "The connector provided has an invalid format!" && exit 1
94+
- name: Link comment to workflow run
95+
if: github.event.inputs.comment-id
96+
uses: peter-evans/create-or-update-comment@v1
97+
with:
98+
comment-id: ${{ github.event.inputs.comment-id }}
99+
body: |
100+
> :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
101+
- name: Checkout Airbyte
102+
uses: actions/checkout@v2
103+
with:
104+
repository: ${{ github.event.inputs.repo }}
105+
ref: ${{ github.event.inputs.gitref }}
106+
token: ${{ secrets.OCTAVIA_PAT }}
107+
- name: Install Java
108+
uses: actions/setup-java@v1
109+
with:
110+
java-version: "17"
111+
- name: Install Python
112+
uses: actions/setup-python@v2
113+
with:
114+
python-version: "3.9"
115+
- name: Install Pyenv and Tox
116+
run: |
117+
python3 -m pip install --quiet virtualenv==16.7.9 --user
118+
python3 -m virtualenv venv
119+
source venv/bin/activate
120+
pip install --quiet tox==3.24.4
121+
- name: Install yq
122+
if: github.event.inputs.bump-version != 'false' && success()
123+
run: |
124+
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
125+
sudo add-apt-repository ppa:rmescandon/yq
126+
sudo apt update
127+
sudo apt install yq -y
128+
- name: Test and install CI scripts
129+
# all CI python packages have the prefix "ci_"
130+
run: |
131+
source venv/bin/activate
132+
tox -r -c ./tools/tox_ci.ini
133+
pip install --quiet -e ./tools/ci_*
134+
- name: Get Credentials for ${{ github.event.inputs.connector }}
135+
run: |
136+
source venv/bin/activate
137+
ci_credentials ${{ github.event.inputs.connector }}
138+
env:
139+
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
140+
# TODO: seems like this should run in post-merge workflow
141+
# - name: Prepare Sentry
142+
# if: startsWith(github.event.inputs.connector, 'connectors')
143+
# run: |
144+
# curl -sL https://sentry.io/get-cli/ | bash
145+
# - name: Create Sentry Release
146+
# if: startsWith(github.event.inputs.connector, 'connectors')
147+
# run: |
148+
# sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing
149+
# env:
150+
# SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
151+
# SENTRY_ORG: airbyte-5j
152+
# SENTRY_PROJECT: airbyte-connectors
153+
############################
154+
## BUMP ##
155+
############################
156+
- name: Bump Connector Version
157+
if: github.event.inputs.bump-version != 'false' && success()
158+
run: ./tools/integrations/manage.sh bump_version airbyte-integrations/${{ github.event.inputs.connector }}
159+
- name: Commit and Push Version Bump
160+
if: github.event.inputs.bump-version != 'false' && success()
161+
run: |
162+
git config user.name 'Octavia Squidington III'
163+
git config user.email '[email protected]'
164+
git add -u
165+
git commit -m "bump-version ${{github.event.inputs.connector}}"
166+
git push origin ${{ github.event.inputs.gitref }}
167+
- name: Add Version Bump Success Comment
168+
if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && success()
169+
uses: peter-evans/create-or-update-comment@v1
170+
with:
171+
comment-id: ${{ github.event.inputs.comment-id }}
172+
body: |
173+
> :rocket: Bumped version for ${{github.event.inputs.connector}}
174+
- name: Add Version Bump Failure Comment
175+
if: github.event.inputs.comment-id && github.event.inputs.bump-version != 'false' && !success()
176+
uses: peter-evans/create-or-update-comment@v1
177+
with:
178+
comment-id: ${{ github.event.inputs.comment-id }}
179+
body: |
180+
> :x: Couldn't bump version for ${{github.event.inputs.connector}}
181+
############################
182+
## BUILD AND TEST ##
183+
############################
184+
- name: Build ${{ github.event.inputs.connector }}
185+
run: ./tools/integrations/manage.sh build_experiment airbyte-integrations/${{ github.event.inputs.connector }}
186+
id: build
187+
env:
188+
PR_NUMBER: ${{ github.event.number }}
189+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
190+
# Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
191+
TZ: UTC
192+
# - name: Test ${{ github.event.inputs.connector }}
193+
# if: github.event.inputs.run-tests == 'true'
194+
# run: ./tools/integrations/manage.sh test airbyte-integrations/${{ github.event.inputs.connector }}
195+
# - name: Finalize Sentry release
196+
# if: startsWith(github.event.inputs.connector, 'connectors')
197+
# run: |
198+
# sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}"
199+
# env:
200+
# SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
201+
# SENTRY_ORG: airbyte-5j
202+
# SENTRY_PROJECT: airbyte-connectors
203+
# - name: Build and Test Success Comment
204+
# if: github.event.inputs.comment-id && success()
205+
# uses: peter-evans/create-or-update-comment@v1
206+
# with:
207+
# comment-id: ${{ github.event.inputs.comment-id }}
208+
# body: |
209+
# > :rocket: Successfully built and tested ${{github.event.inputs.connector}}
210+
# - name: Build and Test Failure Comment
211+
# if: github.event.inputs.comment-id && !success()
212+
# uses: peter-evans/create-or-update-comment@v1
213+
# with:
214+
# comment-id: ${{ github.event.inputs.comment-id }}
215+
# body: |
216+
# > :x: Failed to build and test ${{github.event.inputs.connector}}
217+
# - name: Slack Notification - Failure
218+
# if: failure()
219+
# uses: rtCamp/action-slack-notify@master
220+
# env:
221+
# SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
222+
# SLACK_USERNAME: Buildozer
223+
# SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
224+
# SLACK_COLOR: DC143C
225+
# SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}"
226+
# SLACK_FOOTER: ""
227+
# - name: Add Final Success Comment
228+
# if: github.event.inputs.comment-id && success()
229+
# uses: peter-evans/create-or-update-comment@v1
230+
# with:
231+
# comment-id: ${{ github.event.inputs.comment-id }}
232+
# body: |
233+
# > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
234+
# - name: Set publish label
235+
# if: success()
236+
# run: |
237+
# echo "set some label on PR"
238+
# In case of self-hosted EC2 errors, remove this block.
239+
stop-bump-build-test-connector-runner:
240+
name: Stop Build EC2 Runner
241+
needs:
242+
- start-bump-build-test-connector-runner # required to get output from the start-runner job
243+
- bump-build-test-connector # required to wait when the main job is done
244+
- find_valid_pat
245+
runs-on: ubuntu-latest
246+
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
247+
steps:
248+
- name: Configure AWS credentials
249+
uses: aws-actions/configure-aws-credentials@v1
250+
with:
251+
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
252+
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
253+
aws-region: us-east-2
254+
- name: Stop EC2 runner
255+
uses: supertopher/[email protected]
256+
with:
257+
mode: stop
258+
github-token: ${{ needs.find_valid_pat.outputs.pat }}
259+
label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
260+
ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }}

0 commit comments

Comments
 (0)