|
| 1 | +name: Connectors CI - nightly builds for GA and Beta connectors |
| 2 | + |
| 3 | +on: |
| 4 | + schedule: |
| 5 | + # 11AM UTC is 12AM CET, 4AM PST. |
| 6 | + - cron: "0 11 * * *" |
| 7 | + workflow_dispatch: |
| 8 | + inputs: |
| 9 | + runs-on: |
| 10 | + type: string |
| 11 | + default: dev-large-runner |
| 12 | + required: true |
| 13 | + |
| 14 | +run-name: "Connectors CI - nightly builds for GA and Beta connectors - ${{ inputs.runs-on }}" |
| 15 | + |
| 16 | +jobs: |
| 17 | + test_connectors: |
| 18 | + name: Test GA and Beta connectors |
| 19 | + timeout-minutes: 600 # 10 hours |
| 20 | + runs-on: ${{ inputs.runs-on || 'dev-large-runner' }} |
| 21 | + steps: |
| 22 | + - name: Get start timestamp |
| 23 | + id: get-start-timestamp |
| 24 | + run: echo "::set-output name=start-timestamp::$(date +%s)" |
| 25 | + - name: Login to DockerHub |
| 26 | + run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}" |
| 27 | + env: |
| 28 | + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} |
| 29 | + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} |
| 30 | + - name: Checkout Airbyte |
| 31 | + uses: actions/checkout@v3 |
| 32 | + with: |
| 33 | + repository: ${{ github.event.inputs.repo }} |
| 34 | + ref: ${{ github.event.inputs.gitref }} |
| 35 | + - name: Extract branch name |
| 36 | + shell: bash |
| 37 | + if: github.event_name == 'workflow_dispatch' |
| 38 | + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" |
| 39 | + id: extract_branch |
| 40 | + - name: Install Python 3.10 |
| 41 | + uses: actions/setup-python@v4 |
| 42 | + with: |
| 43 | + python-version: "3.10" |
| 44 | + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} |
| 45 | + - name: Install ci-connector-ops package |
| 46 | + run: pip install ./tools/ci_connector_ops\[pipelines]\ |
| 47 | + - name: Test Python connectors |
| 48 | + run: | |
| 49 | + export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" |
| 50 | + DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" |
| 51 | + DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" |
| 52 | + export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" |
| 53 | + if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then |
| 54 | + mkdir -p "$DAGGER_TMP_BINDIR" |
| 55 | + curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" |
| 56 | + fi |
| 57 | + airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=8 --release-stage=generally_available --release-stage=beta --language=python --language=low-code test |
| 58 | + env: |
| 59 | + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" |
| 60 | + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} |
| 61 | + AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} |
| 62 | + AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} |
| 63 | + AWS_DEFAULT_REGION: "us-east-2" |
| 64 | + TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" |
| 65 | + CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} |
| 66 | + CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} |
| 67 | + CI_CONTEXT: "nightly_builds" |
| 68 | + CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} |
| 69 | + - name: Test Java connectors |
| 70 | + run: | |
| 71 | + export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" |
| 72 | + DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" |
| 73 | + DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" |
| 74 | + export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" |
| 75 | + if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then |
| 76 | + mkdir -p "$DAGGER_TMP_BINDIR" |
| 77 | + curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" |
| 78 | + fi |
| 79 | + airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=1 --release-stage=generally_available --release-stage=beta --language=java test |
| 80 | + env: |
| 81 | + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" |
| 82 | + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} |
| 83 | + AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} |
| 84 | + AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} |
| 85 | + AWS_DEFAULT_REGION: "us-east-2" |
| 86 | + TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" |
| 87 | + CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} |
| 88 | + CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} |
| 89 | + CI_CONTEXT: "nightly_builds" |
| 90 | + CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} |
0 commit comments