Skip to content

Commit 6829e86

Browse files
jrhizorjrhizor
authored
full scheduler v2 CI testing (#10199)
* fix normalization output processing in container orchestrator * add full scheduler v2 acceptance tests * speed up tests * fixes * clean up
1 parent 775d9e9 commit 6829e86

File tree

11 files changed

+399
-12
lines changed

11 files changed

+399
-12
lines changed

‎.github/workflows/gradle.yml

+133-1
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,7 @@ jobs:
564564
HOME: /home/runner
565565

566566
- name: Build Platform Docker Images
567-
run: SUB_BUILD=PLATFORM ./gradlew assemble --scan
567+
run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
568568

569569
- name: Run Logging Tests
570570
run: ./tools/bin/cloud_storage_logging_test.sh
@@ -626,3 +626,135 @@ jobs:
626626
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
627627
label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
628628
ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}
629+
630+
## Kube Acceptance Tests (with scheduler v2 - both temporal changes and container orchestrator)
631+
# In case of self-hosted EC2 errors, remove this block.
632+
# Docker acceptance tests run as part of the build job.
633+
start-kube-acceptance-test-runner-v2:
634+
name: Start Kube Acceptance Test EC2 Runner
635+
timeout-minutes: 10
636+
runs-on: ubuntu-latest
637+
outputs:
638+
label: ${{ steps.start-ec2-runner.outputs.label }}
639+
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
640+
steps:
641+
- name: Checkout Airbyte
642+
uses: actions/checkout@v2
643+
- name: Start AWS Runner
644+
id: start-ec2-runner
645+
uses: ./.github/actions/start-aws-runner
646+
with:
647+
# github-self-hosted-runner-ubuntu-20-with-120gdisk-docker-20.10.7
648+
ec2-image-id: ami-0d4083c04fde515c4
649+
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
650+
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
651+
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
652+
kube-acceptance-test-v2:
653+
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
654+
needs: start-kube-acceptance-test-runner-v2 # required to start the main job when the runner is ready
655+
runs-on: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }} # run the job on the newly created runner
656+
environment: more-secrets
657+
name: Acceptance Tests (Kube v2)
658+
timeout-minutes: 90
659+
steps:
660+
- name: Checkout Airbyte
661+
uses: actions/checkout@v2
662+
663+
- uses: actions/setup-java@v1
664+
with:
665+
java-version: "17"
666+
667+
- uses: actions/setup-node@v1
668+
with:
669+
node-version: "16.13.0"
670+
671+
- name: Fix EC-2 Runner
672+
run: |
673+
mkdir -p /home/runner
674+
675+
- name: Set up CI Gradle Properties
676+
run: |
677+
mkdir -p ~/.gradle/
678+
cat > ~/.gradle/gradle.properties <<EOF
679+
org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
680+
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
681+
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
682+
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
683+
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
684+
org.gradle.workers.max=8
685+
org.gradle.vfs.watch=false
686+
EOF
687+
688+
- name: Install socat (required for port forwarding)
689+
run: |
690+
# wait to receive lock (see https://askubuntu.com/questions/132059/how-to-make-a-package-manager-wait-if-another-instance-of-apt-is-running)
691+
while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
692+
sleep 1
693+
done
694+
695+
sudo apt-get update
696+
sudo apt-get install socat
697+
698+
- name: Create cluster config file
699+
run: |
700+
cat > /tmp/kind-config.yaml <<EOF
701+
kind: Cluster
702+
apiVersion: kind.x-k8s.io/v1alpha4
703+
nodes:
704+
- role: control-plane
705+
- role: worker
706+
EOF
707+
708+
- name: KIND Kubernetes Cluster Setup
709+
uses: helm/[email protected]
710+
with:
711+
node_image: kindest/node:v1.21.2
712+
config: /tmp/kind-config.yaml
713+
# In case of self-hosted EC2 errors, remove this env block.
714+
env:
715+
USER: root
716+
HOME: /home/runner
717+
CHANGE_MINIKUBE_NONE_USER: true
718+
719+
- name: Build Platform Docker Images
720+
run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
721+
722+
- name: Run Kubernetes End-to-End Acceptance Tests
723+
env:
724+
USER: root
725+
HOME: /home/runner
726+
AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
727+
SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
728+
SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
729+
run: |
730+
CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
731+
732+
- uses: actions/upload-artifact@v2
733+
if: failure()
734+
with:
735+
name: Kubernetes Logs
736+
path: /tmp/kubernetes_logs/*
737+
738+
# In case of self-hosted EC2 errors, remove this block.
739+
stop-kube-acceptance-test-runner-v2:
740+
name: Stop Kube Acceptance Test EC2 Runner
741+
timeout-minutes: 10
742+
needs:
743+
- start-kube-acceptance-test-runner-v2 # required to get output from the start-runner job
744+
- kube-acceptance-test-v2 # required to wait when the main job is done
745+
runs-on: ubuntu-latest
746+
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
747+
steps:
748+
- name: Configure AWS credentials
749+
uses: aws-actions/configure-aws-credentials@v1
750+
with:
751+
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
752+
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
753+
aws-region: us-east-2
754+
- name: Stop EC2 runner
755+
uses: machulav/[email protected]
756+
with:
757+
mode: stop
758+
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
759+
label: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }}
760+
ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.ec2-instance-id }}

‎airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java

-2
Original file line numberDiff line numberDiff line change
@@ -721,8 +721,6 @@ public void testIncrementalDedupeSync() throws Exception {
721721

722722
@Test
723723
@Order(14)
724-
@DisabledIfEnvironmentVariable(named = "KUBE",
725-
matches = "true")
726724
public void testCheckpointing() throws Exception {
727725
final SourceDefinitionRead sourceDefinition = apiClient.getSourceDefinitionApi().createSourceDefinition(new SourceDefinitionCreate()
728726
.name("E2E Test Source")

‎kube/overlays/dev-integration-test-schedulerv2/.env

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
AIRBYTE_VERSION=dev
2+
3+
# Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
4+
DATABASE_HOST=airbyte-db-svc
5+
DATABASE_PORT=5432
6+
DATABASE_DB=airbyte
7+
# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB}
8+
DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte
9+
JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001
10+
CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001
11+
12+
# When using the airbyte-db via default docker image:
13+
CONFIG_ROOT=/configs
14+
DATA_DOCKER_MOUNT=airbyte_data
15+
DB_DOCKER_MOUNT=airbyte_db
16+
17+
# Temporal.io worker configuration
18+
TEMPORAL_HOST=airbyte-temporal-svc:7233
19+
TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039,9040
20+
21+
# Workspace storage for running jobs (logs, etc)
22+
WORKSPACE_ROOT=/workspace
23+
WORKSPACE_DOCKER_MOUNT=airbyte_workspace
24+
25+
LOCAL_ROOT=/tmp/airbyte_local
26+
27+
# Maximum total simultaneous jobs across all worker nodes
28+
SUBMITTER_NUM_THREADS=10
29+
30+
# Miscellaneous
31+
TRACKING_STRATEGY=logging
32+
WEBAPP_URL=airbyte-webapp-svc:80
33+
API_URL=/api/v1/
34+
INTERNAL_API_HOST=airbyte-server-svc:8001
35+
36+
WORKER_ENVIRONMENT=kubernetes
37+
FULLSTORY=disabled
38+
IS_DEMO=false
39+
LOG_LEVEL=INFO
40+
41+
# S3/Minio Log Configuration
42+
S3_LOG_BUCKET=airbyte-dev-logs
43+
S3_LOG_BUCKET_REGION=
44+
S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
45+
S3_PATH_STYLE_ACCESS=true
46+
47+
# GCS Log Configuration
48+
GCS_LOG_BUCKET=
49+
50+
# State Storage Configuration
51+
STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs
52+
STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
53+
54+
# Docker Resource Limits
55+
JOB_MAIN_CONTAINER_CPU_REQUEST=
56+
JOB_MAIN_CONTAINER_CPU_LIMIT=
57+
JOB_MAIN_CONTAINER_MEMORY_REQUEST=
58+
JOB_MAIN_CONTAINER_MEMORY_LIMIT=
59+
60+
# Worker pod tolerations and node selectors
61+
JOB_KUBE_TOLERATIONS=
62+
JOB_KUBE_NODE_SELECTORS=
63+
64+
# Job image pull policy
65+
JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY=
66+
67+
# Launch a separate pod to orchestrate sync steps
68+
NEW_SCHEDULER=true
69+
CONTAINER_ORCHESTRATOR_ENABLED=true
70+

‎kube/overlays/dev-integration-test-schedulerv2/.secrets

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
DATABASE_USER=docker
2+
DATABASE_PASSWORD=docker
3+
AWS_ACCESS_KEY_ID=minio
4+
AWS_SECRET_ACCESS_KEY=minio123
5+
GOOGLE_APPLICATION_CREDENTIALS=
6+
STATE_STORAGE_MINIO_ACCESS_KEY=minio
7+
STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123

‎kube/overlays/dev-integration-test-schedulerv2/bootloader-patch.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: airbyte-bootloader
5+
spec:
6+
containers:
7+
- name: airbyte-bootloader-container
8+
env:
9+
- name: NEW_SCHEDULER
10+
valueFrom:
11+
configMapKeyRef:
12+
name: airbyte-env
13+
key: NEW_SCHEDULER

‎kube/overlays/dev-integration-test-schedulerv2/kustomization.yaml

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
namespace: default
5+
6+
bases:
7+
- ../../resources
8+
9+
images:
10+
- name: airbyte/db
11+
newTag: dev
12+
- name: airbyte/bootloader
13+
newTag: dev
14+
- name: airbyte/scheduler
15+
newTag: dev
16+
- name: airbyte/server
17+
newTag: dev
18+
- name: airbyte/webapp
19+
newTag: dev
20+
- name: airbyte/worker
21+
newTag: dev
22+
- name: temporalio/auto-setup
23+
newTag: 1.7.0
24+
25+
configMapGenerator:
26+
- name: airbyte-env
27+
env: .env
28+
29+
secretGenerator:
30+
- name: airbyte-secrets
31+
env: .secrets
32+
33+
patchesStrategicMerge:
34+
- bootloader-patch.yaml
35+
- server-patch.yaml
36+
- scheduler-patch.yaml
37+
- worker-patch.yaml

‎kube/overlays/dev-integration-test-schedulerv2/scheduler-patch.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: airbyte-scheduler
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: airbyte-scheduler-container
10+
env:
11+
- name: NEW_SCHEDULER
12+
valueFrom:
13+
configMapKeyRef:
14+
name: airbyte-env
15+
key: NEW_SCHEDULER

‎kube/overlays/dev-integration-test-schedulerv2/server-patch.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: airbyte-server
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: airbyte-server-container
10+
env:
11+
- name: NEW_SCHEDULER
12+
valueFrom:
13+
configMapKeyRef:
14+
name: airbyte-env
15+
key: NEW_SCHEDULER

‎kube/overlays/dev-integration-test-schedulerv2/worker-patch.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: airbyte-worker
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: airbyte-worker-container
10+
env:
11+
- name: CONTAINER_ORCHESTRATOR_ENABLED
12+
valueFrom:
13+
configMapKeyRef:
14+
name: airbyte-env
15+
key: CONTAINER_ORCHESTRATOR_ENABLED

‎tools/bin/acceptance_test_kube.sh

+11-9
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,17 @@ assert_root
88

99
# Since KIND does not have access to the local docker agent, manually load the minimum images required for the Kubernetes Acceptance Tests.
1010
# See https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster.
11-
echo "Loading images into KIND..."
12-
kind load docker-image airbyte/server:dev --name chart-testing &
13-
kind load docker-image airbyte/scheduler:dev --name chart-testing &
14-
kind load docker-image airbyte/webapp:dev --name chart-testing &
15-
kind load docker-image airbyte/worker:dev --name chart-testing &
16-
kind load docker-image airbyte/db:dev --name chart-testing &
17-
kind load docker-image airbyte/container-orchestrator:dev --name chart-testing &
18-
kind load docker-image airbyte/bootloader:dev --name chart-testing &
19-
wait
11+
if [ -n "$CI" ]; then
12+
echo "Loading images into KIND..."
13+
kind load docker-image airbyte/server:dev --name chart-testing &
14+
kind load docker-image airbyte/scheduler:dev --name chart-testing &
15+
kind load docker-image airbyte/webapp:dev --name chart-testing &
16+
kind load docker-image airbyte/worker:dev --name chart-testing &
17+
kind load docker-image airbyte/db:dev --name chart-testing &
18+
kind load docker-image airbyte/container-orchestrator:dev --name chart-testing &
19+
kind load docker-image airbyte/bootloader:dev --name chart-testing &
20+
wait
21+
fi
2022

2123
echo "Starting app..."
2224

0 commit comments

Comments
 (0)