Skip to content

Commit dd882ff

Browse files
clnollclnoll
committed
feat: connector progressive rollout CLI (#13888)
1 parent 483f470 commit dd882ff

File tree

16 files changed

+679
-30
lines changed

16 files changed

+679
-30
lines changed

‎airbyte-connector-rollout-client/Dockerfile

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.2.2
2+
3+
FROM scratch as builder
4+
WORKDIR /app
5+
ADD airbyte-app.tar /app
6+
7+
FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION}
8+
9+
ENV APPLICATION airbyte-connector-rollout-client
10+
ENV VERSION ${VERSION}
11+
# 5005 is the remote debug port
12+
EXPOSE 5005
13+
14+
WORKDIR /app
15+
COPY --chown=airbyte:airbyte --from=builder /app /app
16+
USER airbyte:airbyte
17+
18+
ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"]

‎airbyte-connector-rollout-client/README.md

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Connector Rollout Client
2+
3+
Manually control progressive connector rollouts.
4+
5+
## Local Development
6+
The Airbyte server, connector rollout worker, and a Temporal server are required for local development and are all deployed to our local kubernetes cluster.
7+
The Temporal UI server is optional but is useful for debugging. It's deployed to our kubernetes cluster in debug mode.
8+
9+
To run the airbyte cluster in kubernetes with the Temporal UI enabled:
10+
```sh
11+
make deploy.oss.debug
12+
```
13+
This will start the connector rollout worker, the Temporal server, and Airbyte server.
14+
15+
Once the pods are running, get a shell into the `ab-temporal` pod and run:
16+
```sh
17+
cd /tmp
18+
curl -sOL https://github.com/andrew-d/static-binaries/raw/master/binaries/linux/x86_64/socat
19+
chmod +x socat
20+
./socat TCP-LISTEN:7777,reuseaddr,fork TCP:$(netstat -tuln | awk '/LISTEN/ {print $4}' | cut -d':' -f1 | head -n 1):7233 &
21+
```
22+
23+
We'll need to forward traffic from localhost to this port to allow CLI requests to hit the Temporal server in kubernetes:
24+
```sh
25+
kubectl port-forward -n ab <ab-temporal POD-NAME> 7233:7777
26+
```
27+
28+
And forward traffic from localhost to 8080 to allow us to access the Temporal UI:
29+
```sh
30+
kubectl port-forward -n ab <ab-temporal-ui POD-NAME> 8080:8080
31+
```
32+
Navigate to the Temporal UI at http://localhost:8080 to verify that Temporal UI is running and communicating with `ab-temporal` server.
33+
34+
35+
## Running connector rollouts
36+
37+
### Start a connector rollout
38+
39+
Execute commands using the CLI:
40+
```sh
41+
# Start a rollout for the source-faker connector with version 6.2.11
42+
# Note: we temporarily give a fake rollout ID (until hooked up to the publishing pipeline)
43+
./gradlew :oss:airbyte-connector-rollout-client:runConnectorRolloutCLI --args="start -d airbyte/source-faker -i 6.2.11 -a dfd88b22-b603-4c3d-aad7-3701784586b1 -r aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa"
44+
45+
# Find information about all rollouts for source-faker version 6.2.11
46+
# This can be used to get the rollout ID
47+
./gradlew :oss:airbyte-connector-rollout-client:runConnectorRolloutCLI --args="find -d airbyte/source-faker -i 6.2.11 -a dfd88b22-b603-4c3d-aad7-3701784586b1"
48+
49+
# Get the rollout status
50+
./gradlew :oss:airbyte-connector-rollout-client:runConnectorRolloutCLI --args="get -d airbyte/source-faker -i 6.2.11 -a dfd88b22-b603-4c3d-aad7-3701784586b1 -r d584520a-1127-4524-a357-52903c58477c"
51+
52+
# Rollout the connector to new actors
53+
./gradlew :oss:airbyte-connector-rollout-client:runConnectorRolloutCLI --args="rollout -d airbyte/source-faker -i 6.2.10 -a dfd88b22-b603-4c3d-aad7-3701784586b1 -r e74f0ae2-1e1f-438f-95e7-9a40ae9ea63d -c a1d07f1a-5dc0-43c9-b938-decc4511e522,b2e1802b-9df1-33b8-6a44-1122adf1e413"
54+
55+
# Promote the rollout
56+
./gradlew :oss:airbyte-connector-rollout-client:runConnectorRolloutCLI --args="promote -d airbyte/source-faker -i 6.2.11 -a dfd88b22-b603-4c3d-aad7-3701784586b1 -r d584520a-1127-4524-a357-52903c58477c"
57+
58+
# Fail the rollout
59+
./gradlew :oss:airbyte-connector-rollout-client:runConnectorRolloutCLI --args="fail -d airbyte/source-faker -i 6.2.11 -a dfd88b22-b603-4c3d-aad7-3701784586b1 -r d584520a-1127-4524-a357-52903c58477c"
60+
61+
# Cancel the rollout
62+
./gradlew :oss:airbyte-connector-rollout-client:runConnectorRolloutCLI --args="cancel -d airbyte/source-faker -i 6.2.11 -a dfd88b22-b603-4c3d-aad7-3701784586b1 -r d584520a-1127-4524-a357-52903c58477c"
63+
64+
65+
...
66+
```
67+
68+
```sh
69+
psql -U airbyte
70+
\c db-airbyte
71+
truncate connector_rollout;
72+
```
73+
74+
## Production usage
75+
76+
TODO

‎airbyte-connector-rollout-client/build.gradle.kts

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
plugins {
2+
id("io.micronaut.application") version "4.4.2"
3+
id("io.airbyte.gradle.jvm.app")
4+
id("io.airbyte.gradle.jvm.lib")
5+
id("io.airbyte.gradle.publish")
6+
id("io.airbyte.gradle.docker")
7+
application
8+
}
9+
10+
group = "io.airbyte.connector.rollout.client"
11+
12+
java {
13+
sourceCompatibility = JavaVersion.VERSION_21
14+
targetCompatibility = JavaVersion.VERSION_21
15+
}
16+
17+
repositories {
18+
mavenCentral()
19+
}
20+
21+
dependencies {
22+
// TODO: remove the deps not being used
23+
compileOnly(libs.lombok)
24+
annotationProcessor(libs.lombok) // Lombok must be added BEFORE Micronaut
25+
26+
implementation("info.picocli:picocli:4.7.4")
27+
implementation("io.micronaut.picocli:micronaut-picocli")
28+
implementation("io.temporal:temporal-sdk:1.25.0")
29+
annotationProcessor("info.picocli:picocli-codegen:4.7.4")
30+
implementation(project(":oss:airbyte-config:config-models"))
31+
implementation(project(":oss:airbyte-connector-rollout-worker"))
32+
implementation(project(":oss:airbyte-api:server-api"))
33+
implementation(project(":oss:airbyte-api:workload-api"))
34+
implementation(libs.airbyte.protocol)
35+
36+
annotationProcessor("io.micronaut:micronaut-http-validation")
37+
annotationProcessor("io.micronaut.serde:micronaut-serde-processor")
38+
implementation("io.micronaut.serde:micronaut-serde-jackson")
39+
compileOnly("io.micronaut:micronaut-http-client")
40+
testImplementation("io.micronaut:micronaut-http-client")
41+
runtimeOnly("org.yaml:snakeyaml:1.33")
42+
43+
}
44+
45+
application {
46+
// Default to running ConnectorRolloutCLI
47+
mainClass.set("io.airbyte.connector.rollout.client.ConnectorRolloutCLI")
48+
}
49+
50+
val runConnectorRolloutCLI by tasks.registering(JavaExec::class) {
51+
group = "application"
52+
description = "Run the ConnectorRolloutCLI with specified arguments"
53+
classpath = sourceSets["main"].runtimeClasspath
54+
mainClass.set("io.airbyte.connector.rollout.client.ConnectorRolloutCLI")
55+
args = listOf("") // Set default CLI command and options here if needed
56+
}
57+
58+
tasks.jar {
59+
manifest {
60+
attributes(
61+
"Main-Class" to "io.airbyte.connector.rollout.client.ConnectorRolloutCLI"
62+
)
63+
}
64+
65+
archiveBaseName.set("run-connector-rollout-cli")
66+
archiveVersion.set("") // Remove the version from the JAR file name
67+
}
68+
69+
tasks.withType<JavaCompile> {
70+
options.encoding = "UTF-8"
71+
options.compilerArgs.addAll(listOf("-Xlint:unchecked", "-Xlint:deprecation"))
72+
}
73+
74+
micronaut {
75+
runtime("netty")
76+
testRuntime("junit5")
77+
processing {
78+
incremental(true)
79+
annotations("io.airbyte.connector.rollout.client.*")
80+
}
81+
}
82+
83+
84+
tasks.withType<Jar> {
85+
duplicatesStrategy = DuplicatesStrategy.INCLUDE
86+
}
87+
88+
airbyte {
89+
application {
90+
mainClass.set("io.airbyte.connector.rollout.client.ConnectorRolloutCLI")
91+
defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0")
92+
localEnvVars.putAll(
93+
mapOf(
94+
"AIRBYTE_ROLE" to "undefined",
95+
"AIRBYTE_VERSION" to "dev",
96+
"DATA_PLANE_ID" to "local",
97+
"MICRONAUT_ENVIRONMENTS" to "test"
98+
)
99+
)
100+
}
101+
docker {
102+
imageName.set("connector-rollout-client")
103+
}
104+
}

0 commit comments

Comments
 (0)