Skip to content

Commit 3598330

Browse files
author
Marius Posta
authored
gradle,.github: add bulk cdk publish workflow (#43361)
1 parent 6a63ffa commit 3598330

File tree

140 files changed

+13091
-13
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

140 files changed

+13091
-13
lines changed

‎.github/workflows/gradle.yml

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ jobs:
3939
- '**/*.gradle'
4040
- '**/*.kt'
4141
- 'airbyte-cdk/java/**/*'
42+
- 'airbyte-cdk/bulk/**/*'
4243
4344
run-check:
4445
needs:

‎.github/workflows/publish-bulk-cdk.yml

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2+
3+
name: Publish Bulk CDK
4+
on:
5+
push:
6+
branches:
7+
- master
8+
paths:
9+
- "airbyte-cdk/bulk"
10+
workflow_dispatch:
11+
inputs:
12+
repo:
13+
description: "Repo to check out code from. Defaults to the main airbyte repo."
14+
type: choice
15+
required: true
16+
default: airbytehq/airbyte
17+
options:
18+
- airbytehq/airbyte
19+
build-number:
20+
description: "Build Number"
21+
required: false
22+
type: number
23+
24+
concurrency:
25+
group: publish-bulk-cdk
26+
cancel-in-progress: false
27+
28+
env:
29+
# Use the provided GITREF or default to the branch triggering the workflow.
30+
GITREF: ${{ github.event.inputs.gitref || github.ref }}
31+
S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
32+
S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
33+
34+
jobs:
35+
publish-bulk-cdk:
36+
name: Publish Bulk CDK
37+
runs-on: connector-test-large
38+
timeout-minutes: 30
39+
steps:
40+
- name: Checkout Airbyte
41+
uses: actions/checkout@v3
42+
with:
43+
ref: ${{ env.GITREF }}
44+
45+
- name: Setup Java
46+
uses: actions/setup-java@v3
47+
with:
48+
distribution: "zulu"
49+
java-version: "21"
50+
51+
- name: Docker login
52+
# Some tests use testcontainers which pull images from DockerHub.
53+
uses: docker/login-action@v1
54+
with:
55+
username: ${{ secrets.DOCKER_HUB_USERNAME }}
56+
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
57+
58+
- name: Set Build Number [manual]
59+
if: github.event.inputs.build-number
60+
env:
61+
BUILD_NUMBER: ${{ github.event.inputs.build-number }}
62+
run: |
63+
mkdir -p airbyte-cdk/bulk/build/generated
64+
echo $BUILD_NUMBER > airbyte-cdk/bulk/build/generated/build.number
65+
66+
- name: Set Build Number [auto]
67+
if: ${{ !github.event.inputs.build-number }}
68+
uses: burrunan/gradle-cache-action@v1
69+
env:
70+
CI: true
71+
with:
72+
job-id: bulk-cdk-publish
73+
concurrent: true
74+
gradle-distribution-sha-256-sum-warning: false
75+
arguments: --scan :airbyte-cdk:bulk:generateBuildNumberFile
76+
77+
- name: Read Build Number
78+
run: echo "BUILD_NUMBER=$(cat airbyte-cdk/bulk/build/generated/build.number)" >>$GITHUB_ENV
79+
80+
- name: Build Bulk CDK
81+
uses: burrunan/gradle-cache-action@v1
82+
env:
83+
CI: true
84+
with:
85+
job-id: bulk-cdk-publish
86+
concurrent: true
87+
gradle-distribution-sha-256-sum-warning: false
88+
arguments: --scan :airbyte-cdk:bulk:bulkCdkBuild
89+
90+
- name: Publish Poms and Jars to CloudRepo
91+
uses: burrunan/gradle-cache-action@v1
92+
env:
93+
CI: true
94+
CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }}
95+
CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
96+
with:
97+
job-id: bulk-cdk-publish
98+
read-only: true
99+
concurrent: true
100+
execution-only-caches: true
101+
gradle-distribution-sha-256-sum-warning: false
102+
arguments: --scan :airbyte-cdk:bulk:bulkCdkPublish
103+
104+
- name: Post failure to Slack channel
105+
if: ${{ env.DRY_RUN == 'false' && failure() }}
106+
uses: slackapi/[email protected]
107+
continue-on-error: true
108+
with:
109+
channel-id: C04J1M66D8B
110+
payload: |
111+
{
112+
"text": "Error while publishing Bulk CDK!",
113+
"blocks": [
114+
{
115+
"type": "section",
116+
"text": {
117+
"type": "mrkdwn",
118+
"text": "Error while publishing Bulk CDK!"
119+
}
120+
},
121+
{
122+
"type": "section",
123+
"text": {
124+
"type": "mrkdwn",
125+
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
126+
}
127+
}
128+
]
129+
}
130+
env:
131+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
132+
133+
- name: Post success to Slack channel
134+
if: ${{ env.DRY_RUN == 'false' && !failure() }}
135+
uses: slackapi/[email protected]
136+
continue-on-error: true
137+
with:
138+
channel-id: C04J1M66D8B
139+
payload: |
140+
{
141+
"text": "Bulk CDK version 0.${{ env.BUILD_NUMBER }} published successfully!",
142+
"blocks": [
143+
{
144+
"type": "section",
145+
"text": {
146+
"type": "mrkdwn",
147+
"text": "Bulk CDK version 0.${{ env.BUILD_NUMBER }} published successfully!"
148+
}
149+
},
150+
{
151+
"type": "section",
152+
"text": {
153+
"type": "mrkdwn",
154+
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
155+
}
156+
}
157+
]
158+
}
159+
env:
160+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}

‎airbyte-cdk/bulk/README.md

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Bulk CDK
2+
3+
The Bulk CDK is the "new java CDK" that's currently incubating.
4+
It's written in Kotlin and consists of a _core_ and a bunch of _toolkits_:
5+
- The _core_ consists of the Micronaut entry point and other objects which are expected in
6+
connectors built using this CDK.
7+
- The _toolkits_ consist of optional modules which contain objects which are common across
8+
multiple (but by no means all) connectors.
9+
10+
While the CDK is incubating, its published version numbers are 0.X where X is monotonically
11+
increasing based on the maximum version value found on the maven repository that the jars are
12+
published to.
13+
Jar publication happens via a github workflow triggered by pushes to the master branch, i.e. after
14+
merging a pull request.

‎airbyte-cdk/bulk/build.gradle

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
final File buildNumberFile = file("${getLayout().buildDirectory.get()}/generated/build.number")
2+
3+
allprojects {
4+
apply plugin: 'java-library'
5+
apply plugin: 'maven-publish'
6+
7+
group 'io.airbyte.bulk-cdk'
8+
9+
// Disable spotbugs on test code, which gets annoying really quickly for @MicronautTest classes.
10+
spotbugsTest {
11+
enabled = false
12+
}
13+
14+
dependencies {
15+
implementation platform('org.jetbrains.kotlin:kotlin-bom:2.0.0')
16+
implementation platform('org.jetbrains.kotlinx:kotlinx-coroutines-bom:1.8.1')
17+
implementation platform('com.fasterxml.jackson:jackson-bom:2.16.1')
18+
implementation platform('io.micronaut:micronaut-core-bom:4.3.13')
19+
implementation platform('org.junit:junit-bom:5.10.2')
20+
implementation platform('org.slf4j:slf4j-bom:2.0.13')
21+
implementation platform('org.apache.logging.log4j:log4j-bom:2.21.1')
22+
implementation platform('org.testcontainers:testcontainers-bom:1.19.8')
23+
24+
implementation 'org.jetbrains.kotlin:kotlin-stdlib'
25+
implementation 'com.google.dagger:dagger-compiler:2.51.1'
26+
ksp 'com.google.dagger:dagger-compiler:2.51.1'
27+
28+
annotationProcessor platform('io.micronaut:micronaut-core-bom:4.3.13')
29+
annotationProcessor 'info.picocli:picocli-codegen:4.7.5'
30+
annotationProcessor 'io.micronaut:micronaut-inject-kotlin'
31+
32+
ksp platform('io.micronaut:micronaut-core-bom:4.3.13')
33+
ksp 'io.micronaut:micronaut-inject-kotlin'
34+
kspTest platform('io.micronaut:micronaut-core-bom:4.3.13')
35+
kspTest 'io.micronaut:micronaut-inject-kotlin'
36+
}
37+
38+
if (buildNumberFile.exists()) {
39+
version = "0.${buildNumberFile.text.trim()}"
40+
publishing {
41+
publications {
42+
cdk(MavenPublication) {
43+
from components.java
44+
}
45+
}
46+
// This repository is only defined and used in the context of publishing artifacts
47+
// It's different from the 'airbyte-public-jars' defined in settings.gradle
48+
// only in its omission of the 'public' directory.
49+
// Any artifacts publish here will be available in the 'airbyte-public-jars' repo.
50+
repositories {
51+
maven {
52+
name 'airbyte-repo'
53+
url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars/'
54+
credentials {
55+
username System.getenv('CLOUDREPO_USER')
56+
password System.getenv('CLOUDREPO_PASSWORD')
57+
}
58+
}
59+
}
60+
}
61+
}
62+
}
63+
64+
if (buildNumberFile.exists()) {
65+
tasks.register('bulkCdkBuild').configure {
66+
dependsOn allprojects.collect {it.tasks.named('build')}
67+
}
68+
tasks.register('bulkCdkPublish').configure {
69+
dependsOn allprojects.collect {it.tasks.named('publish')}
70+
}
71+
}
72+
73+
tasks.register('generateBuildNumberFile') {
74+
description = 'Generates a build.number file in the build directory'
75+
group = 'Custom'
76+
outputs.file buildNumberFile
77+
78+
doLast {
79+
var repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars"
80+
var groupIdUrl = "${repoUrl}/io/airbyte/bulk-cdk"
81+
var artifactUrl = "${groupIdUrl}/bulk"
82+
var metadataXmlUrl = "${artifactUrl}/maven-metadata.xml"
83+
84+
var connection = metadataXmlUrl.toURL().openConnection() as HttpURLConnection
85+
try {
86+
connection.setRequestMethod("GET")
87+
connection.setDoInput(true)
88+
var responseCode = connection.getResponseCode()
89+
if (responseCode != 200) {
90+
throw new GradleException("Unexpected HTTP response code ${responseCode} from ${metadataXmlUrl} : expected 200.")
91+
}
92+
String responseContent = connection.inputStream.text
93+
def xml = new XmlParser().parseText(responseContent)
94+
String latestVersion = xml."versioning"."latest".text()
95+
String buildNumber = latestVersion.replaceFirst('^0\\.', '')
96+
Integer nextBuildNumber = 1 + buildNumber.toInteger()
97+
buildNumberFile.parentFile.mkdirs()
98+
buildNumberFile.text = "$nextBuildNumber"
99+
logger.lifecycle("Wrote Bulk CDK build number ${nextBuildNumber} to ${buildNumberFile.path}.")
100+
} finally {
101+
connection.disconnect()
102+
}
103+
}
104+
}

‎airbyte-cdk/bulk/core/base/build.gradle

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
2+
dependencies {
3+
4+
api 'com.fasterxml.jackson.core:jackson-annotations'
5+
api 'com.fasterxml.jackson.core:jackson-databind'
6+
api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
7+
api 'com.kjetland:mbknor-jackson-jsonschema_2.13:1.0.39'
8+
api('io.airbyte.airbyte-protocol:protocol-models:0.9.0') {
9+
exclude group: 'com.google.guava', module: 'guava'
10+
exclude group: 'com.google.api-client'
11+
exclude group: 'org.apache.logging.log4j'
12+
exclude group: 'javax.validation'
13+
exclude group: 'org.apache.commons'
14+
exclude group: 'commons-io'
15+
}
16+
api 'io.github.oshai:kotlin-logging-jvm:5.1.0'
17+
api 'io.micronaut:micronaut-runtime'
18+
api 'org.apache.sshd:sshd-mina:2.12.1'
19+
api 'org.jetbrains.kotlinx:kotlinx-coroutines-core'
20+
21+
implementation 'com.datadoghq:dd-trace-api:1.28.0'
22+
implementation 'com.datadoghq:dd-trace-ot:1.28.0'
23+
implementation 'com.fasterxml.jackson.module:jackson-module-kotlin'
24+
implementation 'com.fasterxml.jackson.module:jackson-module-afterburner'
25+
implementation 'com.networknt:json-schema-validator:1.4.0'
26+
implementation 'info.picocli:picocli:4.7.6'
27+
implementation 'io.micronaut.picocli:micronaut-picocli:5.2.0'
28+
implementation 'jakarta.validation:jakarta.validation-api:3.0.2'
29+
implementation 'org.apache.commons:commons-lang3:3.14.0'
30+
implementation 'org.apache.logging.log4j:log4j-api'
31+
implementation 'org.apache.logging.log4j:log4j-core'
32+
implementation 'org.apache.logging.log4j:log4j-slf4j-impl'
33+
implementation 'org.apache.logging.log4j:log4j-slf4j2-impl'
34+
implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.17.2'
35+
implementation 'org.bouncycastle:bcpkix-jdk18on:1.77'
36+
implementation 'org.bouncycastle:bcprov-jdk18on:1.77'
37+
implementation 'org.bouncycastle:bctls-jdk18on:1.77'
38+
39+
runtimeOnly 'com.google.guava:guava:33.2.0-jre'
40+
runtimeOnly 'org.apache.commons:commons-compress:1.26.1'
41+
42+
testFixturesApi 'org.jetbrains.kotlin:kotlin-test-junit'
43+
testFixturesApi 'org.jetbrains.kotlin:kotlin-reflect'
44+
testFixturesApi 'org.junit.jupiter:junit-jupiter-api'
45+
testFixturesApi 'org.junit.jupiter:junit-jupiter-params'
46+
testFixturesApi 'org.junit.jupiter:junit-jupiter-engine'
47+
testFixturesApi('org.testcontainers:testcontainers') {
48+
exclude group: 'org.apache.commons', module: 'commons-compress'
49+
}
50+
testFixturesApi 'io.micronaut.test:micronaut-test-core:4.2.1'
51+
testFixturesApi 'io.micronaut.test:micronaut-test-junit5:4.2.1'
52+
testFixturesApi 'com.h2database:h2:2.2.224'
53+
testFixturesApi 'io.github.deblockt:json-diff:1.0.1'
54+
}

‎airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunnable.kt

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */
2+
package io.airbyte.cdk
3+
4+
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
5+
import io.airbyte.cdk.consumers.OutputConsumer
6+
import io.airbyte.cdk.operation.Operation
7+
import io.airbyte.cdk.util.ApmTraceUtils
8+
import io.github.oshai.kotlinlogging.KotlinLogging
9+
import io.micronaut.context.annotation.Value
10+
import jakarta.inject.Inject
11+
12+
private val log = KotlinLogging.logger {}
13+
14+
/** [AirbyteConnectorRunner] tells Micronaut to use this [Runnable] as the entry point. */
15+
@SuppressFBWarnings(value = ["NP_NONNULL_RETURN_VIOLATION"], justification = "Micronaut DI")
16+
class AirbyteConnectorRunnable : Runnable {
17+
@Value("\${airbyte.connector.metadata.docker-repository}") lateinit var connectorName: String
18+
19+
@Inject lateinit var operation: Operation
20+
21+
@Inject lateinit var outputConsumer: OutputConsumer
22+
23+
override fun run() {
24+
log.info { "Executing ${operation::class} operation." }
25+
try {
26+
operation.execute()
27+
} catch (e: Throwable) {
28+
log.error(e) { "Failed ${operation::class} operation execution." }
29+
ApmTraceUtils.addExceptionToTrace(e)
30+
outputConsumer.acceptTraceOnConfigError(e)
31+
throw e
32+
} finally {
33+
log.info { "Flushing output consumer prior to shutdown." }
34+
outputConsumer.close()
35+
log.info { "Completed integration: $connectorName." }
36+
}
37+
}
38+
}

0 commit comments

Comments
 (0)