Skip to content

Commit d800e3d

Browse files
benmoriceaubenmoriceau
authored
Add scheduled service (#15761)
* Add logger scheduled service * Use eager bean initialization
1 parent 7779df9 commit d800e3d

File tree

14 files changed

+208
-0
lines changed

14 files changed

+208
-0
lines changed

‎airbyte-cron/Dockerfile

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
ARG JDK_VERSION=19-slim-bullseye
2+
ARG JDK_IMAGE=openjdk:${JDK_VERSION}
3+
FROM ${JDK_IMAGE} AS cron
4+
5+
ARG VERSION=0.40.0-alpha
6+
7+
ENV APPLICATION airbyte-cron
8+
ENV VERSION ${VERSION}
9+
10+
WORKDIR /app
11+
12+
# This is automatically unzipped by Docker
13+
ADD bin/${APPLICATION}-${VERSION}.tar /app
14+
15+
# wait for upstream dependencies to become available before starting server
16+
ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

‎airbyte-cron/LICENSE

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2022 Airbyte, Inc.
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎airbyte-cron/build.gradle

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
plugins {
2+
id 'application'
3+
}
4+
5+
dependencies {
6+
implementation project(':airbyte-workers')
7+
8+
runtimeOnly 'io.micronaut:micronaut-http-server-netty:3.6.0'
9+
10+
annotationProcessor platform(libs.micronaut.bom)
11+
annotationProcessor libs.bundles.micronaut.annotation.processor
12+
13+
implementation platform(libs.micronaut.bom)
14+
implementation libs.bundles.micronaut
15+
}
16+
17+
mainClassName = 'io.airbyte.cron.MicronautCronRunner'
18+
19+
application {
20+
mainClass = mainClassName
21+
applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
22+
}
23+
24+
task copyGeneratedTar(type: Copy) {
25+
dependsOn copyDocker
26+
dependsOn distTar
27+
28+
from('build/distributions') {
29+
include 'airbyte-cron-*.tar'
30+
}
31+
into 'build/docker/bin'
32+
}
33+
34+
Task dockerBuildTask = getDockerBuildTask("cron", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
35+
dockerBuildTask.dependsOn(copyGeneratedTar)
36+
assemble.dependsOn(dockerBuildTask)

‎airbyte-cron/src/main/java/io/airbyte/cron/MicronautCronRunner.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.cron;
6+
7+
import io.micronaut.runtime.Micronaut;
8+
9+
/**
10+
* Micronaut server responsible of running scheduled method. The methods need to be separated in
11+
* Bean based on what they are cleaning and contain a method annotated with `@Scheduled`
12+
*
13+
* Injected object looks unused but they are not
14+
*/
15+
public class MicronautCronRunner {
16+
17+
public static void main(final String[] args) {
18+
Micronaut.build(args)
19+
.eagerInitSingletons(true)
20+
.mainClass(MicronautCronRunner.class)
21+
.start();
22+
}
23+
24+
}

‎airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/Temporal.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.cron.selfhealing;
6+
7+
import io.micronaut.scheduling.annotation.Scheduled;
8+
import javax.inject.Singleton;
9+
import lombok.extern.slf4j.Slf4j;
10+
11+
@Singleton
12+
@Slf4j
13+
public class Temporal {
14+
15+
public Temporal() {
16+
log.info("Creating temporal self-healing");
17+
}
18+
19+
@Scheduled(fixedRate = "10s")
20+
void cleanTemporal() {}
21+
22+
}

‎docker-compose.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,18 @@ services:
174174
- POSTGRES_USER=${DATABASE_USER}
175175
volumes:
176176
- ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig
177+
airbyte-cron:
178+
image: airbyte/cron:${VERSION}
179+
logging: *default-logging
180+
container_name: airbyte-cron
181+
restart: unless-stopped
182+
environment:
183+
- DB=postgresql
184+
- DB_PORT=${DATABASE_PORT}
185+
- LOG_LEVEL=${LOG_LEVEL}
186+
- POSTGRES_PWD=${DATABASE_PASSWORD}
187+
- POSTGRES_SEEDS=${DATABASE_HOST}
188+
- POSTGRES_USER=${DATABASE_USER}
177189
volumes:
178190
workspace:
179191
name: ${WORKSPACE_DOCKER_MOUNT}

‎kube/overlays/dev-integration-test/kustomization.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ images:
1919
newTag: dev
2020
- name: temporalio/auto-setup
2121
newTag: 1.7.0
22+
- name: airbyte/cron
23+
newTag: dev
2224

2325
configMapGenerator:
2426
- name: airbyte-env

‎kube/overlays/dev/kustomization.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ images:
1919
newTag: dev
2020
- name: temporalio/auto-setup
2121
newTag: 1.7.0
22+
- name: airbyte/cron
23+
newTag: dev
2224

2325
configMapGenerator:
2426
- name: airbyte-env

‎kube/overlays/stable-with-resource-limits/kustomization.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ images:
1919
newTag: 0.40.0-alpha
2020
- name: temporalio/auto-setup
2121
newTag: 1.7.0
22+
- name: airbyte/cron
23+
newTag: 0.40.0-alpha
2224

2325
configMapGenerator:
2426
- name: airbyte-env

‎kube/overlays/stable-with-resource-limits/set-resource-limits.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,3 +107,17 @@ spec:
107107
limits:
108108
cpu: 0.5
109109
memory: 128Mi
110+
---
111+
apiVersion: apps/v1
112+
kind: Deployment
113+
metadata:
114+
name: airbyte-cron
115+
spec:
116+
template:
117+
spec:
118+
containers:
119+
- name: airbyte-cron
120+
resources:
121+
limits:
122+
cpu: 0.5
123+
memory: 128Mi

‎kube/overlays/stable/kustomization.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ images:
1919
newTag: 0.40.0-alpha
2020
- name: temporalio/auto-setup
2121
newTag: 1.7.0
22+
- name: airbyte/cron
23+
newTag: 0.40.0-alpha
2224

2325
configMapGenerator:
2426
- name: airbyte-env

‎kube/resources/cron.yaml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: airbyte-cron
5+
spec:
6+
replicas: 1
7+
selector:
8+
matchLabels:
9+
airbyte: cron
10+
template:
11+
metadata:
12+
labels:
13+
airbyte: cron
14+
spec:
15+
containers:
16+
- name: airbyte-cron-container
17+
image: airbyte/cron
18+
env:
19+
- name: POSTGRES_USER
20+
valueFrom:
21+
secretKeyRef:
22+
name: airbyte-secrets
23+
key: DATABASE_USER
24+
- name: POSTGRES_PWD
25+
valueFrom:
26+
secretKeyRef:
27+
name: airbyte-secrets
28+
key: DATABASE_PASSWORD
29+
- name: DB
30+
value: "postgresql"
31+
- name: DB_PORT
32+
valueFrom:
33+
configMapKeyRef:
34+
name: airbyte-env
35+
key: DATABASE_PORT
36+
- name: POSTGRES_SEEDS
37+
valueFrom:
38+
configMapKeyRef:
39+
name: airbyte-env
40+
key: DATABASE_HOST
41+
volumeMounts:
42+
- name: airbyte-volume-configs
43+
mountPath: /configs
44+
- name: gcs-log-creds-volume
45+
mountPath: /secrets/gcs-log-creds
46+
readOnly: true
47+
volumes:
48+
- name: airbyte-volume-configs
49+
persistentVolumeClaim:
50+
claimName: airbyte-volume-configs
51+
- name: gcs-log-creds-volume
52+
secret:
53+
secretName: gcs-log-creds

‎kube/resources/kustomization.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ kind: Kustomization
44
resources:
55
- airbyte-minio.yaml
66
- bootloader.yaml
7+
- cron.yaml
78
- db.yaml
89
- pod-sweeper.yaml
910
- secret-gcs-log-creds.yaml

‎settings.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
7777
include ':airbyte-tests'
7878
include ':airbyte-webapp'
7979
include ':airbyte-webapp-e2e-tests'
80+
include ':airbyte-cron'
8081
}
8182

8283
// connectors base

0 commit comments

Comments
 (0)