Skip to content

Commit 294ee8f

Browse files
mfsiega-airbyteterencechoalafanechere
authored
Expose cron scheduling in the Connections APIs (#15253)
* Expose cron scheduling in the Connections APIs * Update airbyte-api/src/main/openapi/config.yaml Co-authored-by: terencecho <[email protected]> * Update airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java Co-authored-by: terencecho <[email protected]> * update octavia-cli tests for new schedule schema, and fix update API impl * check for null schedule data before updating * handle new schedule related fields in generate / apply / import * update octavia-cli changelog * ensure that legacy manual schedule flag is consistent with schedule_type * update octavia cli test coverage for new schedule schema * fix failing octavia cli integration tests * fix file diff check * Update octavia-cli/unit_tests/test_apply/test_resources.py Co-authored-by: Augustin <[email protected]> Co-authored-by: terencecho <[email protected]> Co-authored-by: alafanechere <[email protected]>
1 parent f664bc9 commit 294ee8f

File tree

26 files changed

+14939
-12893
lines changed

26 files changed

+14939
-12893
lines changed

‎airbyte-api/src/main/openapi/config.yaml

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3218,6 +3218,10 @@ components:
32183218
$ref: "#/components/schemas/AirbyteCatalog"
32193219
schedule:
32203220
$ref: "#/components/schemas/ConnectionSchedule"
3221+
scheduleType:
3222+
$ref: "#/components/schemas/ConnectionScheduleType"
3223+
scheduleData:
3224+
$ref: "#/components/schemas/ConnectionScheduleData"
32213225
status:
32223226
$ref: "#/components/schemas/ConnectionStatus"
32233227
resourceRequirements:
@@ -3257,6 +3261,10 @@ components:
32573261
$ref: "#/components/schemas/AirbyteCatalog"
32583262
schedule:
32593263
$ref: "#/components/schemas/ConnectionSchedule"
3264+
scheduleType:
3265+
$ref: "#/components/schemas/ConnectionScheduleType"
3266+
scheduleData:
3267+
$ref: "#/components/schemas/ConnectionScheduleData"
32603268
status:
32613269
$ref: "#/components/schemas/ConnectionStatus"
32623270
resourceRequirements:
@@ -3298,6 +3306,10 @@ components:
32983306
$ref: "#/components/schemas/AirbyteCatalog"
32993307
schedule:
33003308
$ref: "#/components/schemas/ConnectionSchedule"
3309+
scheduleType:
3310+
$ref: "#/components/schemas/ConnectionScheduleType"
3311+
scheduleData:
3312+
$ref: "#/components/schemas/ConnectionScheduleData"
33013313
status:
33023314
$ref: "#/components/schemas/ConnectionStatus"
33033315
resourceRequirements:
@@ -3335,6 +3347,10 @@ components:
33353347
$ref: "#/components/schemas/AirbyteCatalog"
33363348
schedule:
33373349
$ref: "#/components/schemas/ConnectionSchedule"
3350+
scheduleType:
3351+
$ref: "#/components/schemas/ConnectionScheduleType"
3352+
scheduleData:
3353+
$ref: "#/components/schemas/ConnectionScheduleData"
33383354
status:
33393355
$ref: "#/components/schemas/ConnectionStatus"
33403356
resourceRequirements:
@@ -3386,6 +3402,10 @@ components:
33863402
$ref: "#/components/schemas/AirbyteCatalog"
33873403
schedule:
33883404
$ref: "#/components/schemas/ConnectionSchedule"
3405+
scheduleType:
3406+
$ref: "#/components/schemas/ConnectionScheduleType"
3407+
scheduleData:
3408+
$ref: "#/components/schemas/ConnectionScheduleData"
33893409
status:
33903410
$ref: "#/components/schemas/ConnectionStatus"
33913411
resourceRequirements:
@@ -3416,6 +3436,10 @@ components:
34163436
$ref: "#/components/schemas/DestinationId"
34173437
schedule:
34183438
$ref: "#/components/schemas/ConnectionSchedule"
3439+
scheduleType:
3440+
$ref: "#/components/schemas/ConnectionScheduleType"
3441+
scheduleData:
3442+
$ref: "#/components/schemas/ConnectionScheduleData"
34193443
status:
34203444
$ref: "#/components/schemas/ConnectionStatus"
34213445
source:
@@ -3445,6 +3469,10 @@ components:
34453469
$ref: "#/components/schemas/DestinationId"
34463470
schedule:
34473471
$ref: "#/components/schemas/ConnectionSchedule"
3472+
scheduleType:
3473+
$ref: "#/components/schemas/ConnectionScheduleType"
3474+
scheduleData:
3475+
$ref: "#/components/schemas/ConnectionScheduleData"
34483476
status:
34493477
$ref: "#/components/schemas/ConnectionStatus"
34503478
source:
@@ -3467,6 +3495,8 @@ components:
34673495
- active
34683496
- inactive
34693497
- deprecated
3498+
# TODO(https://github.com/airbytehq/airbyte/issues/11432): remove.
3499+
# Prefer the ConnectionScheduleType and ConnectionScheduleData properties.
34703500
ConnectionSchedule:
34713501
description: if null, then no schedule is set.
34723502
type: object
@@ -3485,6 +3515,46 @@ components:
34853515
- days
34863516
- weeks
34873517
- months
3518+
ConnectionScheduleType:
3519+
description: determine how the schedule data should be interpreted
3520+
type: string
3521+
enum:
3522+
- manual
3523+
- basic
3524+
- cron
3525+
ConnectionScheduleData:
3526+
description: schedule for when the the connection should run, per the schedule type
3527+
type: object
3528+
properties:
3529+
# This should be populated when schedule type is basic.
3530+
basicSchedule:
3531+
type: object
3532+
required:
3533+
- timeUnit
3534+
- units
3535+
properties:
3536+
timeUnit:
3537+
type: string
3538+
enum:
3539+
- minutes
3540+
- hours
3541+
- days
3542+
- weeks
3543+
- months
3544+
units:
3545+
type: integer
3546+
format: int64
3547+
# This should be populated when schedule type is cron.
3548+
cron:
3549+
type: object
3550+
required:
3551+
- cronExpression
3552+
- cronTimeZone
3553+
properties:
3554+
cronExpression:
3555+
type: string
3556+
cronTimeZone:
3557+
type: string
34883558
NamespaceDefinitionType:
34893559
type: string
34903560
description: Method used for computing final namespace in destination
@@ -4564,6 +4634,10 @@ components:
45644634
$ref: "#/components/schemas/AirbyteCatalog"
45654635
schedule:
45664636
$ref: "#/components/schemas/ConnectionSchedule"
4637+
scheduleType:
4638+
$ref: "#/components/schemas/ConnectionScheduleType"
4639+
scheduleData:
4640+
$ref: "#/components/schemas/ConnectionScheduleData"
45674641
status:
45684642
$ref: "#/components/schemas/ConnectionStatus"
45694643
operationIds:

‎airbyte-server/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ dependencies {
3232
implementation 'org.glassfish.jersey.inject:jersey-hk2'
3333
implementation 'org.glassfish.jersey.media:jersey-media-json-jackson'
3434
implementation 'org.glassfish.jersey.ext:jersey-bean-validation'
35+
implementation 'org.quartz-scheduler:quartz:2.3.2'
3536

3637

3738
testImplementation project(':airbyte-test-utils')

‎airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java

Lines changed: 89 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,13 @@
44

55
package io.airbyte.server.converters;
66

7+
import io.airbyte.api.client.model.generated.ConnectionScheduleType;
78
import io.airbyte.api.model.generated.ActorDefinitionResourceRequirements;
89
import io.airbyte.api.model.generated.ConnectionRead;
910
import io.airbyte.api.model.generated.ConnectionSchedule;
11+
import io.airbyte.api.model.generated.ConnectionScheduleData;
12+
import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule;
13+
import io.airbyte.api.model.generated.ConnectionScheduleDataCron;
1014
import io.airbyte.api.model.generated.ConnectionStatus;
1115
import io.airbyte.api.model.generated.ConnectionUpdate;
1216
import io.airbyte.api.model.generated.JobType;
@@ -17,7 +21,10 @@
1721
import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType;
1822
import io.airbyte.config.Schedule;
1923
import io.airbyte.config.StandardSync;
24+
import io.airbyte.config.StandardSync.ScheduleType;
2025
import io.airbyte.server.handlers.helpers.CatalogConverter;
26+
import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper;
27+
import io.airbyte.validation.json.JsonValidationException;
2128
import java.util.stream.Collectors;
2229

2330
public class ApiPojoConverters {
@@ -78,7 +85,7 @@ public static ResourceRequirements resourceRequirementsToApi(final io.airbyte.co
7885
.memoryLimit(resourceReqs.getMemoryLimit());
7986
}
8087

81-
public static io.airbyte.config.StandardSync connectionUpdateToInternal(final ConnectionUpdate update) {
88+
public static io.airbyte.config.StandardSync connectionUpdateToInternal(final ConnectionUpdate update) throws JsonValidationException {
8289

8390
final StandardSync newConnection = new StandardSync()
8491
.withNamespaceDefinition(Enums.convertTo(update.getNamespaceDefinition(), NamespaceDefinitionType.class))
@@ -99,7 +106,9 @@ public static io.airbyte.config.StandardSync connectionUpdateToInternal(final Co
99106
}
100107

101108
// update sync schedule
102-
if (update.getSchedule() != null) {
109+
if (update.getScheduleType() != null) {
110+
ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(newConnection, update.getScheduleType(), update.getScheduleData());
111+
} else if (update.getSchedule() != null) {
103112
final Schedule newSchedule = new Schedule()
104113
.withTimeUnit(toPersistenceTimeUnit(update.getSchedule().getTimeUnit()))
105114
.withUnits(update.getSchedule().getUnits());
@@ -112,21 +121,12 @@ public static io.airbyte.config.StandardSync connectionUpdateToInternal(final Co
112121
}
113122

114123
public static ConnectionRead internalToConnectionRead(final StandardSync standardSync) {
115-
ConnectionSchedule apiSchedule = null;
116-
117-
if (!standardSync.getManual()) {
118-
apiSchedule = new ConnectionSchedule()
119-
.timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit()))
120-
.units(standardSync.getSchedule().getUnits());
121-
}
122-
123124
final ConnectionRead connectionRead = new ConnectionRead()
124125
.connectionId(standardSync.getConnectionId())
125126
.sourceId(standardSync.getSourceId())
126127
.destinationId(standardSync.getDestinationId())
127128
.operationIds(standardSync.getOperationIds())
128129
.status(toApiStatus(standardSync.getStatus()))
129-
.schedule(apiSchedule)
130130
.name(standardSync.getName())
131131
.namespaceDefinition(Enums.convertTo(standardSync.getNamespaceDefinition(), io.airbyte.api.model.generated.NamespaceDefinitionType.class))
132132
.namespaceFormat(standardSync.getNamespaceFormat())
@@ -138,6 +138,8 @@ public static ConnectionRead internalToConnectionRead(final StandardSync standar
138138
connectionRead.resourceRequirements(resourceRequirementsToApi(standardSync.getResourceRequirements()));
139139
}
140140

141+
populateConnectionReadSchedule(standardSync, connectionRead);
142+
141143
return connectionRead;
142144
}
143145

@@ -149,10 +151,15 @@ public static io.airbyte.config.JobTypeResourceLimit.JobType toInternalJobType(f
149151
return Enums.convertTo(jobType, io.airbyte.config.JobTypeResourceLimit.JobType.class);
150152
}
151153

154+
// TODO(https://github.com/airbytehq/airbyte/issues/11432): remove these helpers.
152155
public static ConnectionSchedule.TimeUnitEnum toApiTimeUnit(final Schedule.TimeUnit apiTimeUnit) {
153156
return Enums.convertTo(apiTimeUnit, ConnectionSchedule.TimeUnitEnum.class);
154157
}
155158

159+
public static ConnectionSchedule.TimeUnitEnum toApiTimeUnit(final BasicSchedule.TimeUnit timeUnit) {
160+
return Enums.convertTo(timeUnit, ConnectionSchedule.TimeUnitEnum.class);
161+
}
162+
156163
public static ConnectionStatus toApiStatus(final StandardSync.Status status) {
157164
return Enums.convertTo(status, ConnectionStatus.class);
158165
}
@@ -169,4 +176,75 @@ public static BasicSchedule.TimeUnit toBasicScheduleTimeUnit(final ConnectionSch
169176
return Enums.convertTo(apiTimeUnit, BasicSchedule.TimeUnit.class);
170177
}
171178

179+
public static BasicSchedule.TimeUnit toBasicScheduleTimeUnit(final ConnectionScheduleDataBasicSchedule.TimeUnitEnum apiTimeUnit) {
180+
return Enums.convertTo(apiTimeUnit, BasicSchedule.TimeUnit.class);
181+
}
182+
183+
public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicScheduleTimeUnit(final BasicSchedule.TimeUnit timeUnit) {
184+
return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class);
185+
}
186+
187+
public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicScheduleTimeUnit(final Schedule.TimeUnit timeUnit) {
188+
return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class);
189+
}
190+
191+
public static void populateConnectionReadSchedule(final StandardSync standardSync, final ConnectionRead connectionRead) {
192+
// TODO(https://github.com/airbytehq/airbyte/issues/11432): only return new schema once frontend is
193+
// ready.
194+
if (standardSync.getScheduleType() != null) {
195+
// Populate everything based on the new schema.
196+
switch (standardSync.getScheduleType()) {
197+
case MANUAL -> {
198+
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL);
199+
}
200+
case BASIC_SCHEDULE -> {
201+
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC);
202+
connectionRead.scheduleData(new ConnectionScheduleData()
203+
.basicSchedule(new ConnectionScheduleDataBasicSchedule()
204+
.timeUnit(toApiBasicScheduleTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit()))
205+
.units(standardSync.getScheduleData().getBasicSchedule().getUnits())));
206+
connectionRead.schedule(new ConnectionSchedule()
207+
.timeUnit(toApiTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit()))
208+
.units(standardSync.getScheduleData().getBasicSchedule().getUnits()));
209+
}
210+
case CRON -> {
211+
// We don't populate any legacy data here.
212+
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.CRON);
213+
connectionRead.scheduleData(new ConnectionScheduleData()
214+
.cron(new ConnectionScheduleDataCron()
215+
.cronExpression(standardSync.getScheduleData().getCron().getCronExpression())
216+
.cronTimeZone(standardSync.getScheduleData().getCron().getCronTimeZone())));
217+
}
218+
}
219+
} else if (standardSync.getManual()) {
220+
// Legacy schema, manual sync.
221+
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL);
222+
} else {
223+
// Legacy schema, basic schedule.
224+
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC);
225+
connectionRead.schedule(new ConnectionSchedule()
226+
.timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit()))
227+
.units(standardSync.getSchedule().getUnits()));
228+
connectionRead.scheduleData(new ConnectionScheduleData()
229+
.basicSchedule(new ConnectionScheduleDataBasicSchedule()
230+
.timeUnit(toApiBasicScheduleTimeUnit(standardSync.getSchedule().getTimeUnit()))
231+
.units(standardSync.getSchedule().getUnits())));
232+
}
233+
}
234+
235+
public static ConnectionScheduleType toApiScheduleType(final ScheduleType scheduleType) {
236+
switch (scheduleType) {
237+
case MANUAL -> {
238+
return ConnectionScheduleType.MANUAL;
239+
}
240+
case BASIC_SCHEDULE -> {
241+
return ConnectionScheduleType.BASIC;
242+
}
243+
case CRON -> {
244+
return ConnectionScheduleType.CRON;
245+
}
246+
}
247+
throw new RuntimeException("Unexpected schedule type");
248+
}
249+
172250
}

‎airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import io.airbyte.server.converters.CatalogDiffConverters;
4848
import io.airbyte.server.handlers.helpers.CatalogConverter;
4949
import io.airbyte.server.handlers.helpers.ConnectionMatcher;
50+
import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper;
5051
import io.airbyte.server.handlers.helpers.DestinationMatcher;
5152
import io.airbyte.server.handlers.helpers.SourceMatcher;
5253
import io.airbyte.validation.json.JsonValidationException;
@@ -140,6 +141,34 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate)
140141
standardSync.withCatalog(new ConfiguredAirbyteCatalog().withStreams(Collections.emptyList()));
141142
}
142143

144+
if (connectionCreate.getSchedule() != null && connectionCreate.getScheduleType() != null) {
145+
throw new JsonValidationException("supply old or new schedule schema but not both");
146+
}
147+
148+
if (connectionCreate.getScheduleType() != null) {
149+
ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(standardSync, connectionCreate.getScheduleType(),
150+
connectionCreate.getScheduleData());
151+
} else {
152+
populateSyncFromLegacySchedule(standardSync, connectionCreate);
153+
}
154+
155+
configRepository.writeStandardSync(standardSync);
156+
157+
trackNewConnection(standardSync);
158+
159+
try {
160+
LOGGER.info("Starting a connection manager workflow");
161+
eventRunner.createConnectionManagerWorkflow(connectionId);
162+
} catch (final Exception e) {
163+
LOGGER.error("Start of the connection manager workflow failed", e);
164+
configRepository.deleteStandardSyncDefinition(standardSync.getConnectionId());
165+
throw e;
166+
}
167+
168+
return buildConnectionRead(connectionId);
169+
}
170+
171+
private void populateSyncFromLegacySchedule(final StandardSync standardSync, final ConnectionCreate connectionCreate) {
143172
if (connectionCreate.getSchedule() != null) {
144173
final Schedule schedule = new Schedule()
145174
.withTimeUnit(ApiPojoConverters.toPersistenceTimeUnit(connectionCreate.getSchedule().getTimeUnit()))
@@ -159,21 +188,6 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate)
159188
standardSync.withManual(true);
160189
standardSync.withScheduleType(ScheduleType.MANUAL);
161190
}
162-
163-
configRepository.writeStandardSync(standardSync);
164-
165-
trackNewConnection(standardSync);
166-
167-
try {
168-
LOGGER.info("Starting a connection manager workflow");
169-
eventRunner.createConnectionManagerWorkflow(connectionId);
170-
} catch (final Exception e) {
171-
LOGGER.error("Start of the connection manager workflow failed", e);
172-
configRepository.deleteStandardSyncDefinition(standardSync.getConnectionId());
173-
throw e;
174-
}
175-
176-
return buildConnectionRead(connectionId);
177191
}
178192

179193
private void trackNewConnection(final StandardSync standardSync) {

0 commit comments

Comments
 (0)