Skip to content

Commit ef673c5

Browse files
authored
Inject check connection resource (#10410)
Make it possible to set resource limits specifically for Check Connection. This helps speed up the Check Connection operation for Java based connectors. After this PR is merged, I will do an OSS release and make the required Helm changes in Cloud.
1 parent 6bc9240 commit ef673c5

File tree

4 files changed

+185
-63
lines changed

4 files changed

+185
-63
lines changed

airbyte-config/models/src/main/java/io/airbyte/config/Configs.java

+24
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,30 @@ public interface Configs {
238238
Map<String, String> getJobDefaultEnvMap();
239239

240240
// Jobs - Kube only
241+
/**
242+
* Define the check job container's minimum CPU request. Defaults to
243+
* {@link #getJobMainContainerCpuRequest()} if not set. Internal-use only.
244+
*/
245+
String getCheckJobMainContainerCpuRequest();
246+
247+
/**
248+
* Define the check job container's maximum CPU usage. Defaults to
249+
* {@link #getJobMainContainerCpuLimit()} if not set. Internal-use only.
250+
*/
251+
String getCheckJobMainContainerCpuLimit();
252+
253+
/**
254+
* Define the job container's minimum RAM usage. Defaults to
255+
* {@link #getJobMainContainerMemoryRequest()} if not set. Internal-use only.
256+
*/
257+
String getCheckJobMainContainerMemoryRequest();
258+
259+
/**
260+
* Define the job container's maximum RAM usage. Defaults to
261+
* {@link #getJobMainContainerMemoryLimit()} if not set. Internal-use only.
262+
*/
263+
String getCheckJobMainContainerMemoryLimit();
264+
241265
/**
242266
* Define one or more Job pod tolerations. Tolerations are separated by ';'. Each toleration
243267
* contains k=v pairs mentioning some/all of key, effect, operator and value and separated by `,`.

airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java

+25
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ public class EnvConfigs implements Configs {
124124
private static final String DISCOVER_WORKER_STATUS_CHECK_INTERVAL = "DISCOVER_WORKER_STATUS_CHECK_INTERVAL";
125125
private static final String REPLICATION_WORKER_STATUS_CHECK_INTERVAL = "REPLICATION_WORKER_STATUS_CHECK_INTERVAL";
126126

127+
static final String CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST = "CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST";
128+
static final String CHECK_JOB_MAIN_CONTAINER_CPU_LIMIT = "CHECK_JOB_MAIN_CONTAINER_CPU_LIMIT";
129+
static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST = "CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST";
130+
static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT = "CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT";
131+
127132
// defaults
128133
private static final String DEFAULT_SPEC_CACHE_BUCKET = "io-airbyte-cloud-spec-cache";
129134
public static final String DEFAULT_JOB_KUBE_NAMESPACE = "default";
@@ -612,6 +617,26 @@ public Map<String, String> getJobDefaultEnvMap() {
612617
.collect(Collectors.toMap(key -> key.replace(JOB_DEFAULT_ENV_PREFIX, ""), getEnv));
613618
}
614619

620+
@Override
621+
public String getCheckJobMainContainerCpuRequest() {
622+
return getEnvOrDefault(CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST, getJobMainContainerCpuRequest());
623+
}
624+
625+
@Override
626+
public String getCheckJobMainContainerCpuLimit() {
627+
return getEnvOrDefault(CHECK_JOB_MAIN_CONTAINER_CPU_LIMIT, getJobMainContainerCpuLimit());
628+
}
629+
630+
@Override
631+
public String getCheckJobMainContainerMemoryRequest() {
632+
return getEnvOrDefault(CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST, getJobMainContainerMemoryRequest());
633+
}
634+
635+
@Override
636+
public String getCheckJobMainContainerMemoryLimit() {
637+
return getEnvOrDefault(CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT, getJobMainContainerMemoryLimit());
638+
}
639+
615640
@Override
616641
public LogConfigs getLogConfigs() {
617642
return logConfigs;

0 commit comments

Comments
 (0)