Remove shared getDockerBuildTask to enable parallel docker builds

airbyte-bootloader/build.gradle

@@ -24,25 +24,15 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-bootloader-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("bootloader", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 // produce reproducible archives
 // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
 tasks.withType(AbstractArchiveTask) {
     preserveFileTimestamps = false
     reproducibleFileOrder = true
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-bootloader/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=bootloader

airbyte-cli/build.gradle

@@ -1,3 +1,3 @@
-Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyDocker)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyDocker
+}

airbyte-cli/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=cli

airbyte-config/init/build.gradle

@@ -22,8 +22,8 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("init", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyScripts)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyScripts
+}
 
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-config/init/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=init

airbyte-container-orchestrator/build.gradle

@@ -36,16 +36,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-container-orchestrator-*.tar'
-    }
-    into 'build/docker/bin'
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
 }
-
-Task dockerBuildTask = getDockerBuildTask("container-orchestrator", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-container-orchestrator/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=container-orchestrator

airbyte-cron/build.gradle

@@ -22,16 +22,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-cron-*.tar'
-    }
-    into 'build/docker/bin'
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
 }
-
-Task dockerBuildTask = getDockerBuildTask("cron", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-cron/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=cron

airbyte-db/db-lib/build.gradle

@@ -1,3 +1,5 @@
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+
 plugins {
     id 'java-library'
 }
@@ -92,8 +94,8 @@ task copyInitSql(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("db", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyInitSql)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyInitSql
+}
 
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-db/db-lib/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=db

airbyte-metrics/reporter/build.gradle

@@ -17,16 +17,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-metrics-reporter-*.tar'
-    }
-    into 'build/docker/bin'
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
 }
-
-Task dockerBuildTask = getDockerBuildTask("metrics-reporter", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)

airbyte-metrics/reporter/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=metrics-reporter

airbyte-server/build.gradle

@@ -84,25 +84,15 @@ run {
     environment "TEMPORAL_HOST", "localhost:7233"
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-server-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 // produce reproducible archives
 // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
 tasks.withType(AbstractArchiveTask) {
     preserveFileTimestamps = false
     reproducibleFileOrder = true
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-server/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=server

airbyte-temporal/build.gradle

@@ -5,6 +5,6 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyScripts)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyScripts
+}

airbyte-temporal/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=temporal

airbyte-webapp/build.gradle

@@ -117,10 +117,10 @@ check.dependsOn validateLinks, licenseCheck, test, stylelint
 
 build.dependsOn buildStorybook
 
-Task dockerBuildTask = getDockerBuildTask("webapp", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyDocker)
-dockerBuildTask.dependsOn(copyBuildOutput)
-dockerBuildTask.dependsOn(copyNginx)
-dockerBuildTask.dependsOn(copyDocs)
-dockerBuildTask.dependsOn(copyDocAssets)
-assemble.dependsOn(dockerBuildTask)
+tasks.named("buildDockerImage") {
+    dependsOn copyDocker
+    dependsOn copyBuildOutput
+    dependsOn copyNginx
+    dependsOn copyDocs
+    dependsOn copyDocAssets
+}

airbyte-webapp/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=webapp

airbyte-workers/build.gradle

@@ -73,20 +73,6 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-workers-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("worker", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 task cloudStorageIntegrationTest(type: Test) {
     useJUnitPlatform {
         includeTags cloudStorageTestTagName
@@ -96,4 +82,8 @@ task cloudStorageIntegrationTest(type: Test) {
     }
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-workers/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=worker

build.gradle

@@ -156,33 +156,6 @@ spotless {
 }
 check.dependsOn 'spotlessApply'
 
-@SuppressWarnings('GroovyAssignabilityCheck')
-def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) {
-    return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) {
-        // This is currently only used for connectors.
-        def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'
-
-        def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase()
-        def isArm64 = arch == "aarch64" || arch == "arm64"
-
-        def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64'
-        def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'
-        def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine'
-        def openjdkImage = System.getenv('JDK_IMAGE') ?: isArm64 ? "arm64v8/openjdk:19-slim-bullseye" : "amd64/openjdk:19-slim-bullseye"
-        def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64'
-
-        inputDir = file("$projectDir/build/docker")
-        platform = buildPlatform
-        images.add("airbyte/$artifactName:$buildTag")
-        buildArgs.put('JDK_VERSION', jdkVersion)
-        buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
-        buildArgs.put('ALPINE_IMAGE', alpineImage)
-        buildArgs.put('NGINX_IMAGE', nginxImage)
-        buildArgs.put('JDK_IMAGE', openjdkImage)
-        buildArgs.put('VERSION', buildVersion)
-    })
-}
-
 @SuppressWarnings('GroovyAssignabilityCheck')
 def Task getPublishArtifactsTask(String buildVersion, Project subproject) {
     // generate a unique task name based on the directory name.
@@ -229,9 +202,62 @@ allprojects {
 }
 
 // Java projects common configurations
-subprojects {
+subprojects { subproj ->
+
+    // Common Docker Configuration:
+    // If subprojects have the dockerImageName property configured in their gradle.properties file,
+    // register:
+    // 1) A copyGeneratedTar task to copy generated TARs. Subprojects that produce TARs can use this
+    //    to copy the produced tar into the docker image.
+    // 2) A buildDockerImage task to build the docker image.
+    // 3) Make the docker image task depend on the default assemble task.
+    if (subproj.hasProperty('dockerImageName')) {
+        project.logger.lifecycle("configuring docker task for $subproj.name")
+
+        // Although this task is defined for every subproject with the dockerImageName property,
+        // It is not necessarily used for all subprojects. Non-TAR producing projects can ignore this.
+        tasks.register("copyGeneratedTar", Copy) {
+            dependsOn copyDocker
+            dependsOn distTar
+            from('build/distributions') {
+                // Assume that tars are named <parent-project-name>-<child-project-name>-*.tar.
+                // Because we only have a handle to the child project, and to keep things simple,
+                // use a * regex to catch all prefixes.
+                include "*$subproj.name-*.tar"
+            }
+            into 'build/docker/bin'
+        }
+
+        tasks.register("buildDockerImage", DockerBuildImage) {
+            // This is currently only used for connectors.
+            def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'
+
+            def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase()
+            def isArm64 = arch == "aarch64" || arch == "arm64"
+
+            def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64'
+            def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'
+            def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine'
+            def openjdkImage = System.getenv('JDK_IMAGE') ?: isArm64 ? "arm64v8/openjdk:19-slim-bullseye" : "amd64/openjdk:19-slim-bullseye"
+            def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64'
+
+            platform = buildPlatform
+            images.add("airbyte/$subproj.dockerImageName:$rootProject.ext.image_tag")
+            buildArgs.put('JDK_VERSION', jdkVersion)
+            buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
+            buildArgs.put('ALPINE_IMAGE', alpineImage)
+            buildArgs.put('NGINX_IMAGE', nginxImage)
+            buildArgs.put('JDK_IMAGE', openjdkImage)
+            buildArgs.put('VERSION', rootProject.ext.version)
+
+        }
+
+        tasks.named("assemble") {
+            dependsOn buildDockerImage
+        }
+    }
 
-    if (project.name == 'airbyte-webapp' || project.name == 'airbyte-webapp-e2e-tests') {
+    if (subproj.name == 'airbyte-webapp' || subproj.name == 'airbyte-webapp-e2e-tests') {
         return
     }
 
@@ -361,7 +387,7 @@ subprojects {
     }
 
     dependencies {
-        if (project.name != 'airbyte-commons') {
+        if (subproj.name != 'airbyte-commons') {
             implementation project(':airbyte-commons')
         }
 

docs/contributing-to-airbyte/developing-on-docker.md

@@ -4,12 +4,14 @@
 
 The docker build is fully incremental for the platform build, which means that it will only build an image if it is needed. We need to keep it that 
 way.
-A task generator, `getDockerBuildTask`, is available for building a docker image for any given module. Behind the scene, it will generate a 
-task which will run the build of a docker image in a specific folder. The goal is to make sure that we have an isolated 
-context which helps with incrementality. All files that need to be present in the docker image will need to be copy into this folder. The generate 
-method takes 2 arguments:
-- The image name, for example if `foo` is given as an image name, the image `airbyte/foo` will be created
-- The project directory folder. It is needed because the `getDockerBuildTask` is declared in the rootProject
+The top level `build.gradle` file defines several convenient tasks for building a docker image.
+1) The `copyGeneratedTar` task copies a generated TAR file from a default location into the default location used by the [docker plugin](https://github.com/bmuschko/gradle-docker-plugin).
+2) The `buildDockerImage` task is a convenience class for configuring the above linked docker plugin that centralizes configuration logic commonly found in our dockerfiles.
+3) Makes the `buildDockerImage` task depend on the Gradle `assemble` task.
+
+These tasks are added to a subproject if the subproject has a `gradle.properties` file with the `dockerImageName` property.
+
+This property is sets the build docker image's name.
 
 ## Adding a new docker build