gradle: split off python cdk (#35306)

Author: Marius Posta

.github/workflows/publish-cdk-command-manually.yml

@@ -73,7 +73,7 @@ jobs:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
       - name: Build CDK Package
-        run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
+        run: (cd airbyte-cdk/python; ./gradlew --no-daemon --no-build-cache :build)
       - name: Post failure to Slack channel dev-connectors-extensibility
         if: ${{ failure() }}
         uses: slackapi/[email protected]

airbyte-cdk/python/bin/build_code_generator_image.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+DOCKER_BUILD_ARCH="${DOCKER_BUILD_ARCH:-amd64}"
+# https://docs.docker.com/develop/develop-images/build_enhancements/
+export DOCKER_BUILDKIT=1
+
+CODE_GENERATOR_DOCKERFILE="$(dirname $0)/../code-generator/Dockerfile"
+test -f $CODE_GENERATOR_DOCKERFILE
+docker build --build-arg DOCKER_BUILD_ARCH="$DOCKER_BUILD_ARCH" -t "airbyte/code-generator:dev" - < $CODE_GENERATOR_DOCKERFILE

airbyte-cdk/python/build.gradle

@@ -1,25 +1,134 @@
+import ru.vyarus.gradle.plugin.python.task.PythonTask
+
 plugins {
-    id 'airbyte-python'
-    id 'airbyte-docker-legacy'
+    id 'base'
+    id 'ru.vyarus.use-python' version '2.3.0'
 }
 
+def generateCodeGeneratorImage = tasks.register('generateCodeGeneratorImage', Exec) {
+    commandLine 'bin/build_code_generator_image.sh'
+}
 def generateComponentManifestClassFiles = tasks.register('generateComponentManifestClassFiles', Exec) {
-    environment 'ROOT_DIR', rootDir.absolutePath
+    environment 'ROOT_DIR', rootDir.parentFile.parentFile.absolutePath
     commandLine 'bin/generate-component-manifest-files.sh'
 }
 generateComponentManifestClassFiles.configure {
-    dependsOn project(':tools:code-generator').tasks.named('assemble')
+    dependsOn generateCodeGeneratorImage
 }
 tasks.register('generate').configure {
     dependsOn generateComponentManifestClassFiles
 }
 
 tasks.register('validateSourceYamlManifest', Exec) {
-    environment 'ROOT_DIR', rootDir.absolutePath
+    environment 'ROOT_DIR', rootDir.parentFile.parentFile.absolutePath
     commandLine 'bin/validate-yaml-schema.sh'
 }
 
 tasks.register('runLowCodeConnectorUnitTests', Exec) {
-    environment 'ROOT_DIR', rootDir.absolutePath
+    environment 'ROOT_DIR', rootDir.parentFile.parentFile.absolutePath
     commandLine 'bin/low-code-unit-tests.sh'
 }
+
+def venvDirectoryName = '.venv'
+
+// Add a task that allows cleaning up venvs to every python project
+def cleanPythonVenv = tasks.register('cleanPythonVenv', Exec) {
+    commandLine 'rm'
+    args '-rf', "${projectDir.absolutePath}/${venvDirectoryName}"
+}
+
+tasks.named('clean').configure {
+    dependsOn cleanPythonVenv
+}
+
+// Configure gradle python plugin.
+python {
+    envPath = venvDirectoryName
+    minPythonVersion '3.10'
+
+    // Amazon Linux support.
+    // The airbyte-ci tool runs gradle tasks in AL2023-based containers.
+    // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly.
+    // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details.
+    try {
+        if ("python3.11 --version".execute().waitFor() == 0) {
+            // python3.11 definitely exists at this point, use it instead of 'python3'.
+            pythonBinary "python3.11"
+        }
+    } catch (IOException _) {
+        // Swallow exception if python3.11 is not installed.
+    }
+    // Pyenv support.
+    try {
+        def pyenvRoot = "pyenv root".execute()
+        def pyenvLatest = "pyenv latest ${minPythonVersion}".execute()
+        // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases.
+        pythonBinary "python"
+        if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) {
+            pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin"
+        }
+    } catch (IOException _) {
+        // Swallow exception if pyenv is not installed.
+    }
+
+    scope 'VIRTUALENV'
+    installVirtualenv = true
+    pip 'pip:23.2.1'
+    pip 'mccabe:0.6.1'
+    // https://github.com/csachs/pyproject-flake8/issues/13
+    pip 'flake8:4.0.1'
+    // flake8 doesn't support pyproject.toml files
+    // and thus there is the wrapper "pyproject-flake8" for this
+    pip 'pyproject-flake8:0.0.1a2'
+    pip 'pytest:6.2.5'
+    pip 'coverage[toml]:6.3.1'
+}
+
+def installLocalReqs = tasks.register('installLocalReqs', PythonTask) {
+    module = "pip"
+    command = "install .[dev,tests]"
+    inputs.file('setup.py')
+    outputs.file('build/installedlocalreqs.txt')
+}
+
+def flakeCheck = tasks.register('flakeCheck', PythonTask) {
+    module = "pflake8"
+    command = "--config pyproject.toml ./"
+}
+
+def installReqs = tasks.register('installReqs', PythonTask) {
+    module = "pip"
+    command = "install .[main]"
+    inputs.file('setup.py')
+    outputs.file('build/installedreqs.txt')
+}
+installReqs.configure {
+    dependsOn installLocalReqs
+}
+
+tasks.named('check').configure {
+    dependsOn installReqs
+    dependsOn flakeCheck
+}
+
+def installTestReqs = tasks.register('installTestReqs', PythonTask) {
+    module = "pip"
+    command = "install .[tests]"
+    inputs.file('setup.py')
+    outputs.file('build/installedtestreqs.txt')
+}
+installTestReqs.configure {
+    dependsOn installReqs
+}
+
+def testTask = tasks.register('testPython', PythonTask) {
+    module = "coverage"
+    command = "run --data-file=unit_tests/.coverage.testPython --rcfile=pyproject.toml -m pytest -s unit_tests -c pytest.ini"
+}
+testTask.configure {
+    dependsOn installTestReqs
+}
+
+tasks.named('check').configure {
+    dependsOn testTask
+}

tools/code-generator/Dockerfile renamed to airbyte-cdk/python/code-generator/Dockerfile

@@ -0,0 +1,11 @@
+# NOTE: some of these values are overwritten in CI!
+# NOTE: if you want to override this for your local machine, set overrides in ~/.gradle/gradle.properties
+
+org.gradle.parallel=true
+org.gradle.caching=true
+
+# Note, this might have issues on the normal Github runner.
+org.gradle.vfs.watch=true
+
+# Tune # of cores Gradle uses.
+# org.gradle.workers.max=3

airbyte-cdk/python/gradle.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists