🎉 Single py checker (#10246)

Author: antixar

.github/actions/ci-java-tests/action.yml

@@ -9,11 +9,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install Java
-      uses: actions/setup-java@v1
-      with:
-        java-version: '17'
-
     - name: "Build"
       shell: bash
       run: |

.github/actions/ci-py-tests/action.yml

@@ -27,57 +27,26 @@ runs:
     - name: Build Coverage Reports
       id: build-coverage-reports
       shell: bash
-      working-directory: ${{ inputs.module-folder }}
       run: |
-        virtualenv .venv
-        source .venv/bin/activate
-        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
-        pip install coverage[toml]~=6.2
-        mkdir -p .venv/source-acceptance-test
-        mkdir -p reports
-        SAT_DIR=$(git rev-parse --show-toplevel)/airbyte-integrations/bases/source-acceptance-test
-        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
-        git ls-tree -r HEAD --name-only $SAT_DIR | while read src; do cp -f $src .venv/source-acceptance-test; done
-        pip install build
-        python -m build .venv/source-acceptance-test
-        pip install .venv/source-acceptance-test/dist/source_acceptance_test-*.whl
-        [ -f requirements.txt ] && pip install --quiet -r requirements.txt
-        pip install .[tests]
-        coverage run --rcfile=${PYPROJECT_CONFIG} -m pytest ./unit_tests || true
-        coverage xml --rcfile=${PYPROJECT_CONFIG} -o reports/coverage.xml || true
+        GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "unitTest")
+        REPORT_FOLDER="${{ inputs.module-folder }}/coverage/"
+        ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
-        rm -rf .venv
-        echo "::set-output name=coverage-paths::reports/coverage.xml"
+        echo "::set-output name=coverage-paths::coverage/coverage.xml"
 
     - name: Upload coverage to Codecov
-      if: ${{ always() }}
       uses: codecov/codecov-action@v2
+      with:
+        file: ${{ steps.build-coverage-reports.outputs.coverage-paths }}
+        name: "UnitTests of ${{ inputs.module-name }}"
 
     - name: Build Linter Reports
       id: build-linter-reports
       shell: bash
-      working-directory: ${{ inputs.module-folder }}
       run: |
-        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
-        REPORT_FOLDER=reports
-        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
-
-        # run mypy
-        pip install lxml~=4.7 mypy~=0.910 .
-        mypy . --config-file=${PYPROJECT_CONFIG} | tee reports/mypy.log || true
-
-        # run black
-        pip install black~=21.12b0
-        XDG_CACHE_HOME=/dev/null black --config ${PYPROJECT_CONFIG} --diff . | tee reports/black.diff
-
-        # run isort
-        pip install isort~=5.10.1
-        cp ${PYPROJECT_CONFIG} ./pyproject.toml
-        isort --diff . | tee reports/isort.diff
-
-        # run flake8
-        pip install mccabe~=0.6.1 pyproject-flake8~=0.0.1a2
-        pflake8 --exit-zero . | grep ^. | tee reports/flake.txt
+        GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "airbytePythonReport")
+        REPORT_FOLDER="${{ inputs.module-folder }}/reports/"
+        ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
         echo "::set-output name=mypy-logs::reports/mypy.log"
         echo "::set-output name=black-diff::reports/black.diff"

.github/actions/ci-tests-runner/action.yml

@@ -32,13 +32,18 @@ runs:
       with:
         python-version: 3.7
 
+    - name: Install Java
+      uses: actions/setup-java@v1
+      with:
+        java-version: '17'
+
     - name: Tests of CI
       shell: bash
       run: |
         # all CI python packages have the prefix "ci_"
         pip install --quiet tox==3.24.4
-        tox -r -c ./tools/tox_ci.ini
         pip install --quiet -e ./tools/ci_*
+        tox -r -c ./tools/tox_ci.ini
         echo "::echo::off"
 
     - name: Auth with gcloud CLI
@@ -109,6 +114,7 @@ runs:
           [ -f ${{ steps.ci-py-tests.outputs.coverage-paths }} ] && OPTIONS+=("-Dsonar.python.coverage.reportPaths=${{ steps.ci-py-tests.outputs.coverage-paths }}")
           [ -f ${{ steps.ci-py-tests.outputs.flake8-logs }} ] && OPTIONS+=("-Dsonar.python.flake8.reportPaths=${{ steps.ci-py-tests.outputs.flake8-logs }}")
         fi
+        cat ${REPORT_FOLDER}/*
         if [ ${{ inputs.module-lang }} == 'java' ]; then
              [ -d "./src/main/java" ] && OPTIONS+=("-Dsonar.sources=./src/main/java")
              [ -d "./src/test/java" ] && OPTIONS+=("-Dsonar.tests=./src/test/java")
@@ -133,6 +139,7 @@ runs:
         MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))")
         echo "::set-output name=module_dir::${MODULE_DIR}"
 
+
     - name: SonarQube Scan
 
       uses: sonarsource/sonarqube-scan-action@master
@@ -148,7 +155,7 @@ runs:
             -Dsonar.language=${{ inputs.module-lang }}
             -Dsonar.sourceEncoding=UTF-8
             -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }}
-            -Dsonar.exclusions=reports/**,*.toml
+            -Dsonar.exclusions=reports/**,*.toml,*_tests/**,setup.py,main.py
             -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }}
             ${{ steps.sq-options.outputs.options }}
 

.github/workflows/sonar-scan.yml

@@ -72,7 +72,7 @@ jobs:
         sonar-token: ${{ secrets.SONAR_TOKEN }}
         sonar-gcp-access-key: ${{ secrets.GCP_SONAR_SA_KEY }}
         pull-request-id: "${{ github.repository }}/${{ github.event.pull_request.number }}"
-        remove-sonar-project: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
+        remove-sonar-project: ${{  github.event.action == 'closed' }}
 
 
 

buildSrc/src/main/groovy/airbyte-python.gradle

@@ -29,19 +29,29 @@ class Helpers {
 
         See https://github.com/airbytehq/airbyte/issues/4979 for original context
         """
-        if (project.file(testFilesDirectory).exists()) {
 
+        if (project.file(testFilesDirectory).exists()) {
+            def outputArg = project.hasProperty('reports_folder') ?"-otemp_coverage.xml" : "--skip-empty"
+            def coverageFormat = project.hasProperty('reports_folder') ? 'xml' : 'report'
+            def testConfig = project.file('pytest.ini').exists() ? 'pytest.ini' : project.rootProject.file('pyproject.toml').absolutePath
             project.projectDir.toPath().resolve(testFilesDirectory).traverse(type: FileType.FILES, nameFilter: ~/(^test_.*|.*_test)\.py$/) { file ->
                 project.task("_${taskName}Coverage", type: PythonTask, dependsOn: taskDependencies) {
                     module = "coverage"
-                    command = "run --data-file=${testFilesDirectory}/.coverage.${taskName} --rcfile=${project.rootProject.file('tools/python/.coveragerc').absolutePath} -m pytest -s ${testFilesDirectory}"
+                    command = "run --data-file=${testFilesDirectory}/.coverage.${taskName} --rcfile=${project.rootProject.file('pyproject.toml').absolutePath} -m pytest -s ${testFilesDirectory} -c ${testConfig}"
                 }
                 // generation of coverage report is optional and we should skip it if tests are empty
+
                 project.task(taskName, type: Exec){
                     commandLine = ".venv/bin/python"
-                    args "-m", "coverage", "report", "--data-file=${testFilesDirectory}/.coverage.${taskName}", "--rcfile=${project.rootProject.file('tools/python/.coveragerc').absolutePath}"
+                    args "-m", "coverage", coverageFormat, "--data-file=${testFilesDirectory}/.coverage.${taskName}", "--rcfile=${project.rootProject.file('pyproject.toml').absolutePath}", outputArg
                     dependsOn project.tasks.findByName("_${taskName}Coverage")
                     setIgnoreExitValue true
+                    doLast {
+                        // try to move a generated report to custom report folder if needed
+                        if (project.file('temp_coverage.xml').exists() && project.hasProperty('reports_folder')) {
+                            project.file('temp_coverage.xml').renameTo(project.file("${project.reports_folder}/coverage.xml"))
+                        }
+                    }
 
                 }
                 // If a file is found, terminate the traversal, thus causing this task to be declared at most once
@@ -63,6 +73,7 @@ class AirbytePythonPlugin implements Plugin<Project> {
 
     void apply(Project project) {
         def extension = project.extensions.create('airbytePython', AirbytePythonConfiguration)
+
         def venvDirectoryName = '.venv'
         project.plugins.apply 'ru.vyarus.use-python'
 
@@ -71,34 +82,63 @@ class AirbytePythonPlugin implements Plugin<Project> {
             minPythonVersion = '3.7'
             scope = 'VIRTUALENV'
             installVirtualenv = true
-            pip 'flake8:3.8.4'
-            pip 'black:21.10b0'
-            pip 'mypy:0.812'
+            pip 'pip:21.3.1'
+            pip 'mccabe:0.6.1'
+            // flake8 doesn't support pyproject.toml files
+            // and thus there is the wrapper "pyproject-flake8" for this
+            pip 'pyproject-flake8:0.0.1a2'
+            pip 'black:22.1.0'
+            pip 'mypy:0.930'
             pip 'isort:5.6.4'
             pip 'pytest:6.1.2'
-            pip 'pip:21.1.3'
             pip 'coverage[toml]:6.3.1'
         }
 
 
         project.task('isortFormat', type: PythonTask) {
             module = "isort"
-            command = ". --settings-file ${project.rootProject.file('tools/python/.isort.cfg').absolutePath}"
+            command = "--settings-file=${project.rootProject.file('pyproject.toml').absolutePath} ./"
+        }
+
+        project.task('isortReport', type: PythonTask) {
+            module = "isort"
+            command = "--settings-file=${project.rootProject.file('pyproject.toml').absolutePath} --diff --quiet ./"
+            outputPrefix = ''
         }
 
         project.task('blackFormat', type: PythonTask) {
             module = "black"
             // the line length should match .isort.cfg
-            command = ". --line-length 140"
+            command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./"
             dependsOn project.rootProject.licenseFormat
             dependsOn project.isortFormat
         }
 
+        project.task('blackReport', type: PythonTask) {
+            module = "black"
+            command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} --diff --quiet ./"
+            outputPrefix = ''
+        }
+
         project.task('flakeCheck', type: PythonTask, dependsOn: project.blackFormat) {
-            module = "flake8"
-            command = ". --config ${project.rootProject.file('tools/python/.flake8').absolutePath}"
+            module = "pflake8"
+            command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./"
+        }
+
+        project.task('flakeReport', type: PythonTask) {
+            module = "pflake8"
+            command = "--exit-zero --config ${project.rootProject.file('pyproject.toml').absolutePath} ./"
+            outputPrefix = ''
+        }
+
+        project.task("mypyReport", type: Exec){
+            commandLine = ".venv/bin/python"
+            args "-m", "mypy", "--config-file", "${project.rootProject.file('pyproject.toml').absolutePath}", "./"
+            setIgnoreExitValue true
         }
 
+
+
         // attempt to install anything in requirements.txt. by convention this should only be dependencies whose source is located in the project.
 
         if (project.file('requirements.txt').exists()) {
@@ -109,7 +149,7 @@ class AirbytePythonPlugin implements Plugin<Project> {
                 outputs.file('build/installedlocalreqs.txt')
 
                 // HACK: makes all integrations depend on installing requirements for bases. long term we should resolve deps and install in order.
-                if (project.getPath().startsWith(":airbyte-integrations:connectors")) {
+                if (project.getPath().startsWith(":airbyte-integrations:connectors") && !project.hasProperty("reports_folder")) {
                     dependsOn project.rootProject.getTasksByName("airbytePythonApply", true).findAll { it.project.getPath().startsWith(":airbyte-integrations:bases") }
                 }
             }
@@ -139,6 +179,7 @@ class AirbytePythonPlugin implements Plugin<Project> {
         }
 
         Helpers.addTestTaskIfTestFilesFound(project, 'unit_tests', 'unitTest', project.installTestReqs)
+
         Helpers.addTestTaskIfTestFilesFound(project, 'integration_tests', 'customIntegrationTests', project.installTestReqs)
         if (!project.tasks.findByName('integrationTest')) {
             project.task('integrationTest')
@@ -148,7 +189,7 @@ class AirbytePythonPlugin implements Plugin<Project> {
         if (extension.moduleDirectory) {
             project.task('mypyCheck', type: PythonTask) {
                 module = "mypy"
-                command = "-m ${extension.moduleDirectory} --config-file ${project.rootProject.file('tools/python/.mypy.ini').absolutePath}"
+                command = "-m ${extension.moduleDirectory} --config-file ${project.rootProject.file('pyproject.toml').absolutePath}"
             }
 
             project.check.dependsOn mypyCheck
@@ -160,6 +201,36 @@ class AirbytePythonPlugin implements Plugin<Project> {
             dependsOn project.flakeCheck
         }
 
+        project.task('airbytePythonReport', type: DefaultTask) {
+            dependsOn project.blackReport
+            dependsOn project.isortReport
+            dependsOn project.flakeReport
+            dependsOn project.mypyReport
+            doLast {
+               if (project.hasProperty('reports_folder')) {
+                      // Gradles adds some log messages to files and we must remote them
+                      // examples of these lines:
+                      // :airbyte-integrations:connectors: ...
+                      // [python] .venv/bin/python -m black  ...
+                      project.fileTree(project.reports_folder).visit { FileVisitDetails details ->
+                          project.println "Found the report file: " + details.file.path
+                          def tempFile = project.file(details.file.path + ".1")
+                          details.file.eachLine { line ->
+                                if ( !line.startsWith(":airbyte") && !line.startsWith("[python]") ) {
+                                    tempFile << line + "\n"
+                                }
+                          }
+                          if (!tempFile.exists()) {
+                                // generate empty file
+                                tempFile << "\n"
+                          }
+                          tempFile.renameTo(details.file)
+
+                      }
+               }
+            }
+        }
+
         project.task('airbytePythonApply', type: DefaultTask) {
             dependsOn project.installReqs
             dependsOn project.airbytePythonFormat
@@ -187,5 +258,53 @@ class AirbytePythonPlugin implements Plugin<Project> {
         project.assemble.dependsOn project.airbytePythonApply
         project.assemble.dependsOn project.airbytePythonTest
         project.test.dependsOn project.airbytePythonTest
+
+                // saves tools reports to a custom folder
+        def reportsFolder = project.hasProperty('reports_folder') ? project.reports_folder : ''
+        if ( reportsFolder != '' ) {
+
+            // clean reports folders
+            project.file(reportsFolder).deleteDir()
+            project.file(reportsFolder).mkdirs()
+
+
+
+            project.tasks.blackReport.configure {
+                it.logging.addStandardOutputListener(new StandardOutputListener() {
+                    @Override
+                    void onOutput(CharSequence charSequence) {
+                        project.file("$reportsFolder/black.diff") << charSequence
+                    }
+                })
+            }
+            project.tasks.isortReport.configure {
+                it.logging.addStandardOutputListener(new StandardOutputListener() {
+                    @Override
+                    void onOutput(CharSequence charSequence) {
+                        project.file("$reportsFolder/isort.diff") << charSequence
+                    }
+                })
+            }
+
+            project.tasks.flakeReport.configure {
+                it.logging.addStandardOutputListener(new StandardOutputListener() {
+                    @Override
+                    void onOutput(CharSequence charSequence) {
+                        project.file("$reportsFolder/flake.txt") << charSequence
+                    }
+                })
+            }
+
+            project.tasks.mypyReport.configure {
+                it.logging.addStandardOutputListener(new StandardOutputListener() {
+                    @Override
+                    void onOutput(CharSequence charSequence) {
+                        project.file("$reportsFolder/mypy.log") << charSequence
+                    }
+                })
+            }
+
+        }
     }
 }
+