Add JUnit Test Reports to build outputs. (#15271)

It's difficult today to parse all the logs from tests. Engineers waste time scrolling through the log outputs and looking for the relevant stack trace.

This PR adds an action to generate a JUnit report so devs can understand test results at a glance. This generates 3 reports for each of the main build jobs when the build completes. We leave the frontend build out since this is aggregated by cypress.

See https://github.com/airbytehq/airbyte/pull/15271/checks?check_run_id=7683783016 for an example of how this works.

Use the https://github.com/dorny/test-reporter action and configure this to look at the Jacoco test report output for top level and second level builds. Note that most of the parameters into the action don't work.

Comment out the trap commands to output logs. Though this can be useful for debugging, there is little practical use in the day-to-day, and results in extremely noisy logs.

Author: davinchia

.github/workflows/gradle.yml

@@ -234,6 +234,15 @@ jobs:
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
 
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v1
+        if: always()
+        with:
+          name: Connectors Base Test Report
+          # Specify top-level and second-level modules. Note there cannot be a space between the comma.
+          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml'
+          reporter: java-junit
+
   # In case of self-hosted EC2 errors, remove this block.
   stop-connectors-base-build-runner:
     name: "Connectors Base: Stop Build EC2 Runner"
@@ -471,7 +480,8 @@ jobs:
           SUB_BUILD=PLATFORM ./gradlew build javadoc --scan
 
       - name: Integration test
-        run: SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
+        run: |
+          SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
 
       - name: Slow integration test
         if: contains(github.ref, 'bump-version') || contains(github.ref, 'master')
@@ -485,16 +495,25 @@ jobs:
         if: success() && github.ref == 'refs/heads/master'
         run: ./tools/site/link_checker.sh check_docs
 
-      # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
-      #      - name: Get Docker Space
-      #        run: docker run --rm busybox df -h
+#       This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
+#      - name: Get Docker Space
+#        run: docker run --rm busybox df -h
 
       - name: Run End-to-End Acceptance Tests
         run: ./tools/bin/acceptance_test.sh
 
       - name: Automatic Migration Acceptance Test
         run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
 
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v1
+        if: always()    # run this step even if previous step failed
+        with:
+          name: Platform Test Report with Docker E2E Test
+          # Specify top-level and second-level modules. Note there cannot be a space between the comma.
+          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml'
+          reporter: java-junit
+
   # In case of self-hosted EC2 errors, remove this block.
   stop-platform-build-runner:
     name: "Platform: Stop Build EC2 Runner"
@@ -626,6 +645,14 @@ jobs:
         run: |
           CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
 
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v1
+        if: always()    # run this step even if previous step failed
+        with:
+          name: Platform Kubernetes E2E Test Report
+          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
+          reporter: java-junit
+
       - uses: actions/upload-artifact@v2
         if: failure()
         with:

airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java

@@ -70,7 +70,9 @@ public void startBlocking() throws IOException, InterruptedException {
   @SuppressWarnings({"unchecked", "rawtypes"})
   public void startAsync() throws IOException, InterruptedException {
     final File cleanedDockerComposeFile = prepareDockerComposeFile(dockerComposeFile);
-    dockerComposeContainer = new DockerComposeContainer(cleanedDockerComposeFile).withEnv(env);
+    dockerComposeContainer = new DockerComposeContainer(cleanedDockerComposeFile)
+        .withEnv(env);
+
     // Only expose logs related to db migrations.
     serviceLogConsumer(dockerComposeContainer, "init");
     serviceLogConsumer(dockerComposeContainer, "bootloader");

tools/bin/acceptance_test.sh

@@ -14,7 +14,7 @@ VERSION=dev TRACKING_STRATEGY=logging USE_STREAM_CAPABLE_STATE=true docker-compo
 # Sometimes source/dest containers using airbyte volumes survive shutdown, which need to be killed in order to shut down properly.
 shutdown_cmd="docker-compose down -v || docker kill \$(docker ps -a -f volume=airbyte_workspace -f volume=airbyte_data -f volume=airbyte_db -q) && docker-compose down -v"
 # Uncomment for debugging. Warning, this is verbose.
-#trap "echo 'docker-compose logs:' && docker-compose logs -t --tail 1000 && $shutdown_cmd" EXIT
+# trap "echo 'docker-compose logs:' && docker-compose logs -t --tail 1000 && $shutdown_cmd" EXIT
 
 echo "Waiting for services to begin"
 while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/api/v1/health)" != "200" ]]; do echo "Waiting for docker deployment.."; sleep 5; done

tools/bin/e2e_test.sh

@@ -16,7 +16,8 @@ mkdir -p /tmp/airbyte_local
 
 # Detach so we can run subsequent commands
 VERSION=dev TRACKING_STRATEGY=logging docker-compose up -d
-trap 'echo "docker-compose logs:" && docker-compose logs -t --tail 1000 && docker-compose down && docker stop airbyte_ci_pg' EXIT
+# Uncomment for debugging. Warning, this is verbose.
+# trap 'echo "docker-compose logs:" && docker-compose logs -t --tail 1000 && docker-compose down && docker stop airbyte_ci_pg' EXIT
 
 docker run --rm -d -p 5433:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci --name airbyte_ci_pg postgres
 echo "Waiting for health API to be available..."