add conformance reporting (#1333)

* add conformance reporting

Author: yliuuuu

.github/workflows/conformance-report.yml

@@ -3,8 +3,10 @@ on: [push, pull_request]
 
 env:
   PATH_TO_TEST_RUNNER: test/partiql-tests-runner
-  CONFORMANCE_REPORT_NAME: conformance_test_results.ion
+  CONFORMANCE_REPORT_RELATIVE_PATH: build/conformance-test-report
   COMPARISON_REPORT_NAME: comparison_report.md
+  COMPARISON_REPORT_NAME_WITH_LIMIT: comparison_report_limited.md
+  COMMENT_SIZE_LIMIT: 10
 
 jobs:
   conformance-report:
@@ -13,6 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
+          ref: ${{ github.event.pull_request.head.sha }}
           submodules: recursive
       - name: Use Java 17
         uses: actions/setup-java@v3
@@ -27,12 +30,12 @@ jobs:
       # Run the conformance tests and save to an Ion file.
       - name: gradle test of the conformance tests (can fail) and save to Ion file
         continue-on-error: true
-        run: gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
+        run: gradle :test:partiql-tests-runner:generateTestReport
       # Upload conformance report for future viewing and comparison with future runs.
-      - name: Upload `conformance_test_results.ion`
+      - name: Upload `conformance-test-report` folder
         uses: actions/upload-artifact@v3
         with:
-          path: ${{ env.PATH_TO_TEST_RUNNER }}/${{ env.CONFORMANCE_REPORT_NAME }}
+          path: ${{ env.PATH_TO_TEST_RUNNER }}/build/conformance-test-report
       # Cache the conformance report for `conformance-report-comparison` job (pull_request event only)
       - name: Cache conformance report and build
         if: github.event_name == 'pull_request'
@@ -86,23 +89,34 @@ jobs:
         continue-on-error: true
         run: |
           cd ${{ github.event.pull_request.base.sha }}
-          gradle :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
+          gradle :test:partiql-tests-runner:generateTestReport
       - name: (If download of target branch conformance report fails) Move conformance test report of target branch to ./artifact directory
         if: ${{ steps.download-report.outcome == 'failure' }}
         continue-on-error: true
         run: |
           mkdir -p $GITHUB_WORKSPACE/artifact
-          cp -r $GITHUB_WORKSPACE/${{ github.event.pull_request.base.sha }}/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_NAME $GITHUB_WORKSPACE/artifact/$CONFORMANCE_REPORT_NAME
+          cp -r $GITHUB_WORKSPACE/${{ github.event.pull_request.base.sha }}/$PATH_TO_TEST_RUNNER/$CONFORMANCE_REPORT_RELATIVE_PATH $GITHUB_WORKSPACE/artifact/$CONFORMANCE_REPORT_RELATIVE_PATH
       # Run conformance report comparison. Generates `comparison_report.md`
-      - name: Run conformance report comparison. Generates `comparison_report.md`
+      - name: Run conformance report comparison for artifact. Generates `comparison_report.md`
         continue-on-error: true
         run: |
-          ARGS="$GITHUB_WORKSPACE/artifact/$CONFORMANCE_REPORT_NAME $CONFORMANCE_REPORT_NAME ${{ github.event.pull_request.base.sha }} $GITHUB_SHA $COMPARISON_REPORT_NAME"
+          ARGS="$GITHUB_WORKSPACE/artifact $CONFORMANCE_REPORT_RELATIVE_PATH ${{ github.event.pull_request.base.sha }} $GITHUB_SHA $COMPARISON_REPORT_NAME"
           gradle :test:partiql-tests-runner:run --args="$ARGS"
       # Print conformance report to GitHub actions workflow summary page
       - name: Print markdown in run
         continue-on-error: true
         run: cat $PATH_TO_TEST_RUNNER/$COMPARISON_REPORT_NAME >> $GITHUB_STEP_SUMMARY
+      # Upload the full comparison report to CI artifact
+      - name: Upload `comparison_report.md`
+        uses: actions/upload-artifact@v3
+        with:
+          path: ${{ env.PATH_TO_TEST_RUNNER }}/comparison_report.md
+      # Rebuild the test report with a size limit for comment
+      - name: Run conformance report comparison for comment. Generates `comparison_report_limited.md`
+        continue-on-error: true
+        run: |
+          ARGS="$GITHUB_WORKSPACE/artifact $CONFORMANCE_REPORT_RELATIVE_PATH ${{ github.event.pull_request.base.sha }} $GITHUB_SHA $COMPARISON_REPORT_NAME_WITH_LIMIT $COMMENT_SIZE_LIMIT"
+          gradle :test:partiql-tests-runner:run --args="$ARGS"
       # Find comment w/ conformance comparison if previous comment published
       - name: Find Comment
         uses: peter-evans/find-comment@v2
@@ -119,5 +133,5 @@ jobs:
         with:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ github.event.pull_request.number }}
-          body-file: ${{ env.PATH_TO_TEST_RUNNER }}/${{ env.COMPARISON_REPORT_NAME }}
+          body-file: ${{ env.PATH_TO_TEST_RUNNER }}/${{ env.COMPARISON_REPORT_NAME_WITH_LIMIT }}
           edit-mode: replace

test/partiql-tests-runner/README.md

@@ -12,22 +12,18 @@ This package enables:
 
 ```shell
 # default, test data from partiql-tests submodule will be used
-./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
+./gradlew :test:partiql-tests-runner:ConformanceTestReport
 
 # override test data location
 PARTIQL_TESTS_DATA=/path/to/partiql-tests/data \
-./gradlew :test:partiql-tests-runner:test --tests "*ConformanceTestReport" -PconformanceReport
+./gradlew :test:partiql-tests-runner:ConformanceTestReport
 ```
-The report is written into file `test/partiql-tests-runner/conformance_test_results.ion`.
-
-## Run Conformance Tests in UI
-
-The above project property `-PconformanceReport` is checked in `test/partiql-tests-runner/build.gradle.kts`,
-to exclude the conformance test suite from executing during a normal project-build test run. 
-Unfortunately, this also disables running `ConformanceTestReport` in a UI runner. 
-To make that possible locally, temporarily comment out the check in `test/partiql-tests-runner/build.gradle.kts`.
+The report is written into folder `test/partiql-tests-runner/build/conformance_test_results`.
 
 ## Compare Conformance Reports locally
+The report contains two type of comparison: 
+1. Cross Commit: Comparing using the same engine based on the pull request commit and head of target branch commit. 
+2. Cross Engine: Comparing using different engine based on the pull request commit. 
 
 ```shell
 ./gradlew :test:partiql-tests-runner:run --args="pathToFirstConformanceTestResults pathToSecondConformanceTestResults firstCommitId secondCommitId pathToComparisonReport"

test/partiql-tests-runner/build.gradle.kts

@@ -31,6 +31,7 @@ dependencies {
 }
 
 val tests = System.getenv()["PARTIQL_TESTS_DATA"] ?: "../partiql-tests/partiql-tests-data"
+val reportDir = file("$buildDir/conformance-test-report").absolutePath
 
 object Env {
     const val PARTIQL_EVAL = "PARTIQL_EVAL_TESTS_DATA"
@@ -43,10 +44,34 @@ tasks.test {
     environment(Env.PARTIQL_EQUIV, file("$tests/eval-equiv/").absolutePath)
 
     // To make it possible to run ConformanceTestReport in unit test UI runner, comment out this check:
-    if (!project.hasProperty("conformanceReport")) {
-        exclude("org/partiql/runner/ConformanceTestReport.class")
-    }
+    exclude("org/partiql/runner/ConformanceTestEval.class", "org/partiql/runner/ConformanceTestLegacy.class")
 
     // May 2023: Disabled conformance testing during regular project build, because fail lists are out of date.
     exclude("org/partiql/runner/ConformanceTest.class")
 }
+
+val createReportDir by tasks.registering {
+    if (File(reportDir).exists()) {
+        delete(File(reportDir))
+    }
+    mkdir(reportDir)
+}
+
+val generateTestReport by tasks.registering(Test::class) {
+    dependsOn(createReportDir)
+    useJUnitPlatform()
+    environment(Env.PARTIQL_EVAL, file("$tests/eval/").absolutePath)
+    environment(Env.PARTIQL_EQUIV, file("$tests/eval-equiv/").absolutePath)
+    environment("conformanceReportDir", reportDir)
+    include("org/partiql/runner/ConformanceTestEval.class", "org/partiql/runner/ConformanceTestLegacy.class")
+    if (project.hasProperty("Engine")) {
+        val engine = property("Engine")!! as String
+        if (engine.toLowerCase() == "legacy") {
+            exclude("org/partiql/runner/ConformanceTestEval.class")
+        } else if (engine.toLowerCase() == "eval") {
+            exclude("org/partiql/runner/ConformanceTestLegacy.class")
+        } else {
+            throw InvalidUserDataException("Expect engine property to be either Legacy or Eval, received $engine")
+        }
+    }
+}