Generate JUnit XML report for tutorials (#4203)

Fixes #4055

---------

Signed-off-by: Pavel Chekin <[email protected]>

Author: pbchekin

.github/workflows/build-test-reusable.yml

@@ -387,6 +387,13 @@ jobs:
           name: pass_rate-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
           path: pass_rate*.json
 
+      - name: Upload tutorials test report
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-reports-tutorials-${{ inputs.python_version }}-${{ inputs.runner_label || inputs.driver_version }}
+          include-hidden-files: true
+          path: reports/tutorials.xml
+
       - name: Upload tutorials performance report
         uses: actions/upload-artifact@v4
         with:

scripts/pass_rate.py

@@ -7,6 +7,8 @@
 import os
 import pathlib
 import platform
+import xml.etree.ElementTree as et
+
 from typing import List
 
 from defusedxml.ElementTree import parse
@@ -124,31 +126,51 @@ def find_stats(stats: List[ReportStats], name: str) -> ReportStats:
     raise ValueError(f'{name} not found')
 
 
-def parse_junit_reports(args: argparse.Namespace) -> List[ReportStats]:
+def parse_junit_reports(reports_path: pathlib.Path) -> List[ReportStats]:
     """Parses junit report in the specified directory."""
-    reports_path = pathlib.Path(args.reports)
     return [parse_report(report) for report in reports_path.glob('*.xml')]
 
 
-def parse_tutorials_reports(args: argparse.Namespace) -> List[ReportStats]:
-    """Parses tutorials reports in the specified directory."""
-    reports_path = pathlib.Path(args.reports)
-    stats = ReportStats(name='tutorials')
-    for report in reports_path.glob('tutorial-*.txt'):
-        result = report.read_text().strip()
-        stats.total += 1
+# pylint: disable=too-many-locals
+def generate_junit_report(reports_path: pathlib.Path):
+    """Parses info files for tutorials and generates JUnit report.
+    The script `run_tutorial.py` generates `tutorial-*.json` files in the reports directory.
+    This function loads them and generates `tutorials.xml` file (JUnit XML report) in the same
+    directory.
+    """
+    testsuites = et.Element('testsuites')
+    testsuite = et.SubElement(testsuites, 'testsuite', name='tutorials')
+
+    total_tests, total_errors, total_failures, total_skipped = 0, 0, 0, 0
+    total_time = 0.0
+
+    for item in reports_path.glob('tutorial-*.json'):
+        data = json.loads(item.read_text())
+        name, result, time = data['name'], data['result'], data.get('time', 0)
+        testcase = et.SubElement(testsuite, 'testcase', name=name)
         if result == 'PASS':
-            stats.passed += 1
+            testcase.set('time', str(time))
         elif result == 'SKIP':
-            stats.skipped += 1
+            total_skipped += 1
+            et.SubElement(testcase, 'skipped', type='pytest.skip')
         elif result == 'FAIL':
-            stats.failed += 1
-    return [stats]
+            total_failures += 1
+            et.SubElement(testcase, 'failure', message=data.get('message', ''))
+        else:
+            continue
+        total_tests += 1
+        total_time += time
 
+    testsuite.set('tests', str(total_tests))
+    testsuite.set('errors', str(total_errors))
+    testsuite.set('failures', str(total_failures))
+    testsuite.set('skipped', str(total_skipped))
+    testsuite.set('time', str(total_time))
 
-def parse_reports(args: argparse.Namespace) -> List[ReportStats]:
-    """Parses all report in the specified directory."""
-    return parse_junit_reports(args) + parse_tutorials_reports(args)
+    report_path = reports_path / 'tutorials.xml'
+    with report_path.open('wb') as f:
+        tree = et.ElementTree(testsuites)
+        tree.write(f, encoding='UTF-8', xml_declaration=True)
 
 
 def print_text_stats(stats: ReportStats):
@@ -194,9 +216,10 @@ def print_json_stats(stats: ReportStats):
 def main():
     """Main."""
     args = create_argument_parser().parse_args()
-    args.report_path = pathlib.Path(args.reports)
 
-    stats = parse_reports(args)
+    reports_path = pathlib.Path(args.reports)
+    generate_junit_report(reports_path)
+    stats = parse_junit_reports(reports_path)
 
     if args.suite == 'all':
         summary = overall_stats(stats)

scripts/pytest-utils.sh

@@ -63,36 +63,18 @@ run_tutorial_test() {
     echo "****** Running $1 test ******"
     echo
 
-    TUTORIAL_RESULT=TODO
-
-    if [[ -f $TRITON_TEST_SKIPLIST_DIR/tutorials.txt ]]; then
-        if grep --fixed-strings --quiet "$1" "$TRITON_TEST_SKIPLIST_DIR/tutorials.txt"; then
-            TUTORIAL_RESULT=SKIP
-        fi
-    fi
-
-    if [[ $TRITON_TEST_REPORTS = true ]]; then
-        RUN_TUTORIAL="python -u $SCRIPTS_DIR/run_tutorial.py --reports $TRITON_TEST_REPORTS_DIR $1.py"
-    else
-        RUN_TUTORIAL="python -u $1.py"
-    fi
-
-    if [[ $TUTORIAL_RESULT = TODO ]]; then
-        if $RUN_TUTORIAL; then
-            TUTORIAL_RESULT=PASS
-        else
-            TUTORIAL_RESULT=FAIL
-        fi
-    fi
+    run_tutorial_args=(
+        "--skip-list=$TRITON_TEST_SKIPLIST_DIR/tutorials.txt"
+        "$1.py"
+    )
 
     if [[ $TRITON_TEST_REPORTS = true ]]; then
-        mkdir -p "$TRITON_TEST_REPORTS_DIR"
-        echo $TUTORIAL_RESULT > "$TRITON_TEST_REPORTS_DIR/tutorial-$1.txt"
+        run_tutorial_args+=(
+            "--reports=$TRITON_TEST_REPORTS_DIR"
+        )
     fi
 
-    if [[ $TUTORIAL_RESULT = FAIL && $TRITON_TEST_IGNORE_ERRORS = false ]]; then
-        exit 1
-    fi
+    python -u "$SCRIPTS_DIR/run_tutorial.py" "${run_tutorial_args[@]}" || $TRITON_TEST_IGNORE_ERRORS
 }
 
 capture_runtime_env() {

scripts/run_tutorial.py

@@ -1,10 +1,15 @@
 """Script to run a Triton tutorial and collect generated csv files."""
 
 import argparse
+import dataclasses
+import datetime
 import importlib.util
+import json
 import pathlib
 import shutil
 import tempfile
+import sys
+from typing import Set, Optional
 
 import triton.testing
 
@@ -33,38 +38,95 @@ def create_argument_parser() -> argparse.ArgumentParser:
     """Creates argument parser."""
     parser = argparse.ArgumentParser()
     parser.add_argument('tutorial', help='Tutorial to run')
-    parser.add_argument('--reports', required=False, type=str, default='.',
-                        help='Directory to store tutorial CSV reports, default: %(default)s')
+    parser.add_argument('--reports', required=False, type=str, help='Directory to store tutorial CSV reports')
+    parser.add_argument('--skip-list', required=False, type=str, help='Skip list for tutorials')
     return parser
 
 
-def run_tutorial(path: pathlib.Path):
-    """Runs """
+def get_skip_list(path: pathlib.Path) -> Set[str]:
+    """Loads skip list from the specified file."""
+    skip_list = set()
+    if path.exists() and path.is_file():
+        for item in path.read_text().splitlines():
+            skip_list.add(item.strip())
+    return skip_list
+
+
+def run_tutorial(path: pathlib.Path) -> float:
+    """Runs tutorial."""
     spec = importlib.util.spec_from_file_location('__main__', path)
     if not spec or not spec.loader:
         raise AssertionError(f'Failed to load module from {path}')
     module = importlib.util.module_from_spec(spec)
-    # set __file__ to the absolute name, a workaround for 10i-experimental-block-pointer, which
+    # Set __file__ to the absolute name, a workaround for 10i-experimental-block-pointer, which
     # uses dirname of its location to find 10-experimental-block-pointer.
     module.__file__ = path.resolve().as_posix()
+    # Reset sys.argv because some tutorials, such as 09, parse their command line arguments.
+    sys.argv = [str(path)]
+    start_time = datetime.datetime.now()
     spec.loader.exec_module(module)
+    elapsed_time = datetime.datetime.now() - start_time
+    return elapsed_time.total_seconds()
+
+
+@dataclasses.dataclass
+class TutorialInfo:
+    """A record about tutorial execution."""
+    name: str
+    path: Optional[pathlib.Path] = None
+
+    def _report(self, **kwargs):
+        """Writes tutorial info."""
+        if self.path:
+            with self.path.open(mode='w') as f:
+                json.dump({'name': self.name} | kwargs, f)
+
+    def report_pass(self, time: float):
+        """Reports successful tutorial."""
+        self._report(result='PASS', time=time)
+
+    def report_skip(self):
+        """Reports skipped tutorial."""
+        self._report(result='SKIP')
+
+    def report_fail(self, message: str):
+        """Reports failed tutorial."""
+        self._report(result='FAIL', message=message)
 
 
 def main():
     """Main."""
+    skip_list = set()
     args = create_argument_parser().parse_args()
     tutorial_path = pathlib.Path(args.tutorial)
-    reports_path = pathlib.Path(args.reports)
+
+    reports_path = pathlib.Path(args.reports) if args.reports else None
+    if args.skip_list:
+        skip_list = get_skip_list(pathlib.Path(args.skip_list))
+
     name = tutorial_path.stem
-    report_path = reports_path / name
-    report_path.mkdir(parents=True, exist_ok=True)
+    if reports_path:
+        report_path = reports_path / name
+        report_path.mkdir(parents=True, exist_ok=True)
+        info = TutorialInfo(name=name, path=reports_path / f'tutorial-{name}.json')
+    else:
+        info = TutorialInfo(name=name)
 
     def perf_report(benchmarks):
         """Marks a function for benchmarking."""
         return lambda fn: CustomMark(fn, benchmarks, report_path)
 
-    triton.testing.perf_report = perf_report
-    run_tutorial(tutorial_path)
+    if name in skip_list:
+        info.report_skip()
+    else:
+        if reports_path:
+            triton.testing.perf_report = perf_report
+        try:
+            time = run_tutorial(tutorial_path)
+            info.report_pass(time)
+        except Exception as e:
+            info.report_fail(str(e))
+            raise
 
 
 if __name__ == '__main__':

scripts/test_pass_rate.py

@@ -1,3 +1,5 @@
+from xml.etree.ElementTree import parse
+
 import pass_rate
 
 WARNINGS = """\
@@ -34,3 +36,37 @@ def test_get_warnings(tmp_path):
     assert len(warnings) == 1
     assert warnings[0].location == 'location'
     assert warnings[0].message == 'message'
+
+
+def test_generate_junit_report(tmp_path):
+    file1 = tmp_path / 'tutorial-1.json'
+    file2 = tmp_path / 'tutorial-2.json'
+    file3 = tmp_path / 'tutorial-3.json'
+
+    file1.write_text('{"name": "1", "result": "PASS", "time": 1.0}')
+    file2.write_text('{"name": "2", "result": "SKIP"}')
+    file3.write_text('{"name": "3", "result": "FAIL", "message": "Error"}')
+
+    pass_rate.generate_junit_report(tmp_path)
+
+    report_path = tmp_path / 'tutorials.xml'
+    assert report_path.exists()
+
+    xml = parse(report_path)
+    testsuites = xml.getroot()
+    assert testsuites.tag == 'testsuites'
+
+    testsuite = testsuites[0]
+    assert testsuite.tag == 'testsuite'
+    assert testsuite.get('name') == 'tutorials'
+    assert testsuite.get('tests') == '3'
+    assert testsuite.get('skipped') == '1'
+    assert testsuite.get('failures') == '1'
+
+    stats = pass_rate.parse_junit_reports(tmp_path)
+    assert len(stats) == 1
+    assert stats[0].name == 'tutorials'
+    assert stats[0].passed == 1
+    assert stats[0].failed == 1
+    assert stats[0].skipped == 1
+    assert stats[0].total == 3