ci: summarize CTest outcomes and retain results

.github/workflows/build.yml

@@ -141,7 +141,16 @@ jobs:
         run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH"
       - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON
       - run: cmake --build build-relinker --parallel 3
-      - run: ctest --test-dir build-relinker --output-on-failure --timeout 120
+      - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 --output-junit "${{ github.workspace }}/ctest-results.xml"
+      - name: Summarize CTest results
+        if: ${{ !cancelled() }}
+        run: python tools/test_summary.py ctest-results.xml
+      - name: Retain CTest results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ctest-relinker-${{ matrix.os }}
+          path: ctest-results.xml
 
   linux:
     needs: eligibility
@@ -166,10 +175,21 @@ jobs:
         run: python3 tools/check_test_registration.py build
       - run: cmake --build build
       - run: cmake --build build --target libs
-      - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log
+      - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit "${{ github.workspace }}/ctest-results.xml"
         env:
           ANYPS5_REQUIRE_VULKAN: 1
           GALLIUM_OVERRIDE_CPU_CAPS: avx
+      - name: Summarize CTest results
+        if: ${{ !cancelled() }}
+        run: python3 tools/test_summary.py ctest-results.xml
+      - name: Retain CTest results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ctest-linux
+          path: |
+            ctest-results.xml
+            ctest.log
       - name: Report tests slower than 30 s
         if: ${{ !cancelled() }}
         shell: bash
@@ -221,7 +241,18 @@ jobs:
       - shell: cmd
         run: cmake --build build --target libs
       - shell: cmd
-        run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log
+        run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit "${{ github.workspace }}/ctest-results.xml"
+      - name: Summarize CTest results
+        if: ${{ !cancelled() }}
+        run: python tools/test_summary.py ctest-results.xml
+      - name: Retain CTest results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ctest-windows
+          path: |
+            ctest-results.xml
+            ctest.log
       - name: Report tests slower than 30 s
         if: ${{ !cancelled() }}
         shell: bash

CMakeLists.txt

@@ -39,6 +39,8 @@ if(BUILD_TESTING)
     if(Python3_Interpreter_FOUND)
         add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
+        add_test(NAME test_summary COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_summary.py)
         add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py)
         add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}

tools/test_summary.py

@@ -0,0 +1,76 @@
+import argparse
+import html
+import os
+import sys
+import xml.etree.ElementTree as ET
+from collections import Counter
+from pathlib import Path
+
+
+def result(test):
+    status = test.get("status")
+    if status == "run":
+        return "Passed", ""
+    if status == "fail":
+        failure = test.find("failure")
+        return "Failed", failure.get("message", "") if failure is not None else ""
+    if status == "disabled":
+        return "Not run", "Disabled"
+    if status == "notrun":
+        skipped = test.find("skipped")
+        reason = skipped.get("message", "") if skipped is not None else ""
+        return ("Skipped" if reason.startswith("SKIP_") else "Not run"), reason
+    raise ValueError(f"Unknown CTest status: {status!r}")
+
+
+def cell(value):
+    return html.escape(value).replace("|", "\\|").replace("\n", " ").replace("\r", " ")
+
+
+def summarize(path):
+    root = ET.parse(path).getroot()
+    if root.tag != "testsuite":
+        raise ValueError("Expected a CTest JUnit testsuite")
+    tests = root.findall("testcase")
+    if not tests:
+        raise ValueError("CTest report contains zero tests; no execution evidence is available")
+    if int(root.get("tests", "-1")) != len(tests):
+        raise ValueError("CTest report test count does not match its results")
+    outcomes = [(test, *result(test)) for test in tests]
+    counts = Counter(status for _, status, _ in outcomes)
+    lines = ["### CTest results", "", "| Selected | Passed | Failed | Skipped | Not run |",
+             "| --- | --- | --- | --- | --- |",
+             f"| {len(tests)} | {counts['Passed']} | {counts['Failed']} | {counts['Skipped']} | {counts['Not run']} |",
+             "", "Counts describe the selected test cases in this CTest report, not individual repeat attempts.",
+             "Skipped tests ran and requested a skip. Not-run tests include disabled tests and tests blocked by failed fixtures.",
+             "The CTest artifact retains names, durations and captured output."]
+    incomplete = [(test, status, reason) for test, status, reason in outcomes if status != "Passed"]
+    if incomplete:
+        lines += ["", "| Test | Result | Reason |", "| --- | --- | --- |"]
+        lines += [f"| {cell(test.get('name', ''))} | {status} | {cell(reason)} |"
+                  for test, status, reason in incomplete]
+    return "\n".join(lines) + "\n"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Summarize CTest JUnit execution and skip counts")
+    parser.add_argument("report", type=Path)
+    args = parser.parse_args()
+    try:
+        summary = summarize(args.report)
+        code = 0
+    except (OSError, ET.ParseError, ValueError) as error:
+        summary = f"### CTest results\n\nNo usable CTest report: {cell(str(error))}.\n"
+        code = 1
+    print(summary)
+    destination = os.environ.get("GITHUB_STEP_SUMMARY")
+    if destination:
+        with open(destination, "a", encoding="utf-8") as stream:
+            stream.write(summary)
+    return code
+
+
+if __name__ == "__main__":
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(encoding="utf-8")
+    raise SystemExit(main())

tools/tests/test_test_summary.py

@@ -0,0 +1,72 @@
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import test_summary
+
+
+class TestSummaryTests(unittest.TestCase):
+    def run_report(self, report):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "results.xml"
+            if report is not None:
+                path.write_text(report, encoding="utf-8")
+            summary = Path(directory) / "summary.md"
+            summary.write_text("Existing summary\n", encoding="utf-8")
+            process = subprocess.run(
+                [sys.executable, test_summary.__file__, str(path)],
+                capture_output=True, encoding="utf-8", timeout=10,
+                env={**os.environ, "GITHUB_STEP_SUMMARY": str(summary)},
+            )
+            self.assertEqual(summary.read_text(encoding="utf-8"), "Existing summary\n" + process.stdout.rstrip("\n") + "\n")
+            return process
+
+    def test_pass_skip_failure_and_not_run_are_distinct(self):
+        process = self.run_report('''<testsuite tests="6">
+            <testcase name="passed" status="run" time="0.1"/>
+            <testcase name="failed" status="fail"><failure message="Timeout"/></testcase>
+            <testcase name="capability" status="notrun"><skipped message="SKIP_RETURN_CODE"/></testcase>
+            <testcase name="skip regex" status="notrun"><skipped message="SKIP_REGULAR_EXPRESSION_MATCHED"/></testcase>
+            <testcase name="disabled" status="disabled"/>
+            <testcase name="blocked" status="notrun"><skipped message="Failed test dependencies: setup"/></testcase>
+        </testsuite>''')
+        self.assertEqual(process.returncode, 0, process.stderr)
+        self.assertIn("| 6 | 1 | 1 | 2 | 2 |", process.stdout)
+        self.assertIn("| disabled | Not run | Disabled |", process.stdout)
+        self.assertIn("| blocked | Not run | Failed test dependencies: setup |", process.stdout)
+        self.assertIn("| failed | Failed | Timeout |", process.stdout)
+
+    def test_all_passed(self):
+        process = self.run_report('<testsuite tests="1"><testcase name="pass" status="run"/></testsuite>')
+        self.assertEqual(process.returncode, 0)
+        self.assertIn("| 1 | 1 | 0 | 0 | 0 |", process.stdout)
+        self.assertNotIn("| Test |", process.stdout)
+
+    def test_missing_empty_and_malformed_reports_fail(self):
+        for report in (None, "broken", '<testsuite tests="0"/>',
+                       '<testsuite tests="2"><testcase status="run"/></testsuite>',
+                       '<testsuite tests="1"><testcase status="unknown"/></testsuite>',
+                       '<other/>'):
+            with self.subTest(report=report):
+                process = self.run_report(report)
+                self.assertEqual(process.returncode, 1)
+                self.assertIn("No usable CTest report:", process.stdout)
+                self.assertNotIn("| Selected |", process.stdout)
+
+    def test_unicode_and_markdown_are_preserved_safely(self):
+        process = self.run_report('''<testsuite tests="1">
+            <testcase name="Yōtei | &lt;tag&gt;" status="notrun">
+                <skipped message="Missing | fixture"/>
+            </testcase>
+        </testsuite>''')
+        self.assertEqual(process.returncode, 0)
+        self.assertIn("Yōtei \\| &lt;tag&gt;", process.stdout)
+        self.assertIn("Missing \\| fixture", process.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()