From 723d25d8c614cd70ed82d824d19f6c693d14553b Mon Sep 17 00:00:00 2001 From: Apostolis <58576290+Apopal1@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:39:05 +0300 Subject: [PATCH] ci: summarize CTest outcomes and retain results --- .github/workflows/build.yml | 37 ++++++++++++++-- CMakeLists.txt | 2 + tools/test_summary.py | 76 ++++++++++++++++++++++++++++++++ tools/tests/test_test_summary.py | 72 ++++++++++++++++++++++++++++++ 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 tools/test_summary.py create mode 100644 tools/tests/test_test_summary.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c067651f330..62ce9a4cf86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -141,7 +141,16 @@ jobs: run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH" - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON - run: cmake --build build-relinker --parallel 3 - - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 + - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 --output-junit "${{ github.workspace }}/ctest-results.xml" + - name: Summarize CTest results + if: ${{ !cancelled() }} + run: python tools/test_summary.py ctest-results.xml + - name: Retain CTest results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ctest-relinker-${{ matrix.os }} + path: ctest-results.xml linux: needs: eligibility @@ -166,10 +175,21 @@ jobs: run: python3 tools/check_test_registration.py build - run: cmake --build build - run: cmake --build build --target libs - - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log + - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit "${{ github.workspace }}/ctest-results.xml" env: ANYPS5_REQUIRE_VULKAN: 1 GALLIUM_OVERRIDE_CPU_CAPS: avx + - name: Summarize CTest results + if: ${{ !cancelled() }} + run: python3 tools/test_summary.py ctest-results.xml + - name: Retain CTest results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ctest-linux + path: | + ctest-results.xml + ctest.log - name: Report tests slower than 30 s if: ${{ !cancelled() }} shell: bash @@ -221,7 +241,18 @@ jobs: - shell: cmd run: cmake --build build --target libs - shell: cmd - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log + run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit "${{ github.workspace }}/ctest-results.xml" + - name: Summarize CTest results + if: ${{ !cancelled() }} + run: python tools/test_summary.py ctest-results.xml + - name: Retain CTest results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ctest-windows + path: | + ctest-results.xml + ctest.log - name: Report tests slower than 30 s if: ${{ !cancelled() }} shell: bash diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c6923e7ebc..e284d1799c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,8 @@ if(BUILD_TESTING) if(Python3_Interpreter_FOUND) add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py) + add_test(NAME test_summary COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_summary.py) add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py) add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE} diff --git a/tools/test_summary.py b/tools/test_summary.py new file mode 100644 index 00000000000..2b53fa3af08 --- /dev/null +++ b/tools/test_summary.py @@ -0,0 +1,76 @@ +import argparse +import html +import os +import sys +import xml.etree.ElementTree as ET +from collections import Counter +from pathlib import Path + + +def result(test): + status = test.get("status") + if status == "run": + return "Passed", "" + if status == "fail": + failure = test.find("failure") + return "Failed", failure.get("message", "") if failure is not None else "" + if status == "disabled": + return "Not run", "Disabled" + if status == "notrun": + skipped = test.find("skipped") + reason = skipped.get("message", "") if skipped is not None else "" + return ("Skipped" if reason.startswith("SKIP_") else "Not run"), reason + raise ValueError(f"Unknown CTest status: {status!r}") + + +def cell(value): + return html.escape(value).replace("|", "\\|").replace("\n", " ").replace("\r", " ") + + +def summarize(path): + root = ET.parse(path).getroot() + if root.tag != "testsuite": + raise ValueError("Expected a CTest JUnit testsuite") + tests = root.findall("testcase") + if not tests: + raise ValueError("CTest report contains zero tests; no execution evidence is available") + if int(root.get("tests", "-1")) != len(tests): + raise ValueError("CTest report test count does not match its results") + outcomes = [(test, *result(test)) for test in tests] + counts = Counter(status for _, status, _ in outcomes) + lines = ["### CTest results", "", "| Selected | Passed | Failed | Skipped | Not run |", + "| --- | --- | --- | --- | --- |", + f"| {len(tests)} | {counts['Passed']} | {counts['Failed']} | {counts['Skipped']} | {counts['Not run']} |", + "", "Counts describe the selected test cases in this CTest report, not individual repeat attempts.", + "Skipped tests ran and requested a skip. Not-run tests include disabled tests and tests blocked by failed fixtures.", + "The CTest artifact retains names, durations and captured output."] + incomplete = [(test, status, reason) for test, status, reason in outcomes if status != "Passed"] + if incomplete: + lines += ["", "| Test | Result | Reason |", "| --- | --- | --- |"] + lines += [f"| {cell(test.get('name', ''))} | {status} | {cell(reason)} |" + for test, status, reason in incomplete] + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description="Summarize CTest JUnit execution and skip counts") + parser.add_argument("report", type=Path) + args = parser.parse_args() + try: + summary = summarize(args.report) + code = 0 + except (OSError, ET.ParseError, ValueError) as error: + summary = f"### CTest results\n\nNo usable CTest report: {cell(str(error))}.\n" + code = 1 + print(summary) + destination = os.environ.get("GITHUB_STEP_SUMMARY") + if destination: + with open(destination, "a", encoding="utf-8") as stream: + stream.write(summary) + return code + + +if __name__ == "__main__": + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + raise SystemExit(main()) diff --git a/tools/tests/test_test_summary.py b/tools/tests/test_test_summary.py new file mode 100644 index 00000000000..ad4955872af --- /dev/null +++ b/tools/tests/test_test_summary.py @@ -0,0 +1,72 @@ +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import test_summary + + +class TestSummaryTests(unittest.TestCase): + def run_report(self, report): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "results.xml" + if report is not None: + path.write_text(report, encoding="utf-8") + summary = Path(directory) / "summary.md" + summary.write_text("Existing summary\n", encoding="utf-8") + process = subprocess.run( + [sys.executable, test_summary.__file__, str(path)], + capture_output=True, encoding="utf-8", timeout=10, + env={**os.environ, "GITHUB_STEP_SUMMARY": str(summary)}, + ) + self.assertEqual(summary.read_text(encoding="utf-8"), "Existing summary\n" + process.stdout.rstrip("\n") + "\n") + return process + + def test_pass_skip_failure_and_not_run_are_distinct(self): + process = self.run_report(''' + + + + + + + ''') + self.assertEqual(process.returncode, 0, process.stderr) + self.assertIn("| 6 | 1 | 1 | 2 | 2 |", process.stdout) + self.assertIn("| disabled | Not run | Disabled |", process.stdout) + self.assertIn("| blocked | Not run | Failed test dependencies: setup |", process.stdout) + self.assertIn("| failed | Failed | Timeout |", process.stdout) + + def test_all_passed(self): + process = self.run_report('') + self.assertEqual(process.returncode, 0) + self.assertIn("| 1 | 1 | 0 | 0 | 0 |", process.stdout) + self.assertNotIn("| Test |", process.stdout) + + def test_missing_empty_and_malformed_reports_fail(self): + for report in (None, "broken", '', + '', + '', + ''): + with self.subTest(report=report): + process = self.run_report(report) + self.assertEqual(process.returncode, 1) + self.assertIn("No usable CTest report:", process.stdout) + self.assertNotIn("| Selected |", process.stdout) + + def test_unicode_and_markdown_are_preserved_safely(self): + process = self.run_report(''' + + + + ''') + self.assertEqual(process.returncode, 0) + self.assertIn("Yōtei \\| <tag>", process.stdout) + self.assertIn("Missing \\| fixture", process.stdout) + + +if __name__ == "__main__": + unittest.main()