diff --git a/.github/workflows/scripts_new/macosx/4_test.sh b/.github/workflows/scripts_new/macosx/4_test.sh index 71037e2471..20fe537fef 100644 --- a/.github/workflows/scripts_new/macosx/4_test.sh +++ b/.github/workflows/scripts_new/macosx/4_test.sh @@ -2,6 +2,9 @@ set -ex +export QD_FILE_TIMING=1 +export QD_FILE_TIMING_OUTPUT="${RUNNER_TEMP}/file_timing.md" + pip install --prefer-binary --group test pip install -r requirements_test_xdist.txt find . -name '*.bc' @@ -17,3 +20,7 @@ python tests/run_tests.py -v -r 1 --arch metal,vulkan,cpu -m "not needs_torch" # TODO: revert to stable torch after 2.9.2 release pip install --pre --upgrade torch --index-url https://download.pytorch.org/whl/nightly/cpu python tests/run_tests.py -v -r 1 --arch metal,vulkan,cpu -m needs_torch + +if [ -f "$QD_FILE_TIMING_OUTPUT" ]; then + cat "$QD_FILE_TIMING_OUTPUT" >> "$GITHUB_STEP_SUMMARY" +fi diff --git a/docs/source/user_guide/contributing.md b/docs/source/user_guide/contributing.md index 50b971c43c..42ebb295ab 100644 --- a/docs/source/user_guide/contributing.md +++ b/docs/source/user_guide/contributing.md @@ -2,7 +2,7 @@ ## Good practice reminder -* *testing*: Any new features or modified code should be tested. You have to run the test suite using `python tests/run_tests.py` which sets up the right test environment for `pytest`. CLI arguments are forwarded to `pytest`. Do not use `pytest` directly as it behaves differently. +* *testing*: Any new features or modified code should be tested. You have to run the test suite using `python tests/run_tests.py` which sets up the right test environment for `pytest`. CLI arguments are forwarded to `pytest`. Do not use `pytest` directly as it behaves differently. To see a per-file timing breakdown (useful for identifying slow test files), set `QD_FILE_TIMING=1` — e.g. `QD_FILE_TIMING=1 python tests/run_tests.py`. This is enabled by default in the Mac CI job and the results appear in the GitHub Actions job summary. * *format/linter*: Before pushing any commits, ensure you set up `pre-commit` and run it using `pre-commit run -a` * No need to force push to keep a clean history as the merging is eventually done by squashing commits. diff --git a/tests/python/pytest_file_timing.py b/tests/python/pytest_file_timing.py new file mode 100644 index 0000000000..d5a8e28492 --- /dev/null +++ b/tests/python/pytest_file_timing.py @@ -0,0 +1,75 @@ +"""Pytest plugin that reports wall-clock time spent per test file. + +Activated by the environment variable QD_FILE_TIMING=1. Collects the "call" phase duration of each test item and +prints a sorted summary at the end of the session. + +Works correctly with pytest-xdist: the controller process receives forwarded reports from all workers and aggregates +them here. + +Set QD_FILE_TIMING_OUTPUT to a file path to also write the results as markdown (useful for GitHub Actions job +summaries). +""" + +import os +from collections import defaultdict + +_active = os.environ.get("QD_FILE_TIMING", "0") == "1" + +_file_durations: dict[str, float] = defaultdict(float) +_file_test_counts: dict[str, int] = defaultdict(int) + + +def pytest_runtest_logreport(report): + if not _active: + return + if report.when == "call": + fspath = report.fspath + _file_durations[fspath] += report.duration + _file_test_counts[fspath] += 1 + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + if not _active: + return + if not _file_durations: + return + + tw = terminalreporter._tw + tw.sep("=", "per-file timing summary") + tw.line(f"{'Duration (s)':>12} {'Tests':>6} File") + tw.sep("-") + + sorted_files = sorted(_file_durations.items(), key=lambda x: -x[1]) + + total = 0.0 + for fspath, duration in sorted_files: + count = _file_test_counts[fspath] + tw.line(f"{duration:12.2f} {count:6d} {fspath}") + total += duration + + tw.sep("-") + total_tests = sum(_file_test_counts.values()) + tw.line(f"{total:12.2f} {total_tests:6d} TOTAL (sum of per-file call durations)") + tw.sep("=") + + output_path = os.environ.get("QD_FILE_TIMING_OUTPUT") + if output_path: + _write_markdown(sorted_files, total, total_tests, output_path) + + +def _write_markdown(sorted_files, total, total_tests, path): + lines = [ + "### Per-file test timing", + "", + "| Duration (s) | Tests | File |", + "|---:|---:|:---|", + ] + for fspath, duration in sorted_files: + count = _file_test_counts[fspath] + basename = os.path.basename(fspath) + lines.append(f"| {duration:.2f} | {count} | `{basename}` |") + lines.append(f"| **{total:.2f}** | **{total_tests}** | **TOTAL** |") + lines.append("") + + with open(path, "a") as f: + f.write("\n".join(lines)) diff --git a/tests/run_tests.py b/tests/run_tests.py index 7f9335b99e..a752fe75fc 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -101,6 +101,13 @@ def _test_python(args, default_dir="python"): else: if int(threads) > 1: pytest_args += ["-n", str(threads), "--dist=worksteal"] + if os.environ.get("QD_FILE_TIMING", "0") == "1": + import sys as _sys + + if test_dir not in _sys.path: + _sys.path.insert(0, test_dir) + pytest_args += ["-p", "pytest_file_timing"] + import pytest # pylint: disable=C0415 return int(pytest.main(pytest_args))