diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e82a867..1af33c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,19 +13,21 @@ uv venv && uv pip install -e ".[dev]" # or: python -m venv .venv && pip insta pytest ``` -21 tests, about two seconds. If they pass, you are ready. +66 tests, about ten seconds. If they pass, you are ready. ## How the pieces fit -Five files, each with one job: +Seven files, each with one job: | File | Job | |---|---| | `recorder.py` | Wraps your package's functions, pickles the arguments they receive | | `plugin.py` | The pytest hook that turns recording on | | `replay.py` | Feeds recorded inputs back into one version of the code | +| `sideeffects.py` | Watches the files a call writes, so writes count as behaviour | | `fingerprint.py` | Reduces any Python value to something comparable across processes | | `compare.py` | Diffs two replay runs, quarantines anything nondeterministic | +| `cli.py` | The `record` and `check` commands | The flow is always: **record → replay twice on the baseline → replay the candidate → compare.** @@ -55,22 +57,25 @@ touch one file and need one test. The broad areas that need help: -- **Side-effect capture.** Right now only return values, exceptions and - argument mutation are compared. Network calls, file writes and database - queries are invisible. Each library (`requests`, `sqlalchemy`, `pathlib`) - is a separate, independent piece of work. -- **Comparators.** Some types need care: numpy arrays want tolerance, floats - want a policy, datetimes are often incidental. -- **Performance.** Recording currently costs 15–20x. `sys.monitoring` +- **Side-effect capture.** Return values, exceptions, argument mutation and + file writes are compared. Network calls and database queries are still + invisible. Each library (`requests`, `sqlalchemy`) is a separate, + independent piece of work. +- **Comparators.** Some types need care. Two are settled: numpy arrays + compare by shape, dtype and contents with no tolerance, and datetimes and + UUIDs are quarantined rather than normalised (issue #12). Reopening either + means naming the real change the new policy would hide. +- **Performance.** Recording currently costs ~8x. `sys.monitoring` (Python 3.12+) should be much cheaper than wrapping functions. -- **Portability.** The per-call timeout uses `SIGALRM`, so Windows is - unsupported. +- **Portability.** Windows is supported and tested in CI. Where `SIGALRM` + does not exist the per-call timeout falls back to a watchdog thread, which + cannot interrupt a call blocked inside C code. ## Pull requests - Add a test. If you are fixing a bug, name the test after the bug. -- Run `pytest` before pushing; CI runs Ubuntu and macOS across Python - 3.9, 3.11 and 3.13. +- Run `pytest` before pushing; CI runs Ubuntu, macOS and Windows across + Python 3.9, 3.11 and 3.13. - Small and focused beats large and complete. - If you change what the tool reports, say so plainly in the README. The README documents the limits honestly and should stay that way. diff --git a/README.md b/README.md index 9ccb407..fd07c06 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ You changed 200 lines. Your tests pass. Did anything *actually* change? Tests check what someone remembered to check. `nodrift` compares everything -observable: return values, exceptions raised, and whether arguments were -mutated, and the files it writes — using the real inputs your test suite -already produces. +observable: return values, exceptions raised, whether arguments were mutated, +and the files it writes — using the real inputs your test suite already +produces. No model reviews the code. The verdict comes from execution. @@ -27,8 +27,8 @@ $ nodrift check HEAD~1 mypkg.dates:parse 3 of 47 inputs differ - before: ["raise", ["exception", "ValueError", "invalid date"]] - after: ["return", ["object", "datetime.date", ...]] + before: [["raise", ["exception", "builtins.ValueError", "invalid date", ... + after: [["return", ["object", "datetime.date", ... ``` That last block is the point. Not *"this looks risky"* — **here is the input, @@ -77,14 +77,16 @@ nondeterministic. ### Recording part of a package One noisy or vendored module can dominate a recording. Both options take -`fnmatch` patterns against the full `module:Qualname` target and repeat: +`fnmatch` patterns against the full `module:Qualname` target and repeat. Note +the `:` — `"mypkg.core.*"` matches `mypkg.core.dates:parse` but *not* +`mypkg.core:parse`, so match the module name without a trailing dot: ```bash -nodrift record -p mypkg --exclude "mypkg.vendored.*" -nodrift record -p mypkg --include "mypkg.core.*" +nodrift record -p mypkg --exclude "mypkg.vendored*" +nodrift record -p mypkg --include "mypkg.core*" ``` -Whatever the patterns skip is printed, not hidden. +How many callables the patterns skip is printed, not hidden. ## How it works @@ -162,7 +164,8 @@ most of it touches one file and needs one test. Start with the [good first issues][gfi]. The broad areas that need help: - **Side-effect capture** — intercept `requests`, `sqlalchemy` -- **Comparators** for types that need a policy, e.g. datetimes and UUIDs +- **Comparators** for types that still need a policy — datetimes and UUIDs are + settled: quarantined, not normalised (issue #12) - **Performance** — recording currently costs ~8x - **Framework adapters** beyond pytest diff --git a/src/nodrift/__init__.py b/src/nodrift/__init__.py index 039c540..147e552 100644 --- a/src/nodrift/__init__.py +++ b/src/nodrift/__init__.py @@ -2,7 +2,7 @@ Records the real inputs your test suite already produces, then replays them against two versions of your code and compares everything observable: return -values, exceptions, and argument mutation. +values, exceptions, argument mutation, and the files a call writes. No model reviews the code. The verdict comes from execution. """ diff --git a/src/nodrift/cli.py b/src/nodrift/cli.py index 57f5c59..4dfae6a 100644 --- a/src/nodrift/cli.py +++ b/src/nodrift/cli.py @@ -2,6 +2,7 @@ nodrift record --package mypkg run the test suite, capture real inputs nodrift check HEAD~1 replay them against then and now + nodrift check HEAD~1 HEAD or against two refs, neither checked out """ from __future__ import annotations @@ -276,17 +277,20 @@ def main(argv: list[str] | None = None) -> int: help="max distinct inputs per function (default 600)") rec.add_argument("--include", action="append", default=None, metavar="PATTERN", - help="record only targets matching this fnmatch pattern, " - "e.g. 'mypkg.core.*' (repeatable)") + help="record only targets matching this fnmatch pattern " + "against 'module:Qualname', e.g. 'mypkg.core*' " + "(repeatable)") rec.add_argument("--exclude", action="append", default=None, metavar="PATTERN", - help="skip targets matching this fnmatch pattern, " - "e.g. 'mypkg.vendored.*' (repeatable)") + help="skip targets matching this fnmatch pattern " + "against 'module:Qualname', e.g. 'mypkg.vendored*' " + "(repeatable)") rec.add_argument("pytest_args", nargs="*", help="extra arguments passed through to pytest") rec.set_defaults(func=cmd_record) - chk = sub.add_parser("check", help="compare a git ref against the working tree") + chk = sub.add_parser( + "check", help="compare a git ref against the working tree, or two refs") chk.add_argument("ref", nargs="?", default="HEAD", help="git ref to treat as the baseline (default HEAD)") chk.add_argument("against", nargs="?", default=None, @@ -297,7 +301,8 @@ def main(argv: list[str] | None = None) -> int: help="path within the repo holding the package (e.g. src)") chk.add_argument("--json", action="store_true", help="emit the full report") chk.add_argument("--verbose", "-v", action="store_true", - help="name the functions that could not be recorded") + help="name the functions that were quarantined or could " + "not be recorded") chk.set_defaults(func=cmd_check) args = parser.parse_args(argv) diff --git a/src/nodrift/plugin.py b/src/nodrift/plugin.py index b3a394d..46bbee4 100644 --- a/src/nodrift/plugin.py +++ b/src/nodrift/plugin.py @@ -55,16 +55,17 @@ def pytest_addoption(parser): action="store", default=None, metavar="PATTERNS", - help="Comma-separated fnmatch patterns; record only matching targets " - "(e.g. 'mypkg.core.*').", + help="Comma-separated fnmatch patterns matched against " + "'module:Qualname'; record only matching targets " + "(e.g. 'mypkg.core*').", ) group.addoption( "--nodrift-exclude", action="store", default=None, metavar="PATTERNS", - help="Comma-separated fnmatch patterns to skip " - "(e.g. 'mypkg.vendored.*').", + help="Comma-separated fnmatch patterns matched against " + "'module:Qualname', to skip (e.g. 'mypkg.vendored*').", )