From dde862a8ed55f3816880c69afcfefbfde8495cc1 Mon Sep 17 00:00:00 2001 From: LuShadowX Date: Sat, 8 Aug 2026 10:17:46 +0530 Subject: [PATCH] Reset the random streams before every replayed record random.seed(0) ran once per replay process, so the records shared one stream. A candidate that changes how many values one function draws shifts every record after it, and a function nobody touched is reported as different. Reproduced with two functions, one drawing randoms and one merely formatting one: adding a single extra draw to the first made the second report 2 of 2 inputs differ. It was never edited. That is a false positive, which is the one error this tool cannot afford, and it was reachable by any refactor touching random use. Reseeding per record costs nothing and hides nothing: a function that stops using randomness still returns something other than the seeded value, so real changes are still caught. The regression test is the reproduction. --- src/nodrift/replay.py | 25 ++++++++++++++++++ tests/test_end_to_end.py | 55 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/src/nodrift/replay.py b/src/nodrift/replay.py index aa0913d..5726889 100644 --- a/src/nodrift/replay.py +++ b/src/nodrift/replay.py @@ -152,6 +152,8 @@ def run(recording_path: str, deterministic: bool) -> dict: resolved: dict[str, object] = {} for index, record in enumerate(records): + if deterministic: + _reseed() target = record["target"] key = f"{target}#{index}" @@ -188,6 +190,29 @@ def run(recording_path: str, deterministic: bool) -> dict: return results +def _reseed() -> None: + """Reset the random streams before every record. + + Seeding once per process couples the records together: if the candidate + changes how many values one function draws, every record after it reads + from a shifted stream and a function nobody touched reports a difference. + That is a false positive, which is the one error this tool cannot afford. + + Reseeding per record does not hide anything in exchange — a function that + stopped using randomness still returns something other than the seeded + value, so the change is still caught. + """ + import random + + random.seed(0) + numpy = sys.modules.get("numpy") + if numpy is not None: + try: + numpy.random.seed(0) + except Exception: + pass + + def _install_determinism_controls() -> None: """Neutralise the common sources of run-to-run variation. diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 3237a30..1eddaf4 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -632,3 +632,58 @@ def test_quarantined_functions_are_named_not_just_counted(recorded, tmp_path): quiet = _nodrift(repo, "check", "HEAD") assert "quarantined" in quiet.stdout, quiet.stdout assert "toy:Box" in quiet.stdout, quiet.stdout + + +RANDOM_PKG = ''' +import random + + +def upstream(n): + return [random.random() for _ in range(n)] + + +def downstream(tag): + return f"{tag}-{random.random():.6f}" +''' + +RANDOM_TESTS = ''' +import rnd + + +def test_all(): + assert rnd.upstream(2) + assert rnd.downstream("a") + assert rnd.downstream("b") +''' + + +def test_random_draws_do_not_leak_between_records(tmp_path): + """One function's randomness must not implicate the next function. + + Seeding once per process couples every record to the ones before it: a + candidate that draws one extra value shifts the stream, and a function + nobody touched reports a difference. A false positive is the one error + this tool cannot afford. + """ + repo = tmp_path / "rndrepo" + repo.mkdir() + _write(str(repo), "rnd.py", RANDOM_PKG) + _write(str(repo), "test_rnd.py", RANDOM_TESTS) + for argv in (["init", "-q", "."], ["add", "-A"], + ["-c", "user.email=t@t", "-c", "user.name=t", + "commit", "-qm", "initial"]): + subprocess.run(["git", *argv], cwd=str(repo), check=True, + capture_output=True) + + _nodrift(repo, "record", "--package", "rnd") + + source = (repo / "rnd.py").read_text() + (repo / "rnd.py").write_text(source.replace("range(n)]", "range(n + 1)]")) + + result = _nodrift(repo, "check", "HEAD") + assert result.returncode == 1, "the real change in upstream went unnoticed" + assert "rnd:upstream" in result.stdout, result.stdout + assert "rnd:downstream" not in result.stdout, ( + "downstream was never touched but was reported as changed\n" + + result.stdout + )