diff --git a/c/tests/check_data_logprob_gaps.py b/c/tests/check_data_logprob_gaps.py index 0abb41655..2496c67e4 100644 --- a/c/tests/check_data_logprob_gaps.py +++ b/c/tests/check_data_logprob_gaps.py @@ -4,18 +4,24 @@ Asserts, over a captured raw engine-stdout transcript, that EVERY generated DATA frame of an opted-in request carries the per-token numeric channel ("DATA [tid tlp]*k") -- accepted draft tokens included. -Speculatively-accepted draft tokens bypass the pick_tok call sites in the mux -loop (packet Fork 5's implementation risk), so a gap would show up as a -legacy 3-field DATA frame in the middle of an opted-in generation. +On today's engine, an accepted speculative-draft token is emitted through +the SAME mux_data() call as any other generated token (mux_spec_emit -> +mux_data, passing the same logits row and requested top-k the ordinary +emit sites use), so the gap this check hunts -- a legacy 3-field DATA +frame appearing mid-generation because a draft-accept path bypassed the +numeric-channel emit -- is NOT producible by the engine as shipped. This +check is retained as REGRESSION COVERAGE for that invariant (a future mux +change that adds a new emit call site without the tail would reintroduce +exactly this defect class), not as a currently-live defect hunt. -Run recipe (orchestrator; needs a real model -- CPU or CUDA build): +Run recipe (orchestrator; needs a real model -- CPU or explicit-CUDA build): # single KV slot + model drafts live = the speculative serve regime cd c && make glm SERVE=1 SERVE_BATCH=1 KV_SLOTS=1 DRAFT=2 CTX=4096 \ - ./colibri < submit.txt | tee engine_stdout.raw + ./colibri < submit.raw | tee engine_stdout.raw - where submit.txt contains one opted-in generation request, e.g. (prompt + where submit.raw contains one opted-in generation request, e.g. (prompt "Hello" = 5 payload bytes, 64 new tokens, greedy, logprobs top-5): SUBMIT 7 0 5 64 0 1 0 logprobs=5 @@ -27,109 +33,635 @@ stderr log ("[MTP] ..." / spec acceptance lines) so the run genuinely exercised the draft-accept emit path rather than trivially passing. -Then: +A raw capture taken this way is a plain pipeline: it is not itself hash-bound +to the binary, container, input, and environment that produced it, and +NOTHING in this module binds it either -- the preamble gate below checks +only that the BANNER/LOADED text is a well-formed, self-consistent record of +what the engine printed about itself; it does not verify that text against +an independently computed binary or container digest. Whichever capture step +feeds this checker, prefer one that additionally binds and retains that +provenance (distinct raw stdout/stderr, the direct engine exit status, and a +hash over the binary/container/input/environment/protocol payloads) over a +bare pipeline or a lone DATA/HIT record, which is not decisive evidence on +its own. - python3 tests/check_data_logprob_gaps.py engine_stdout.raw --id 7 --topk 5 +Then, against the resulting transcript, the numeric-channel check runs: + + python3 tests/check_data_logprob_gaps.py engine_stdout.raw \ + --id 7 --topk 5 --vocab 154880 Checks performed: - - every DATA frame for --id has exactly 5 + 2k header fields, a parseable - float lp, and k == --topk (k may be < topk only if vocab < topk). - A non-finite lp ("nan"/"inf": degenerate logits, e.g. an all -inf row - after grammar masking) counts as PRESENT -- the channel carried a value, - which is exactly what this audit is for -- and is FLAGGED in the output - without failing the check (whether the engine should serialize such rows - differently is a U7b server-side register question, not a gap); - - ECHO frames (if the request also echoed) cover contiguous positions - 0..P-1, position 0 carrying "nan 0"; + - the transcript's capture shape (full-process / ready-suffix / request- + only) is named by capture_mode() and enforced explicitly: the leading + line must be a recognized global preamble record or a legitimate + request-frame kind, and a transcript with any global record present + but not correctly led (a "invalid" capture per capture_mode()) fails + loud -- no transcript reaches the rest of the checks unclassified; + - the startup BANNER/LOADED preamble, where present, parses exactly + (the engine_evidence grammar); an unparsed preamble is a named + failure quoting the offending line, never a silent pass; + - exactly one ACCEPT and one DONE exist for --id, DONE's emitted count + equals the positive DATA count, and no targeted ERROR or post-DONE + frame exists; + - every DATA/ECHO numeric tail has exactly the advertised fields and + min(--topk,--vocab) unique token ids in range. Each numeric token + (target logprob and every top-k logprob) must be a finite, + non-positive value spelled as the fixed six-decimal form dev's + engine prints today ("fixed6", e.g. "-0.300000"), the %.17g form an + earlier engine build printed ("c17g", e.g. "-2.7000000000000002"), or + an exact nan/inf/-inf spelling -- whichever form parses. A token that + happens to satisfy BOTH spellings exactly (e.g. '%.17g' % -1.234567 + == '-1.234567', itself also an exact six-decimal spelling) is + "ambiguous": there is no way to tell, from the token alone, which + engine format produced it, so no per-frame or per-transcript + "consistent format" rule is enforced -- one was tried and had to be + dropped (see the module history) because it produced false rejections + on real %.17g transcripts purely from this ambiguity. The observed + form counts (fixed6-only / c17g-only / ambiguous / special) are + reported in the summary line as INFORMATION ONLY; they never affect + the verdict; + - ACCEPT's canonical prompt length P has exactly P ECHO frames at + positions 0..P-1, position 0 carrying "nan 0"; - payload framing (n bytes + newline) stays byte-exact throughout, so a - single malformed frame cannot hide by desynchronizing the parse. + single malformed frame cannot hide by desynchronizing the parse; every + reported problem cites the byte offset of the offending frame's own + header line (not the frame that follows it). Exit 0 = no gaps; non-zero = at least one gap/malformed frame (listed). """ import argparse +import collections import math +import os +import re import sys +_TOOLS_DIR = os.path.join(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))), "tools") +if _TOOLS_DIR not in sys.path: + sys.path.insert(0, _TOOLS_DIR) +from engine_evidence import PreambleError, parse_engine_preamble + + +_INT32_MAX = 2**31 - 1 +_UINT64_MAX = 2**64 - 1 +_UINT_RE = re.compile(rb"(?:0|[1-9][0-9]*)") +_C17G_RE = re.compile( + rb"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?" + rb"(?:e[+-](?:0[0-9]|[1-9][0-9]{1,2}))?") +_FIXED6_RE = re.compile(rb"-?(?:0|[1-9][0-9]*)\.[0-9]{6}") +_SPECIAL_TOKENS = {b"nan": math.nan, b"inf": math.inf, b"-inf": -math.inf} +_READY = b"\x01\x01READY\x01\x01" +_GLOBAL_KINDS = frozenset(( + b"BANNER", b"LOADED", b"READY", b"STAT", b"HWINFO", b"TIERS", + b"EMAP", b"HITS", b"PROF", +)) +_TARGETED_KINDS = frozenset(( + b"ACCEPT", b"DATA", b"ECHO", b"DONE", b"ERROR", +)) +_LOWER_HEX_RE = re.compile(rb"[0-9a-f]*") + + +def _uint(token, label, maximum=_INT32_MAX): + if not _UINT_RE.fullmatch(token): + raise ValueError(f"noncanonical {label}: {token!r}") + value = int(token.decode("ascii")) + if value > maximum: + raise ValueError(f"{label} exceeds {maximum}: {token!r}") + return value + + +def _c17g(token, label): + if not _C17G_RE.fullmatch(token): + raise ValueError(f"noncanonical {label} %.17g token: {token!r}") + value = float(token.decode("ascii")) + if not math.isfinite(value) or format(value, ".17g").encode("ascii") != token: + raise ValueError(f"not an exact finite {label} %.17g spelling: {token!r}") + return value + + +def _fixed6(token, label): + if not _FIXED6_RE.fullmatch(token): + raise ValueError(f"noncanonical {label} %.6f token: {token!r}") + value = float(token.decode("ascii")) + if not math.isfinite(value) or format(value, ".6f").encode("ascii") != token: + raise ValueError(f"not an exact finite {label} %.6f spelling: {token!r}") + return value + + +def _numeric_value(token, label): + """Classify and parse one engine numeric-tail token. + + Returns (value, form). "special" is an exact nan/inf/-inf spelling -- + libc's own %f/%g rendering of a non-finite double, which either wire + format's snprintf call can emit identically. Otherwise the token is + checked against BOTH finite grammars independently (never short- + circuited): the fixed six-decimal form dev's engine prints today + ("fixed6") and the %.17g form an earlier engine build printed + ("c17g"). A token can satisfy both -- '%.17g' % -1.234567 == + '-1.234567', which is ALSO the exact six-decimal spelling of that + same double -- and when it does, the form is "ambiguous": there is no + way to tell, from the token alone, which engine format produced it. + A token matching neither raises ValueError. + + An earlier version of this function tried fixed6 first and reported + it whenever fixed6 matched, silently hiding the ambiguous case; that + misclassified a large fraction of genuine %.17g tokens as fixed6 and + fed a since-removed "consistent form per frame" check false mixed- + form rejections on real transcripts. Classification is now purely + informational (see check()/main()) precisely because it cannot be + made unambiguous from the token alone. + """ + if token in _SPECIAL_TOKENS: + return _SPECIAL_TOKENS[token], "special" + value = None + is_fixed6 = is_c17g = False + try: + value = _fixed6(token, label) + is_fixed6 = True + except ValueError: + pass + try: + value = _c17g(token, label) + is_c17g = True + except ValueError: + pass + if is_fixed6 and is_c17g: + return value, "ambiguous" + if is_fixed6: + return value, "fixed6" + if is_c17g: + return value, "c17g" + raise ValueError( + f"{label} matches neither the fixed6 nor the c17g nor the " + f"nan/inf spelling: {token!r}") + + +def _fixed_metric(token, places, label, lower=0.0, upper=None): + pattern=rb"-?(?:0|[1-9][0-9]*)\."+rb"[0-9]{"+str(places).encode()+rb"}" + if not re.fullmatch(pattern,token): + raise ValueError(f"noncanonical {label}: {token!r}") + value=float(token.decode("ascii")) + if not math.isfinite(value) or valueupper): + raise ValueError(f"{label} outside [{lower},{upper}]: {token!r}") + return value + + +def _header_fields(line, byte_offset): + problems=[] + if not line: + return [],[f"blank protocol header at byte {byte_offset}"] + if (any(byte<0x20 or byte>0x7e for byte in line) or + line.startswith(b" ") or line.endswith(b" ") or b" " in line): + problems.append(f"noncanonical ASCII/space header at byte {byte_offset}: {line!r}") + return line.split(),problems + + +def _global_header(line, byte_offset): + """Validate one exact production mux-global line. + + Return None when the line is not a recognized global. Recognized globals + return a synthetic one-field marker so check() owns their process-level + lifecycle before any numeric token can collide with a request id. + """ + if (line.startswith(b"== GLM C engine") or + line.startswith(b"loaded in")): + kind = b"BANNER" if line.startswith(b"==") else b"LOADED" + problems = [] + try: + text = line.decode("ascii") + parsed = parse_engine_preamble(text) + if parsed is None or parsed["kind"].encode("ascii") != kind: + raise PreambleError("owned preamble kind mismatch") + except (UnicodeDecodeError, PreambleError) as exc: + problems.append( + f"malformed {kind.decode()} preamble at byte {byte_offset}: " + f"{exc}; {line!r}") + return [kind], problems + + if line == _READY: + return [b"READY"], [] + + kind = line.split(b" ", 1)[0] + if kind == b"READY": + return [b"READY"], [ + f"malformed READY global at byte {byte_offset}: expected {_READY!r}; {line!r}" + ] + if kind not in _GLOBAL_KINDS - {b"READY"}: + return None + problems = [] + try: + if kind == b"STAT": + fields = line.split(b" ") + if (len(fields) != 5 or fields[1:4] != [b"0", b"0.00", b"0.0"]): + raise ValueError("expected exact startup STAT fields") + _fixed_metric(fields[4], 2, "STAT RSS") + elif kind == b"HWINFO": + # The final CPU|GPU field is produced by two %s conversions. It is + # printable free text and may contain repeated spaces; the six + # numeric/structural prefixes remain exact single-space fields. + fields = line.split(b" ", 6) + if len(fields) != 7: + raise ValueError("expected six HWINFO prefixes and CPU|GPU tail") + _uint(fields[1], "HWINFO core count") + _fixed_metric(fields[2], 1, "HWINFO total RAM") + _fixed_metric(fields[3], 1, "HWINFO available RAM") + _uint(fields[4], "HWINFO GPU count") + _fixed_metric(fields[5], 1, "HWINFO total VRAM") + tail = fields[6] + if (tail.count(b"|") != 1 or + any(byte < 0x20 or byte > 0x7e for byte in tail)): + raise ValueError("HWINFO tail is not printable CPU|GPU text") + elif kind == b"TIERS": + fields = line.split(b" ") + if len(fields) != 6: + raise ValueError("expected five TIERS fields") + for index, label in enumerate(("VRAM", "RAM", "disk"), 1): + _uint(fields[index], f"TIERS {label} count") + _fixed_metric(fields[4], 2, "TIERS VRAM GB") + _fixed_metric(fields[5], 2, "TIERS RAM GB") + elif kind in (b"EMAP", b"HITS"): + fields = line.split(b" ") + if len(fields) != 4: + raise ValueError(f"expected three {kind.decode()} fields") + rows = _uint(fields[1], f"{kind.decode()} row count") + cols = _uint(fields[2], f"{kind.decode()} column count") + if cols < 1 or (kind == b"HITS" and rows < 1): + raise ValueError( + f"{kind.decode()} rows/columns outside producer domain") + payload = fields[3] + if not _LOWER_HEX_RE.fullmatch(payload): + raise ValueError(f"{kind.decode()} payload is not lowercase hex") + cells = rows * cols + expected = cells * 2 if kind == b"EMAP" else ((cells + 7) // 8) * 2 + if len(payload) != expected: + raise ValueError( + f"{kind.decode()} payload length {len(payload)} != {expected}") + if kind == b"EMAP": + for index in range(cells): + cell = int(payload[2 * index:2 * index + 2], 16) + tier, heat = cell >> 6, cell & 0x3f + if tier > 2: + raise ValueError( + f"EMAP cell {index} tier {tier} outside [0,2]") + if heat > 32: + raise ValueError( + f"EMAP cell {index} heat {heat} outside [0,32]") + elif cells & 7: + final = int(payload[-2:], 16) + used_mask = (1 << (cells & 7)) - 1 + if final & ~used_mask: + raise ValueError("HITS final byte has nonzero padding bits") + else: # PROF + fields = line.split(b" ") + if len(fields) != 10: + raise ValueError("expected nine PROF fields") + _fixed_metric(fields[1], 3, "PROF wall seconds") + _uint(fields[2], "PROF prompt count") + _uint(fields[3], "PROF completion count") + for index, label in enumerate( + ("disk", "wait", "matmul", "attention", "head"), 4): + _fixed_metric(fields[index], 3, f"PROF {label} seconds") + _uint(fields[9], "PROF forward count", _UINT64_MAX) + except (ValueError, IndexError) as exc: + problems.append( + f"malformed {kind.decode()} global at byte {byte_offset}: {exc}; {line!r}") + return [kind], problems + + def parse_frames(blob): - """Parse the serve-mux stdout byte stream into (header_fields, payload).""" + """Parse frames fail-closed, returning (frames, framing_problems). + + Each frame is (fields, payload, byte_offset), where byte_offset is the + offset of THIS frame's own header line -- captured before the cursor + advances past it, so every problem this module reports can cite the + byte offset of the frame it is actually complaining about, not the + frame that happens to follow it in the transcript. + """ frames = [] + problems = [] i = 0 n = len(blob) while i < n: + line_start = i j = blob.find(b"\n", i) if j < 0: + problems.append(f"truncated header at byte {line_start}") break line = blob[i:j] i = j + 1 - fields = line.split() + global_header = _global_header(line, line_start) + if global_header is None: + fields, header_problems = _header_fields(line, line_start) + else: + fields, header_problems = global_header + problems.extend(header_problems) if not fields: continue kind = fields[0] - if kind in (b"DATA", b"ECHO") and len(fields) >= 3: + if kind in (b"DATA", b"ECHO"): + if len(fields) < 3: + problems.append( + f"short {kind.decode(errors='replace')} header at " + f"byte {line_start}: {fields!r}") + break try: - size = int(fields[2]) + size = _uint(fields[2],"payload size") except ValueError: - frames.append((fields, None)) - continue + problems.append( + f"invalid payload size at byte {line_start}: {fields!r}") + break + if size < 0: + problems.append( + f"negative payload size at byte {line_start}: {fields!r}") + break + if size > n - i: + problems.append( + f"truncated payload at byte {line_start}: need {size} bytes") + break payload = blob[i:i + size] i += size if i < n and blob[i:i + 1] == b"\n": i += 1 else: - frames.append((fields, b"")) - continue - frames.append((fields, payload)) + problems.append( + f"missing payload terminator at byte {line_start} " + f"after {fields!r}") + break + frames.append((fields, payload, line_start)) else: - frames.append((fields, None)) - return frames + frames.append((fields, None, line_start)) + return frames, problems + +def capture_mode(frames): + """Name the explicit transcript shape without grading its lifecycle.""" + if not frames: + return "request-only" + first = frames[0][0][0] + if first == b"BANNER": + return "full-process" + if first == b"READY": + return "ready-suffix" + if any(fields[0] in _GLOBAL_KINDS for fields, _, _ in frames): + return "invalid" + return "request-only" -def check(frames, request_id, topk): - rid = str(request_id).encode() + +def _numeric_tail(fields, field_offset, expected_k, vocab, label, byte_offset): + """Validate one DATA/ECHO numeric tail starting at fields[field_offset]. + + Returns (problems, forms) where forms is the LIST of numeric-token + "form" tags (see _numeric_value) observed in this tail, in order -- + a list, not a set, so the caller can tally counts. No rule requires a + tail's forms to agree with one another: per-token classification is + inherently ambiguous (a token can be an exact spelling under both + finite grammars at once), so there is no reliable way to tell a + genuinely mixed-format tail from an entirely single-format one that + merely contains some ambiguous tokens -- see _numeric_value. + """ problems = [] - flags = [] + forms = [] + try: + lp, lp_form = _numeric_value(fields[field_offset], f"{label} target logprob") + k = _uint(fields[field_offset + 1], f"{label} top-k count", 32) + except (ValueError, IndexError): + return ([f"malformed {label} numeric fields at byte {byte_offset}: " + f"{fields!r}"], forms) + forms.append(lp_form) + if not math.isfinite(lp) or lp > 0.0: + problems.append( + f"{label} target logprob is not finite/non-positive at byte " + f"{byte_offset}: {fields!r}") + if k != expected_k: + problems.append( + f"{label} top-k {k} != expected {expected_k} at byte " + f"{byte_offset}: {fields!r}") + want_fields = field_offset + 2 + 2 * k + if len(fields) != want_fields: + return (problems + [f"{label} field count {len(fields)} != " + f"{want_fields} at byte {byte_offset}: {fields!r}"], + forms) + ids = [] + for idx in range(k): + try: + token_id = _uint(fields[field_offset + 2 + 2 * idx], + f"{label} token id", _INT32_MAX) + token_lp, token_form = _numeric_value( + fields[field_offset + 3 + 2 * idx], f"{label} token logprob") + except ValueError: + problems.append( + f"malformed {label} top-k pair {idx} at byte {byte_offset}: " + f"{fields!r}") + continue + forms.append(token_form) + if not 0 <= token_id < vocab: + problems.append( + f"{label} token id {token_id} outside [0,{vocab}) at byte " + f"{byte_offset}: {fields!r}") + if token_id in ids: + problems.append( + f"{label} duplicate token id {token_id} at byte " + f"{byte_offset}: {fields!r}") + ids.append(token_id) + if not math.isfinite(token_lp) or token_lp > 0.0: + problems.append( + f"{label} token {token_id} logprob is not finite/non-positive " + f"at byte {byte_offset}: {fields!r}") + return problems, forms + + +def check(frames, framing_problems, request_id, topk, vocab): + problems = list(framing_problems) + mode = capture_mode(frames) + if frames: + first_kind = frames[0][0][0] + if first_kind not in _GLOBAL_KINDS and first_kind not in _TARGETED_KINDS: + problems.append( + f"unrecognized frame opens the transcript at byte " + f"{frames[0][2]} (not a global preamble or a request " + f"frame): {frames[0][0]!r}") + if mode == "invalid": + problems.append( + "invalid capture mode: global records present without a " + "recognized BANNER/READY lead frame") + try: + rid = str(request_id).encode("ascii",errors="strict") + rid_value=_uint(rid,"requested id",_UINT64_MAX) + if rid_value < 1: + raise ValueError("requested id must be positive") + except (UnicodeEncodeError,ValueError) as exc: + return 0, 0, problems+[str(exc)], mode, frozenset() data_frames = 0 echo_positions = [] - for fields, payload in frames: - if len(fields) < 2 or fields[1] != rid: - continue + accepts = [] + dones = [] + accept_prompts = [] + done_values = [] + done_seen = False + data_seen = False + global_indices = {kind: [] for kind in _GLOBAL_KINDS} + expected_k = min(topk, vocab) + forms_used = [] + for frame_index, (fields, payload, offset) in enumerate(frames): kind = fields[0] + if kind in _GLOBAL_KINDS: + global_indices[kind].append(frame_index) + continue + if len(fields)<2: + continue + try: + frame_id=_uint(fields[1],"frame request id",_UINT64_MAX) + except ValueError as exc: + if kind in _TARGETED_KINDS: + problems.append(f"{exc} at byte {offset}") + continue + if frame_id!=rid_value: + continue + if done_seen: + problems.append( + f"frame for id {request_id} after DONE at byte {offset}: " + f"{fields!r}") + if kind == b"ACCEPT": + accepts.append(frame_index) + if len(fields) != 3: + problems.append( + f"malformed ACCEPT frame at byte {offset}: {fields!r}") + else: + try: + prompt=_uint(fields[2],"ACCEPT prompt length") + if prompt<1: + raise ValueError + except ValueError: + problems.append( + f"invalid ACCEPT prompt length at byte {offset}: " + f"{fields!r}") + else: + accept_prompts.append(prompt) + continue + if kind == b"ERROR": + problems.append( + f"target request returned ERROR at byte {offset}: {fields!r}") + continue + if kind == b"DONE": + dones.append(frame_index) + done_seen = True + if len(fields) != 9 or fields[2] != b"STAT": + problems.append( + f"malformed DONE frame at byte {offset}: {fields!r}") + done_values.append(None) + continue + try: + emitted=_uint(fields[3],"DONE emitted count") + tps=_fixed_metric(fields[4],2,"DONE tokens/second") + hit=_fixed_metric(fields[5],1,"DONE hit percentage",upper=100.0) + rss=_fixed_metric(fields[6],2,"DONE RSS") + prompt=_uint(fields[7],"DONE prompt-token count") + flag=_uint(fields[8],"DONE length_limited",1) + if prompt<1: + raise ValueError("DONE prompt-token count is not positive") + except ValueError: + problems.append( + f"malformed DONE stats at byte {offset}: {fields!r}") + done_values.append(None) + continue + done_values.append((emitted,tps,hit,rss,prompt,flag)) + continue if kind == b"DATA": + data_seen = True data_frames += 1 + if not accepts: + problems.append( + f"DATA before ACCEPT at byte {offset}: {fields!r}") + if payload is None: + problems.append( + f"DATA missing validated payload at byte {offset}: " + f"{fields!r}") if len(fields) == 3: - problems.append(f"GAP: legacy 3-field DATA frame #{data_frames}" - f" (payload {payload!r}) has no logprob") - continue - try: - lp = float(fields[3]) - k = int(fields[4]) - except (ValueError, IndexError): - problems.append(f"malformed DATA numeric fields: {fields!r}") + problems.append( + f"GAP: legacy 3-field DATA frame #{data_frames} at byte " + f"{offset} (payload {payload!r}) has no logprob") continue - if not math.isfinite(lp): - # PRESENT, not a gap: the channel carried a value; degenerate - # logits (an all -inf row, say) legitimately produce nan/inf. - # Flagged so a reviewer sees it; never fails the audit. - flags.append(f"non-finite lp in DATA frame #{data_frames}" - f" (present, flagged): {fields!r}") - if k != topk: - problems.append(f"DATA top-k {k} != requested {topk}: {fields!r}") - if len(fields) != 5 + 2 * k: - problems.append(f"DATA field count {len(fields)} != 5+2k: {fields!r}") + tail_problems, tail_forms = _numeric_tail( + fields, 3, expected_k, vocab, "DATA", offset) + problems.extend(tail_problems) + forms_used.extend(tail_forms) elif kind == b"ECHO": + if data_seen: + problems.append( + f"ECHO after DATA at byte {offset}: {fields!r}") + if not accepts: + problems.append( + f"ECHO before ACCEPT at byte {offset}: {fields!r}") + if payload is None: + problems.append( + f"ECHO missing validated payload at byte {offset}: " + f"{fields!r}") try: - pos = int(fields[3]) + pos = _uint(fields[3],"ECHO position") except (ValueError, IndexError): - problems.append(f"malformed ECHO frame: {fields!r}") + problems.append( + f"malformed ECHO frame at byte {offset}: {fields!r}") continue echo_positions.append(pos) - if pos == 0 and (fields[4] != b"nan" or fields[5] != b"0"): - problems.append(f"ECHO position 0 should carry 'nan 0': {fields!r}") - if pos > 0 and len(fields) < 6: - problems.append(f"short ECHO frame: {fields!r}") - if echo_positions and echo_positions != list(range(len(echo_positions))): - problems.append(f"ECHO positions not contiguous from 0: {echo_positions}") - return data_frames, len(echo_positions), problems, flags + if pos == 0: + if len(fields) != 6 or fields[4] != b"nan" or fields[5] != b"0": + problems.append( + f"ECHO position 0 should carry exactly 'nan 0' at " + f"byte {offset}: {fields!r}") + else: + tail_problems, tail_forms = _numeric_tail( + fields, 4, expected_k, vocab, "ECHO", offset) + problems.extend(tail_problems) + forms_used.extend(tail_forms) + else: + problems.append( + f"unknown targeted frame kind at byte {offset}: {fields!r}") + any_globals = any(global_indices[kind] for kind in _GLOBAL_KINDS) + if global_indices[b"BANNER"] or global_indices[b"LOADED"]: + required = ( + (b"BANNER", 0), (b"LOADED", 1), (b"READY", 2), (b"STAT", 3), + (b"HWINFO", 4), (b"TIERS", 5), (b"EMAP", 6), + ) + for owned_kind, expected_index in required: + found = global_indices[owned_kind] + if (owned_kind in (b"BANNER", b"LOADED", b"READY", b"STAT") and + found != [expected_index]): + problems.append( + f"expected one {owned_kind.decode()} as frame " + f"{expected_index}, found {found}") + elif (owned_kind not in (b"BANNER", b"LOADED", b"READY", b"STAT") and + (not found or found[0] != expected_index)): + problems.append( + f"expected startup {owned_kind.decode()} as frame " + f"{expected_index}, found {found}") + if len(frames) > 7 and frames[7][0][0] not in (b"ACCEPT", b"ERROR"): + problems.append( + f"unexpected full-process pre-request frame 7: {frames[7][0]!r}") + elif any_globals: + ready = global_indices[b"READY"] + stat = global_indices[b"STAT"] + if ready != [0]: + problems.append(f"expected one READY as frame 0, found {ready}") + if stat != [1]: + problems.append(f"expected one startup STAT as frame 1, found {stat}") + if len(accepts) != 1: + problems.append(f"expected exactly one ACCEPT, found {len(accepts)}") + elif len(accept_prompts)==1: + prompt=accept_prompts[0] + if echo_positions!=list(range(prompt)): + problems.append(f"ECHO positions {echo_positions} != required 0..{prompt-1}") + if len(dones) != 1: + problems.append(f"expected exactly one DONE, found {len(dones)}") + elif len(done_values)==1 and done_values[0] is not None: + emitted,_,_,_,done_prompt,_=done_values[0] + if emitted!=data_frames: + problems.append(f"DONE emitted {emitted} != observed DATA {data_frames}") + if len(accept_prompts)==1 and done_prompt!=accept_prompts[0]: + problems.append(f"DONE prompt count {done_prompt} != ACCEPT {accept_prompts[0]}") + if data_frames<=0: + problems.append("no DATA frames found for this id -- wrong id, or the run produced nothing") + return data_frames, len(echo_positions), problems, mode, collections.Counter(forms_used) def main(): @@ -138,21 +670,35 @@ def main(): parser.add_argument("--id", required=True, help="request id to audit") parser.add_argument("--topk", type=int, required=True, help="the SUBMIT logprobs=k value the request used") + parser.add_argument("--vocab", type=int, required=True, + help="checkpoint vocabulary size used to bound token ids") args = parser.parse_args() + try: + if _uint(args.id.encode("ascii"), "requested id", _UINT64_MAX) < 1: + raise ValueError("requested id must be positive") + except (UnicodeEncodeError, ValueError) as exc: + parser.error(str(exc)) + if not 1 <= args.topk <= 32: + parser.error("--topk must be in 1..32 for an opted-in B1 evidence run") + if args.vocab < 1: + parser.error("--vocab must be positive") blob = open(args.transcript, "rb").read() - frames = parse_frames(blob) - data_frames, echo_frames, problems, flags = check(frames, args.id, args.topk) - print(f"[gapcheck] id={args.id}: {data_frames} DATA frames, " - f"{echo_frames} ECHO frames, {len(flags)} flagged non-finite") - if data_frames == 0: - problems.append("no DATA frames found for this id -- wrong id, or the " - "run produced nothing (check ERROR frames)") - for flag in flags: - print(f"[gapcheck] FLAG: {flag}") + frames, framing_problems = parse_frames(blob) + data_frames, echo_frames, problems, mode, forms_used = check( + frames, framing_problems, args.id, args.topk, args.vocab) + # Informational only -- see _numeric_value: per-token form + # classification is inherently ambiguous, so this tally never affects + # the verdict, only what a reviewer sees. + forms_summary = (", ".join(f"{form}={forms_used[form]}" + for form in sorted(forms_used)) + if forms_used else "none") + print(f"[gapcheck] id={args.id}: capture_mode={mode}, {data_frames} DATA " + f"frames, {echo_frames} ECHO frames, numeric form tally: " + f"{forms_summary}") for problem in problems: print(f"[gapcheck] {problem}") - verdict = ("FAIL" if problems - else "PASS: every generated token carries a logprob value") + verdict = ("FAIL" if problems else + "PASS: complete request; every generated token carries a valid logprob table") print(f"[gapcheck] {verdict}") return 1 if problems else 0 diff --git a/c/tests/check_native_mtp_witness.py b/c/tests/check_native_mtp_witness.py new file mode 100644 index 000000000..930143ca1 --- /dev/null +++ b/c/tests/check_native_mtp_witness.py @@ -0,0 +1,1060 @@ +#!/usr/bin/env python3 +"""Capture or validate one hash-bound native-MTP accepted-token witness. + +The decisive future run is launched directly, never through a shell +pipeline. The capture command takes only absolute paths and supplies the +resolved model snapshot to the child through ``SNAP`` in the recorded +environment:: + + python3 tests/check_native_mtp_witness.py capture \ + --binary /absolute/build/colibri \ + --snapshot /absolute/model \ + --container-manifest /absolute/model-payloads.sha256 \ + --input /absolute/submit.raw --run-dir /absolute/evidence/run-001 \ + --id 7 --topk 5 --vocab 154880 + +It resolves the snapshot once, then launches and binds that exact root +through ``SNAP``. The frozen twelve-key CPU environment includes +``KVSAVE=0``, ``USAGE_SAVE=0``, ``COLI_NO_OMP_TUNE=1``, and C locales +rather than inheriting ambient state. It captures stdout and stderr +separately, writes the direct child status, then freezes every artifact +and protocol payload in one content-addressed binding. + +The container manifest is an exact sorted ``sha256sum`` inventory of every +regular snapshot file (````); +symlinks and undeclared or missing payloads fail closed before the child +runs. Exit zero requires the full compound witness -- a configuration +banner alone, or a bare acceptance marker alone, is never enough; both are +refused independently below. + +``validate`` re-checks an already-frozen binding against the artifacts it +names. That replay proves the recorded bytes are internally consistent, +but it cannot itself prove the artifacts came from one common run -- only +a live ``capture`` can claim that, so ``validate`` always reports its +provenance as incomplete even when the replay is clean. Both subcommands +apply the same "exactly one accepted token" requirement to the underlying +evidence: a replay that binds zero or more than one accepted token is +refused exactly like a live capture would be. + +Explicit-CUDA drafting and CUDA auto-off are separate future runs. The +decisive witness above retains explicit positive ``DRAFT``. The auto-off +run must omit ``DRAFT`` and may legitimately report the engine's inactive- +draft form; it cannot claim an accepted native-MTP witness and is not a +``capture`` success for this instrument -- the frozen environment always +binds an explicit positive ``DRAFT`` and refuses before the child ever +launches otherwise. + +Exit status is one of four values, each with its own stderr prefix so the +caller can tell them apart without parsing the message body: ``0`` is a +live-capture ``PASS`` (stdout only, no stderr); ``1`` is every ordinary +``WitnessError``/``OSError`` (a malformed, incomplete, or inconsistent +witness, or an offline ``validate`` replay, which is always incomplete on +provenance even when clean) and prints ``[native-mtp] INCOMPLETE: ...``; +``2`` is reserved by ``argparse`` for command-line usage errors; ``3`` is +the distinct ``EngineWitnessUnsupported`` outcome below and prints +``[native-mtp] UNSUPPORTED: ...`` instead of the generic incomplete +prefix, because that case is not a malformed witness -- it is proof the +engine build cannot produce this witness at all. + +Current limitation: the accepted-token proof below depends on the engine +printing an explicit per-emission stderr record (guarded by the +``MTP_DEBUG`` environment key this tool always sets) naming which decoded +token came from an accepted native-MTP draft. A build of the engine that +does not print that record can still show every other sign of an active, +proposing native-MTP session -- the loaded banner, generated tokens, HIT +proposals -- without ever proving that any generated token was itself the +accepted one. This tool tells that apart from an ordinary validation +failure with a distinct, named status rather than reporting either a bare +failure or a false success; whichever run recipe launches the engine +should confirm at build time that this record is compiled in before +trusting a bare "no witness" result to mean "no acceptance happened". + +No model is run by the committed tests -- they drive this tool against an +injected stand-in for the direct child launch. +""" + +import argparse +from dataclasses import dataclass +import hashlib +import importlib.util +import json +import os +import pathlib +import re +import stat +import subprocess +import sys +import tempfile +from types import MappingProxyType + + +HERE = pathlib.Path(__file__).resolve().parent +TOOLS = HERE.parent / "tools" +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) +from engine_evidence import PreambleError, parse_engine_loaded + + +def _load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +GAPS = _load_module("native_mtp_gap_checker", HERE / "check_data_logprob_gaps.py") + + +SCHEMA = "colibri-b1-native-mtp-witness/2" +SNAPSHOT_SCHEMA = "colibri-snapshot-inventory/1" +CAPTURE_OUTCOME_SCHEMA = "colibri-b1-native-mtp-live-capture-outcome/1" +_INT32_MAX = 2**31 - 1 +_MAX_PROMPT_BYTES = 16 * 1024 * 1024 +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_SUBMIT_RE = re.compile( + rb"SUBMIT (?P[1-9][0-9]*) 0 (?P0|[1-9][0-9]*) " + rb"(?P0|[1-9][0-9]*) 0 1 0 logprobs=(?P[1-9][0-9]*)") +_STOP_RE = re.compile( + r"^\[stop\] (?P0|[1-9][0-9]*) stop tokens:" + r"(?P(?: (?:0|[1-9][0-9]*))*)" + r"(?: \((?P0|[1-9][0-9]*) from the tokenizer's special set\))?$") +_MTPDBG_RE = re.compile( + r"^\[mtpdbg\] draft0=(?P0|[1-9][0-9]*) " + r"verified=(?P0|[1-9][0-9]*) (?PHIT|miss)$") +_MTPEMIT_RE = re.compile( + r"^\[mtpemit\] request=(?P[1-9][0-9]*) " + r"ordinal=(?P0|[1-9][0-9]*) " + r"token=(?P0|[1-9][0-9]*)$") +_SNAPSHOT_MANIFEST_RE = re.compile( + rb"(?P[0-9a-f]{64}) (?P[\x21-\x7e]+)") + + +class WitnessError(ValueError): + """The bundle cannot prove the complete native-MTP witness.""" + + +class EngineWitnessUnsupported(WitnessError): + """The captured engine build never printed the accepted-token record. + + Distinct from every other WitnessError: this is not a malformed or + incomplete witness, it is a witness the engine build cannot produce at + all because the per-emission stderr record this tool binds to + (guarded by ``MTP_DEBUG`` in the frozen environment) was never + printed, even though the run otherwise shows an active, proposing + native-MTP session. + """ + + +@dataclass(frozen=True) +class _DirectCaptureRun: + root: pathlib.Path + binary: pathlib.Path + container: pathlib.Path + snapshot: pathlib.Path + environment: pathlib.Path + input: pathlib.Path + status: pathlib.Path + stdout: pathlib.Path + stderr: pathlib.Path + request_id: int + topk: int + vocab: int + + @property + def artifacts(self): + return MappingProxyType({ + "binary": self.binary, + "container": self.container, + "environment": self.environment, + "input": self.input, + "status": self.status, + "stdout": self.stdout, + "stderr": self.stderr, + }) + + +def _reject_constant(value): + raise WitnessError(f"non-JSON constant: {value}") + + +def _canonical_json(value): + return (json.dumps(value, ensure_ascii=True, sort_keys=True, + separators=(",", ":")) + "\n").encode("ascii") + + +def _strict_json_bytes(raw, label, require_canonical=True): + try: + value = json.loads(raw.decode("ascii"), parse_constant=_reject_constant) + except (UnicodeDecodeError, json.JSONDecodeError, WitnessError) as exc: + raise WitnessError(f"invalid {label} JSON: {exc}") from exc + if require_canonical and raw != _canonical_json(value): + raise WitnessError(f"noncanonical {label} JSON bytes") + return value + + +def _sha256_bytes(raw): + return hashlib.sha256(raw).hexdigest() + + +def _absolute_file(path, label): + candidate = pathlib.Path(path) + if not candidate.is_absolute() or str(candidate) != os.path.normpath(str(candidate)): + raise WitnessError(f"{label} path must be canonical absolute: {path!r}") + if not candidate.is_file(): + raise WitnessError(f"{label} is not a file: {path!r}") + return candidate + + +def _read_artifact(path, label): + """Read one regular artifact once and derive metadata from those bytes.""" + path = _absolute_file(path, label).resolve() + raw = path.read_bytes() + return (_artifact_from_bytes(path, raw), raw) + + +def _snapshot_stat(st_result): + """Metadata that changes on replacement or mutate-then-restore.""" + return ( + st_result.st_dev, st_result.st_ino, stat.S_IFMT(st_result.st_mode), + st_result.st_size, st_result.st_mtime_ns, st_result.st_ctime_ns, + ) + + +def _parse_snapshot_manifest(raw): + if not raw or not raw.endswith(b"\n") or b"\r" in raw or b"\0" in raw: + raise WitnessError( + "snapshot manifest must be nonempty canonical LF records") + entries = [] + previous = None + for raw_line in raw.splitlines(): + match = _SNAPSHOT_MANIFEST_RE.fullmatch(raw_line) + if not match: + raise WitnessError(f"malformed snapshot manifest record: {raw_line!r}") + path_raw = match.group("path") + try: + path_text = path_raw.decode("ascii") + except UnicodeDecodeError as exc: + raise WitnessError("snapshot manifest path is not ASCII") from exc + pure = pathlib.PurePosixPath(path_text) + if (not pure.parts or path_text in (".", "..") or + pure.is_absolute() or path_text != pure.as_posix() or + path_text.startswith("/") or "\\" in path_text or + re.match(r"^[A-Za-z]:", path_text) or + any(part in ("", ".", "..") for part in pure.parts) or + "//" in path_text or any(ord(char) < 0x21 for char in path_text)): + raise WitnessError( + f"snapshot manifest path is not canonical relative: {path_text!r}") + if previous is not None and path_text <= previous: + raise WitnessError("snapshot manifest paths are duplicate or unsorted") + previous = path_text + entries.append((path_text, match.group("digest").decode("ascii"))) + if not entries: + raise WitnessError("snapshot manifest has no payload denominator") + return tuple(entries) + + +def _walk_snapshot(root): + """Return every regular payload, byte-sorted by relative path (the same + order a sorted ``sha256sum`` manifest uses), and a no-follow stability + fingerprint. The payload order and the manifest's declared order must + agree independent of directory-tree shape -- a filesystem walk that is + merely sorted per-directory (e.g. "model" before "model.json", because + the walk compares bare entry names) does not agree with a manifest + sorted by full relative path (where "model.json" < "model/inner.bin" + because "." sorts before "/"), so the payload list is explicitly + re-sorted by its own full path text before being handed back. + """ + root = pathlib.Path(root) + try: + root_stat = os.lstat(root) + except FileNotFoundError as exc: + raise WitnessError("snapshot directory is unavailable") from exc + if stat.S_ISLNK(root_stat.st_mode) or not stat.S_ISDIR(root_stat.st_mode): + raise WitnessError("snapshot root must be a real directory, not a symlink") + payloads = [] + fingerprint = [(".", _snapshot_stat(root_stat))] + + def visit(directory, prefix): + try: + with os.scandir(directory) as scan: + entries = sorted(scan, key=lambda entry: entry.name) + except OSError as exc: + raise WitnessError(f"cannot inventory snapshot directory {directory}") from exc + for entry in entries: + if "/" in entry.name or entry.name in (".", ".."): + raise WitnessError("snapshot contains a noncanonical entry name") + rel = entry.name if not prefix else f"{prefix}/{entry.name}" + try: + item_stat = entry.stat(follow_symlinks=False) + except OSError as exc: + raise WitnessError(f"cannot stat snapshot entry {rel!r}") from exc + fingerprint.append((rel, _snapshot_stat(item_stat))) + if stat.S_ISLNK(item_stat.st_mode): + raise WitnessError(f"snapshot symlink is forbidden: {rel!r}") + if stat.S_ISDIR(item_stat.st_mode): + visit(pathlib.Path(directory) / entry.name, rel) + elif stat.S_ISREG(item_stat.st_mode): + payloads.append(rel) + else: + raise WitnessError(f"snapshot entry is not regular: {rel!r}") + + visit(root, "") + return tuple(sorted(payloads)), tuple(fingerprint) + + +def _hash_snapshot_payload(root, relative): + if sys.platform == "win32": + # The identity check below requires POSIX stat semantics: fstat of the + # open descriptor must agree with lstat of the path on device and + # inode. Windows derives those differently for a handle and a path + # (CI observed disagreement on a subset of files), so the witness + # refuses rather than report a spurious "payload changed". + raise WitnessError( + "snapshot payload identity requires POSIX stat semantics " + "(fstat/lstat agreement on device and inode); Windows is unsupported") + path = pathlib.Path(root).joinpath(*relative.split("/")) + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise WitnessError(f"cannot open snapshot payload {relative!r}") from exc + digest = hashlib.sha256() + size = 0 + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise WitnessError(f"snapshot payload is not regular: {relative!r}") + while True: + block = os.read(descriptor, 1024 * 1024) + if not block: + break + digest.update(block) + size += len(block) + after = os.fstat(descriptor) + finally: + os.close(descriptor) + try: + linked = os.lstat(path) + except FileNotFoundError as exc: + raise WitnessError(f"snapshot payload vanished: {relative!r}") from exc + if (_snapshot_stat(before) != _snapshot_stat(after) or + _snapshot_stat(after) != _snapshot_stat(linked)): + raise WitnessError(f"snapshot payload changed while hashing: {relative!r}") + return digest.hexdigest(), size + + +def _apply_snapshot_manifest(snapshot, manifest_raw): + """Apply exact manifest bytes to the complete no-follow snapshot inventory.""" + root = pathlib.Path(snapshot) + entries = _parse_snapshot_manifest(manifest_raw) + declared = tuple(path for path, _ in entries) + actual, before = _walk_snapshot(root) + if actual != declared: + missing = sorted(set(declared) - set(actual)) + extra = sorted(set(actual) - set(declared)) + raise WitnessError( + f"snapshot inventory mismatch; missing={missing!r} extra={extra!r}") + total = 0 + for relative, expected in entries: + observed, size = _hash_snapshot_payload(root, relative) + if observed != expected: + raise WitnessError(f"snapshot payload SHA-256 mismatch: {relative!r}") + total += size + actual_after, after = _walk_snapshot(root) + if actual_after != actual or after != before: + raise WitnessError("snapshot inventory changed while applying manifest") + inventory_digest = _sha256_bytes( + (SNAPSHOT_SCHEMA + "\n").encode("ascii") + manifest_raw) + summary = { + "schema": SNAPSHOT_SCHEMA, "files": len(entries), + "bytes": total, "sha256": inventory_digest, + } + return summary, after + + +def _artifact_from_bytes(path, raw): + path = pathlib.Path(path).resolve() + return { + "path": str(path), "size": len(raw), "sha256": _sha256_bytes(raw), + } + + +def _materialize_capture_artifact(path, root, label, raw): + """Materialize immutable capture bytes without accepting a supplied path.""" + stream = _open_capture_file(path, label) + try: + stream.write(raw) + stream.flush() + opened = os.fstat(stream.fileno()) + try: + entry = os.lstat(path) + except FileNotFoundError as exc: + raise WitnessError(f"{label} capture path was removed") from exc + if (not stat.S_ISREG(entry.st_mode) or + (entry.st_dev, entry.st_ino) != (opened.st_dev, opened.st_ino)): + raise WitnessError(f"{label} capture path was replaced") + resolved = pathlib.Path(path).resolve(strict=True) + if resolved.parent != pathlib.Path(root).resolve(strict=True): + raise WitnessError(f"{label} capture path escaped the run root") + if opened.st_size != len(raw): + raise WitnessError(f"{label} capture size changed while materializing") + return (_artifact_from_bytes(resolved, raw), raw) + finally: + stream.close() + + +def _freeze_anonymous_stream(stream): + """Read immutable bytes from an anonymous descriptor after child exit.""" + stream.flush() + stream.seek(0) + return stream.read() + + +def _open_capture_file(path, label): + """Create one capture-owned regular file without following an entry.""" + try: + return open(path, "x+b") + except FileExistsError as exc: + raise WitnessError(f"{label} capture path was precreated") from exc + + +def _payload_records(stdout_blob): + frames, _framing = GAPS.parse_frames(stdout_blob) + records = [] + for fields, payload, _offset in frames: + if fields[0] not in (b"DATA", b"ECHO") or payload is None: + continue + try: + request_id = int(fields[1].decode("ascii")) + except (IndexError, UnicodeDecodeError, ValueError) as exc: + raise WitnessError(f"cannot bind malformed payload frame: {fields!r}") from exc + records.append({ + "kind": fields[0].decode("ascii"), "id": request_id, + "size": len(payload), "sha256": _sha256_bytes(payload), + }) + return records + + +def _binding_id(record): + body = dict(record) + body.pop("binding_id", None) + return _sha256_bytes(_canonical_json(body)) + + +def _build_capture_binding(run, external_before, owned_capture, + snapshot_inventory): + """Freeze the one direct-capture run; no post-hoc path accepts artifacts.""" + run_artifacts = run.artifacts + owned_names = {"environment", "input", "status", "stdout", "stderr"} + if set(owned_capture) != owned_names: + raise WitnessError("owned capture denominator is not exact") + for name in owned_names: + if run_artifacts[name].parent != run.root: + raise WitnessError(f"{name} is outside the direct-capture run root") + artifacts = {name: owned_capture[name][0] for name in owned_names} + blobs = {name: owned_capture[name][1] for name in owned_names} + for name in ("binary", "container"): + path = run_artifacts[name] + item, raw = _read_artifact(path, name) + if name == "container" and not raw: + raise WitnessError("container artifact must be a nonempty regular file") + if item != external_before[name]: + raise WitnessError(f"{name} changed during direct capture") + artifacts[name] = item + blobs[name] = raw + record = { + "schema": SCHEMA, + "snapshot": str(run.snapshot), + "request": { + "id": run.request_id, "topk": run.topk, "vocab": run.vocab, + }, + "snapshot_inventory": snapshot_inventory, + "artifacts": {name: artifacts[name] for name in sorted(artifacts)}, + "payloads": _payload_records(blobs["stdout"]), + } + record["binding_id"] = _binding_id(record) + return record + + +def _write_binding(path, record): + output = pathlib.Path(path) + raw = _canonical_json(record) + with open(output, "xb") as stream: + if stream.write(raw) != len(raw): + raise WitnessError("short write while freezing binding") + stream.flush() + os.fsync(stream.fileno()) + + +def _parse_binding(raw): + """Strictly parse one already-frozen binding byte image.""" + record = _strict_json_bytes(raw, "binding") + if set(record) != { + "schema", "binding_id", "snapshot", "request", "artifacts", + "payloads", "snapshot_inventory"}: + raise WitnessError("binding keys are not exact") + if record["schema"] != SCHEMA: + raise WitnessError(f"unknown binding schema: {record['schema']!r}") + if (not isinstance(record["binding_id"], str) or + not _SHA256_RE.fullmatch(record["binding_id"])): + raise WitnessError("binding_id is not canonical SHA-256") + if record["binding_id"] != _binding_id(record): + raise WitnessError("binding_id does not bind this record") + return record + + +def _load_binding(path): + raw = pathlib.Path(path).read_bytes() + return _parse_binding(raw) + + +def _validate_artifacts(record): + artifacts = record["artifacts"] + expected_names = { + "binary", "container", "environment", "input", "status", "stdout", + "stderr", + } + if not isinstance(artifacts, dict) or set(artifacts) != expected_names: + raise WitnessError("artifact denominator is not exact") + paths = [] + blobs = {} + for name in sorted(expected_names): + item = artifacts[name] + if not isinstance(item, dict) or set(item) != {"path", "size", "sha256"}: + raise WitnessError(f"{name} artifact keys are not exact") + if (not isinstance(item["path"], str) or + type(item["size"]) is not int or item["size"] < 0 or + not isinstance(item["sha256"], str) or + not _SHA256_RE.fullmatch(item["sha256"])): + raise WitnessError(f"{name} artifact metadata is malformed") + actual, raw = _read_artifact(item["path"], name) + if actual["path"] != item["path"]: + raise WitnessError(f"{name} artifact path is not resolved canonical") + paths.append(actual["path"]) + if actual["size"] != item["size"]: + raise WitnessError(f"{name} artifact size mismatch") + if actual["sha256"] != item["sha256"]: + raise WitnessError(f"{name} artifact SHA-256 mismatch") + if name == "container" and not raw: + raise WitnessError("container artifact must be a nonempty regular file") + blobs[name] = raw + if len(set(paths)) != len(paths): + raise WitnessError("artifact paths must be distinct") + return MappingProxyType(blobs) + + +def _validate_request_meta(record): + request = record["request"] + if not isinstance(request, dict) or set(request) != {"id", "topk", "vocab"}: + raise WitnessError("request binding keys are not exact") + request_id, topk, vocab = request["id"], request["topk"], request["vocab"] + if (type(request_id) is not int or not 1 <= request_id <= 2**64 - 1 or + type(topk) is not int or not 1 <= topk <= 32 or + type(vocab) is not int or not 1 <= vocab <= _INT32_MAX): + raise WitnessError("request binding values are outside their domains") + return request_id, topk, vocab + + +def _validate_environment(raw, snapshot): + environment = _strict_json_bytes(raw, "environment") + if (not isinstance(environment, dict) or + any(not isinstance(k, str) or not isinstance(v, str) + for k, v in environment.items())): + raise WitnessError("environment must be an exact string map") + required = { + "SNAP": snapshot, "SERVE": "1", "SERVE_BATCH": "1", + "KV_SLOTS": "1", "MTP_DEBUG": "1", "KVSAVE": "0", + "USAGE_SAVE": "0", "COLI_NO_OMP_TUNE": "1", + "LANG": "C", "LC_ALL": "C", + } + for key, expected in required.items(): + if environment.get(key) != expected: + raise WitnessError(f"environment {key} must be {expected!r}") + draft = environment.get("DRAFT") + if (draft is None or not re.fullmatch(r"(?:[1-9]|[1-5][0-9]|6[0-3])", draft)): + raise WitnessError("environment DRAFT must be explicit positive 1..63") + ctx = environment.get("CTX") + if (ctx is None or not re.fullmatch(r"(?:0|[1-9][0-9]*)", ctx) or + not 1 <= int(ctx) <= _INT32_MAX): + raise WitnessError("environment CTX must be canonical positive int32") + if set(environment) != set(required) | {"DRAFT", "CTX"}: + raise WitnessError("environment keys must equal the exact CPU witness map") + return environment + + +def _validate_input(blob, request_id, topk): + newline = blob.find(b"\n") + if newline < 0: + raise WitnessError("input lacks a complete SUBMIT header") + match = _SUBMIT_RE.fullmatch(blob[:newline]) + if not match: + raise WitnessError("input must contain one canonical greedy grammar-free SUBMIT") + values = {name: int(match.group(name)) + for name in ("id", "size", "maximum", "topk")} + if values["id"] != request_id or values["topk"] != topk: + raise WitnessError("input SUBMIT does not match bound request") + if not 1 <= values["size"] <= _MAX_PROMPT_BYTES: + raise WitnessError("input prompt payload must be 1..16 MiB") + if not 1 <= values["maximum"] <= _INT32_MAX: + raise WitnessError("input maximum must be a positive int32") + payload_start = newline + 1 + payload_end = payload_start + values["size"] + if (payload_end >= len(blob) or blob[payload_end:payload_end + 1] != b"\n" or + payload_end + 1 != len(blob)): + raise WitnessError("input must end after exactly one framed SUBMIT payload") + payload = blob[payload_start:payload_end] + if b"\0" in payload: + raise WitnessError("input prompt payload contains NUL") + return values, payload + + +def _validate_status(raw): + if raw != b"0\n": + raise WitnessError(f"engine status is not exact zero: {raw!r}") + + +def _validate_stdout(blob, request_id, topk, vocab, expected_payloads): + frames, framing = GAPS.parse_frames(blob) + if GAPS.capture_mode(frames) != "full-process": + raise WitnessError("stdout is not a complete-process capture") + data, echo, problems, _mode, _forms = GAPS.check( + frames, framing, str(request_id), topk, vocab) + if problems: + raise WitnessError("stdout gap check failed: " + "; ".join(problems)) + # data/echo are both guaranteed >= 1 here: GAPS.check() itself + # reports "no DATA frames found" whenever data == 0, and the same + # ACCEPT-prompt/ECHO-position accounting forces echo >= 1 whenever + # ACCEPT is present at all -- a locally repeated ">= 1" guard here + # would be unreachable dead code given that guarantee. + for fields, _payload, _offset in frames: + if fields[0] in GAPS._GLOBAL_KINDS: + continue + if fields[0] not in (b"ACCEPT", b"ECHO", b"DATA", b"DONE"): + raise WitnessError(f"stdout contains an unowned request record: {fields!r}") + try: + frame_id = int(fields[1].decode("ascii")) + except (IndexError, UnicodeDecodeError, ValueError) as exc: + raise WitnessError(f"stdout request id is malformed: {fields!r}") from exc + if frame_id != request_id: + raise WitnessError(f"stdout contains second request id {frame_id}") + + lines = blob.split(b"\n", 2) + if len(lines) != 3: + raise WitnessError("stdout lacks complete banner/load prefix") + try: + loaded = parse_engine_loaded(lines[1].decode("ascii")) + except (UnicodeDecodeError, PreambleError) as exc: + raise WitnessError(f"stdout load record is invalid: {exc}") from exc + if loaded["mtp"] != "ACTIVE" or not 1 <= loaded["draft"] <= 63: + raise WitnessError("stdout does not prove active positive-depth native MTP") + actual_payloads = _payload_records(blob) + if actual_payloads != expected_payloads: + raise WitnessError("bound protocol payload hashes do not match stdout") + data_rows = [] + for fields, _payload, _offset in frames: + if fields[0] != b"DATA": + continue + try: + row_topk = int(fields[4].decode("ascii")) + pairs = { + int(fields[5 + 2 * index].decode("ascii")): + fields[6 + 2 * index] + for index in range(row_topk) + } + except (IndexError, UnicodeDecodeError, ValueError) as exc: + raise WitnessError(f"cannot bind malformed DATA row: {fields!r}") from exc + if len(pairs) != row_topk: + raise WitnessError("DATA top-k token identities are not unique") + data_rows.append({"target": fields[3], "topk": pairs}) + if len(data_rows) != data: + raise WitnessError("DATA denominator changed while binding rows") + return loaded, data_rows + + +def _validate_stderr(raw, vocab, request_id, data_rows): + if not raw or not raw.endswith(b"\n") or b"\r" in raw: + raise WitnessError("stderr must be nonempty canonical newline records") + try: + lines = raw.decode("utf-8").splitlines() + except UnicodeDecodeError as exc: + raise WitnessError("stderr is not UTF-8") from exc + if any(line.startswith("[GRAMMAR]") or line.startswith("[CORPUS]") + for line in lines): + raise WitnessError("stderr reports an alternate grammar/corpus draft source") + + stop_matches = [] + stop_index = None + for index, line in enumerate(lines): + match = _STOP_RE.fullmatch(line) + if match: + stop_matches.append(match) + stop_index = index + elif line.startswith("[stop] ") and " stop tokens:" in line: + raise WitnessError(f"malformed armed-stop record: {line!r}") + if len(stop_matches) != 1: + raise WitnessError(f"expected one armed-stop record, found {len(stop_matches)}") + match = stop_matches[0] + count = int(match.group("count")) + ids_text = match.group("ids").strip() + stop_ids = [] if not ids_text else [int(value) for value in ids_text.split(" ")] + if count != len(stop_ids) or count > 64 or len(set(stop_ids)) != len(stop_ids): + raise WitnessError("armed-stop count/uniqueness mismatch") + if any(token < 0 or token >= vocab for token in stop_ids): + raise WitnessError("armed-stop token is outside the vocabulary") + special = match.group("special") + if special is not None and not 1 <= int(special) <= count: + raise WitnessError("armed-stop special-token count is invalid") + + qualifying = [] + pending = [] + bound_ordinals = set() + markers = 0 + mtpemit_records = 0 + for index, line in enumerate(lines): + if line.startswith("[mtpemit]"): + mtpemit_records += 1 + bound = _MTPEMIT_RE.fullmatch(line) + if not bound: + raise WitnessError(f"malformed mtpemit record: {line!r}") + if index <= stop_index: + raise WitnessError("mtpemit record precedes the same-run stop set") + bound_request = int(bound.group("request")) + ordinal = int(bound.group("ordinal")) + token = int(bound.group("token")) + if bound_request != request_id: + raise WitnessError("mtpemit request does not match the bound request") + if token >= vocab: + raise WitnessError("mtpemit token is outside the vocabulary") + if not pending: + raise WitnessError("mtpemit has no preceding qualifying HIT") + if token != pending.pop(0): + raise WitnessError("mtpemit token does not match its qualifying HIT") + if ordinal in bound_ordinals: + raise WitnessError("duplicate/replayed mtpemit emission ordinal") + if bound_ordinals and ordinal <= max(bound_ordinals): + raise WitnessError("mtpemit emission ordinals are not strictly ordered") + if ordinal >= len(data_rows): + raise WitnessError("mtpemit emission ordinal has no DATA row") + row = data_rows[ordinal] + if token not in row["topk"]: + raise WitnessError("mtpemit token is absent from its DATA top-k") + if row["topk"][token] != row["target"]: + raise WitnessError("mtpemit token does not own its DATA target logprob") + bound_ordinals.add(ordinal) + continue + if not line.startswith("[mtpdbg]"): + continue + marker = _MTPDBG_RE.fullmatch(line) + if not marker: + raise WitnessError(f"malformed mtpdbg record: {line!r}") + markers += 1 + if index <= stop_index: + raise WitnessError("mtpdbg proposal precedes the same-run stop set") + draft = int(marker.group("draft")) + verified = int(marker.group("verified")) + result = marker.group("result") + if draft >= vocab or verified >= vocab: + raise WitnessError("mtpdbg token is outside the vocabulary") + expected = "HIT" if draft == verified else "miss" + if result != expected: + raise WitnessError("mtpdbg HIT/miss label contradicts token equality") + if result == "HIT" and draft not in stop_ids: + qualifying.append(draft) + pending.append(draft) + if markers < 1: + raise WitnessError("stderr contains no mtpdbg proposal") + if not qualifying: + raise WitnessError("stderr has no equal non-stop native-MTP HIT") + if mtpemit_records < 1: + raise EngineWitnessUnsupported( + "engine does not emit the accepted-token witness line " + "(requires MTP_DEBUG support)") + if pending: + raise WitnessError("qualifying HIT is missing its mtpemit/DATA binding") + if len(bound_ordinals) != len(qualifying): + raise WitnessError("qualifying HIT/mtpemit denominator mismatch") + return qualifying + + +def _validate_components(record, blobs): + """Apply the one semantic validator to one record and immutable byte map.""" + request_id, topk, vocab = _validate_request_meta(record) + snapshot = record["snapshot"] + if (not isinstance(snapshot, str) or not pathlib.Path(snapshot).is_absolute() or + snapshot != os.path.normpath(snapshot)): + raise WitnessError("bound snapshot path is not canonical absolute") + if not pathlib.Path(snapshot).is_dir(): + raise WitnessError("bound snapshot directory is unavailable") + inventory, _ = _apply_snapshot_manifest(snapshot, blobs["container"]) + if record["snapshot_inventory"] != inventory: + raise WitnessError("bound snapshot inventory does not match applied manifest") + environment = _validate_environment(blobs["environment"], snapshot) + input_meta, _ = _validate_input(blobs["input"], request_id, topk) + _validate_status(blobs["status"]) + loaded, data_rows = _validate_stdout( + blobs["stdout"], request_id, topk, vocab, record["payloads"]) + qualifying = _validate_stderr( + blobs["stderr"], vocab, request_id, data_rows) + if int(environment["DRAFT"]) != loaded["draft"]: + raise WitnessError("configured DRAFT does not equal loaded effective draft") + if len(data_rows) > input_meta["maximum"]: + raise WitnessError("emitted DATA count exceeds bound input maximum") + return { + "binding_id": record["binding_id"], "request_id": request_id, + "accepted_tokens": qualifying, + } + + +def validate_binding(path): + """Replay a frozen bundle without claiming common-process provenance.""" + record = _load_binding(path) + blobs = _validate_artifacts(record) + result = _validate_components(record, blobs) + if len(result["accepted_tokens"]) != 1: + raise WitnessError("replay does not bind exactly one accepted token") + result.update({ + "replay_verdict": "COMPLETE", + "provenance_verdict": "INCOMPLETE", + }) + return result + + +def _write_capture_outcome(path, outcome): + """Exclusively freeze and byte-revalidate one live capture outcome.""" + expected = { + "schema", "verdict", "binding_id", "binding_sha256", "request_id", + "accepted_tokens", "run_root", + } + if (not isinstance(outcome, dict) or set(outcome) != expected or + outcome["schema"] != CAPTURE_OUTCOME_SCHEMA or + outcome["verdict"] != "PASS" or + not isinstance(outcome["binding_id"], str) or + not _SHA256_RE.fullmatch(outcome["binding_id"]) or + not isinstance(outcome["binding_sha256"], str) or + not _SHA256_RE.fullmatch(outcome["binding_sha256"]) or + type(outcome["request_id"]) is not int or + not 1 <= outcome["request_id"] <= 2**64 - 1 or + not isinstance(outcome["accepted_tokens"], list) or + len(outcome["accepted_tokens"]) != 1 or + type(outcome["accepted_tokens"][0]) is not int or + not isinstance(outcome["run_root"], str) or + not pathlib.Path(outcome["run_root"]).is_absolute() or + outcome["run_root"] != os.path.normpath(outcome["run_root"])): + raise WitnessError("live capture outcome is malformed") + raw = _canonical_json(outcome) + output = pathlib.Path(path) + with open(output, "xb") as stream: + if stream.write(raw) != len(raw): + raise WitnessError("short write while freezing live capture outcome") + stream.flush() + os.fsync(stream.fileno()) + frozen = output.read_bytes() + if frozen != raw or _strict_json_bytes( + frozen, "live capture outcome") != outcome: + raise WitnessError("live capture outcome failed byte revalidation") + return _sha256_bytes(frozen) + + +def _capture_environment(snapshot, draft, ctx): + return { + "SNAP": snapshot, "SERVE": "1", "SERVE_BATCH": "1", + "KV_SLOTS": "1", "DRAFT": str(draft), "CTX": str(ctx), + "MTP_DEBUG": "1", "KVSAVE": "0", "USAGE_SAVE": "0", + "COLI_NO_OMP_TUNE": "1", "LANG": "C", "LC_ALL": "C", + } + + +def capture_bundle(binary, snapshot, container, input_path, run_dir, + request_id, topk, vocab, draft=1, ctx=4096, + run_fn=subprocess.run): + binary_path = _absolute_file(binary, "binary").resolve() + container_path = _absolute_file(container, "container").resolve() + input_source = _absolute_file(input_path, "input").resolve() + snapshot_path = pathlib.Path(snapshot) + output = pathlib.Path(run_dir) + if (not snapshot_path.is_absolute() or + str(snapshot_path) != os.path.normpath(str(snapshot_path)) or + not snapshot_path.is_dir()): + raise WitnessError("snapshot must be an existing canonical absolute directory") + try: + snapshot_entry = os.lstat(snapshot_path) + except OSError as exc: + raise WitnessError("snapshot directory is unavailable") from exc + if stat.S_ISLNK(snapshot_entry.st_mode): + raise WitnessError("snapshot root symlink is forbidden") + if (not output.is_absolute() or + str(output) != os.path.normpath(str(output)) or output.exists()): + raise WitnessError("run directory must be a new canonical absolute path") + if (type(draft) is not int or not 1 <= draft <= 63 or + type(ctx) is not int or not 1 <= ctx <= _INT32_MAX): + raise WitnessError("capture draft/CTX is outside the decisive domain") + if (type(request_id) is not int or not 1 <= request_id <= 2**64 - 1 or + type(topk) is not int or not 1 <= topk <= 32 or + type(vocab) is not int or not 1 <= vocab <= _INT32_MAX): + raise WitnessError("capture request metadata is outside its domain") + binary_before, binary_raw = _read_artifact(binary_path, "binary") + container_before, container_raw = _read_artifact( + container_path, "container") + resolved_snapshot = snapshot_path.resolve(strict=True) + if (container_path == resolved_snapshot or + resolved_snapshot in container_path.parents): + raise WitnessError("snapshot manifest must be outside the snapshot") + try: + resolved_output_parent = output.parent.resolve(strict=True) + except OSError as exc: + raise WitnessError("run directory parent is unavailable") from exc + if (resolved_output_parent == resolved_snapshot or + resolved_snapshot in resolved_output_parent.parents): + raise WitnessError("run directory must be outside the snapshot") + snapshot_before, stability_before = _apply_snapshot_manifest( + resolved_snapshot, container_raw) + _, input_raw = _read_artifact(input_source, "input") + _validate_input(input_raw, request_id, topk) + output.mkdir(mode=0o700) + output = output.resolve(strict=True) + + paths = { + "environment": output / "environment.json", + "input": output / "request.raw", + "status": output / "engine_status.txt", + "stdout": output / "engine_stdout.raw", + "stderr": output / "engine_stderr.raw", + } + environment = _capture_environment(str(resolved_snapshot), draft, ctx) + environment_raw = _canonical_json(environment) + with tempfile.TemporaryFile(mode="w+b") as stdin, \ + tempfile.TemporaryFile(mode="w+b") as stdout, \ + tempfile.TemporaryFile(mode="w+b") as stderr: + stdin.write(input_raw) + stdin.flush() + stdin.seek(0) + result = run_fn( + [str(binary_path)], stdin=stdin, stdout=stdout, stderr=stderr, + env=environment, cwd=str(output), check=False, shell=False) + stdout_raw = _freeze_anonymous_stream(stdout) + stderr_raw = _freeze_anonymous_stream(stderr) + snapshot_after_child, stability_after_child = _apply_snapshot_manifest( + resolved_snapshot, container_raw) + if (snapshot_after_child != snapshot_before or + stability_after_child != stability_before): + raise WitnessError("snapshot changed during direct child execution") + captured_raw = { + "environment": environment_raw, "input": input_raw, + "status": f"{result.returncode}\n".encode("ascii"), + "stdout": stdout_raw, "stderr": stderr_raw, + } + owned_capture = { + name: _materialize_capture_artifact( + paths[name], output, name, captured_raw[name]) + for name in ("environment", "input", "status", "stdout", "stderr") + } + snapshot_after_capture, stability_after_capture = _apply_snapshot_manifest( + resolved_snapshot, container_raw) + if (snapshot_after_capture != snapshot_before or + stability_after_capture != stability_before): + raise WitnessError("snapshot changed while materializing capture") + + run = _DirectCaptureRun( + root=output, binary=binary_path, container=container_path, + snapshot=resolved_snapshot, environment=paths["environment"], + input=paths["input"], status=paths["status"], + stdout=paths["stdout"], stderr=paths["stderr"], + request_id=request_id, topk=topk, vocab=vocab) + record = _build_capture_binding( + run, MappingProxyType({ + "binary": binary_before, "container": container_before, + }), MappingProxyType(owned_capture), snapshot_before) + snapshot_before_binding, stability_before_binding = _apply_snapshot_manifest( + resolved_snapshot, container_raw) + if (snapshot_before_binding != snapshot_before or + stability_before_binding != stability_before): + raise WitnessError("snapshot changed before binding issuance") + binding_path = output / "binding.json" + component_blobs = MappingProxyType({ + "binary": binary_raw, "container": container_raw, + **captured_raw, + }) + _write_binding(binding_path, record) + expected_binding_raw = _canonical_json(record) + try: + binding_raw = binding_path.read_bytes() + except OSError as exc: + raise WitnessError("frozen binding is unavailable") from exc + if binding_raw != expected_binding_raw: + raise WitnessError("frozen binding failed exact byte revalidation") + frozen_record = _parse_binding(binding_raw) + if frozen_record != record: + raise WitnessError("frozen binding does not equal the issued record") + live_result = _validate_components(frozen_record, component_blobs) + if len(live_result["accepted_tokens"]) != 1: + raise WitnessError("live capture requires exactly one accepted token") + outcome = { + "schema": CAPTURE_OUTCOME_SCHEMA, + "verdict": "PASS", + "binding_id": frozen_record["binding_id"], + "binding_sha256": _sha256_bytes(binding_raw), + "request_id": live_result["request_id"], + "accepted_tokens": live_result["accepted_tokens"], + "run_root": str(output), + } + outcome_sha256 = _write_capture_outcome( + output / "capture_outcome.json", outcome) + live_result.update({ + "replay_verdict": "COMPLETE", + "provenance_verdict": "PASS", + "outcome_sha256": outcome_sha256, + }) + return live_result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + validate = sub.add_parser("validate", help="validate a frozen binding") + validate.add_argument("binding") + + capture = sub.add_parser("capture", help="run one decisive direct capture") + capture.add_argument("--binary", required=True) + capture.add_argument("--snapshot", required=True) + capture.add_argument("--container-manifest", required=True) + capture.add_argument("--input", required=True) + capture.add_argument("--run-dir", required=True) + capture.add_argument("--id", dest="request_id", type=int, required=True) + capture.add_argument("--topk", type=int, required=True) + capture.add_argument("--vocab", type=int, required=True) + capture.add_argument("--draft", type=int, default=1) + capture.add_argument("--ctx", type=int, default=4096) + + args = parser.parse_args(argv) + try: + if args.command == "validate": + result = validate_binding(args.binding) + print(f"[native-mtp] REPLAY binding={result['binding_id']} " + f"request={result['request_id']} " + f"accepted={len(result['accepted_tokens'])}") + print("[native-mtp] INCOMPLETE: offline replay cannot establish " + "common-run provenance", file=sys.stderr) + return 1 + result = capture_bundle( + args.binary, args.snapshot, args.container_manifest, args.input, + args.run_dir, args.request_id, args.topk, args.vocab, + args.draft, args.ctx) + print(f"[native-mtp] PASS provenance=live-capture " + f"binding={result['binding_id']} " + f"request={result['request_id']} " + f"accepted={len(result['accepted_tokens'])} " + f"outcome_sha256={result['outcome_sha256']}") + return 0 + except EngineWitnessUnsupported as exc: + print(f"[native-mtp] UNSUPPORTED: {exc}", file=sys.stderr) + return 3 + except (OSError, WitnessError) as exc: + print(f"[native-mtp] INCOMPLETE: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/c/tests/test_check_ablate_evidence.py b/c/tests/test_check_ablate_evidence.py new file mode 100644 index 000000000..906cd5525 --- /dev/null +++ b/c/tests/test_check_ablate_evidence.py @@ -0,0 +1,915 @@ +"""tools/check_ablate_evidence.py must accept only a complete, self-binding +ABLATE evidence artifact and reject every other input: a truncated or +replayed record stream, a header that does not bind the manifest or the +external config.json it is checked against, any field with the wrong +type, range, or key set at any of the four record kinds (header, item +header, target row, terminal completion), and a target row whose fields +contradict one another in a way the producer can never emit. + +Checks enumerated from the source (`tools/check_ablate_evidence.py`, +read in full before writing this module) and covered below, grouped by +the function that performs them: + +- `_checked_engine_text_size` / `_bounded_config_bytes`: the 256 MiB + inclusive engine text limit, both sides. +- `_reject_constant`, `_reject_duplicate_keys`, `_json_record`: no + NaN/Infinity JSON constants, no duplicate object keys, invalid + JSON/non-ASCII text rejected. +- `_config_identity`: empty file; invalid JSON; non-object root; each + of vocab_size/num_hidden_layers/n_routed_experts/first_k_dense_replace + missing or out of range. +- `_manifest_proof`: framing -- an empty manifest, an empty record, a + carriage return inside a record and an embedded NUL are refused, while + CRLF endings and a missing final newline are accepted and reduced to + the canonical form the engine binds; non-ASCII line; non-canonical integer grammar; too few fields; + every per-item field bound (item id, T, prompt, mode, cell count); + the mode/cell-count pairing rule; the field-count/denominator + arithmetic; every per-cell bound (layer, expert, applied-target, + mode-3 vs other-mode applied-target rule, duplicate cell); + out-of-vocabulary tokens; duplicate item ids across lines. +- `validate_ablate_evidence`: the evidence framing check; the header's + key set, type, and range checks; the header-vs-config identity + check; the header-vs-manifest-proof binding check; the item header's + key set, type, and manifest-order check; the target row's key set, + type, and identity checks; the three cross-field invariants (below); + the top-k list's shape, range, and + uniqueness checks; the terminal record's key set, bounds, and exact + content check; missing/extra/trailing records at every boundary. + +Every fixture here is a literal artifact built by hand from the +module's documented wire schema (`coli-ablate/2`) and hashed with the +stdlib `hashlib` directly -- no expected value is produced by calling +the validator under test. +""" +import copy +import hashlib +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from tools import check_ablate_evidence as ABLATE +from tools import engine_evidence + +_DOMAIN = b"coli-ablate-manifest/2\n" + + +def _serialize(records): + return b"".join( + json.dumps(record, separators=(",", ":")).encode("ascii") + b"\n" + for record in records) + + +def _write(root, manifest_raw, evidence_raw, config_raw): + manifest = root / "manifest.txt" + evidence = root / "evidence.jsonl" + config = root / "config.json" + manifest.write_bytes(manifest_raw) + evidence.write_bytes(evidence_raw) + config.write_bytes(config_raw) + return manifest, evidence, config + + +def _run_cli(manifest, evidence, config): + return subprocess.run( + [sys.executable, str(pathlib.Path(ABLATE.__file__)), + str(manifest), str(evidence), "--config", str(config)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + + +class _GoldenFixture(unittest.TestCase): + """Shared two-item artifact, hand-derived from the documented schema. + + Manifest: item 1 (T=3, prompt=1, baseline, tokens 0,1,2) then item 2 + (T=2, prompt=1, mode 1 with one ablated cell at layer 1/expert 2, + tokens 3,0). vocab=4, n_layers=4, first_dense=1, n_experts=5, so + topk = min(32, 4) = 4. Positions/gold are derived by hand from the + manifest's own tokens: item 1 has positions [0, 1] with gold tokens + 1 and 2; item 2 has position [0] with gold token 0. + """ + + MANIFEST = b"1 3 1 0 0 0 1 2\n2 2 1 1 1 1 2 -1 3 0\n" + CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest() + CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest() + + @classmethod + def golden_records(cls): + header = { + "t": "hdr", "schema": "coli-ablate/2", "vocab": 4, "topk": 4, + "n_layers": 4, "first_dense": 1, "n_experts": 5, + "config_sha256": cls.CONFIG_SHA256, + "manifest_sha256": cls.MANIFEST_SHA256, + "expected_items": 2, "expected_targets": 3, + } + item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0, + "T": 3, "n_prompt": 1, "cells": []} + row1 = {"t": "lg", "item": 1, "pos": 0, "gold": 1, + "nll": 0.2, "glogit": 1.0, "molo": 0.5, "mgn": 0.5, + "am": 1, "amlogit": 1.0, "logZ": 1.3, "corr": 1, + "tk": [[0, 0.1], [1, 1.0], [2, 0.3], [3, -0.2]]} + row2 = {"t": "lg", "item": 1, "pos": 1, "gold": 2, + "nll": 0.7, "glogit": 0.4, "molo": 0.9, "mgn": -0.5, + "am": 0, "amlogit": 0.9, "logZ": 1.1, "corr": 0, + "tk": [[0, 0.9], [1, 0.1], [2, 0.4], [3, -0.3]]} + item2_header = {"t": "ah", "item": 2, "mode": 1, "ncells": 1, + "T": 2, "n_prompt": 1, "cells": [[1, 2, -1]]} + row3 = {"t": "lg", "item": 2, "pos": 0, "gold": 0, + "nll": 0.0, "glogit": 2.0, "molo": -1e30, "mgn": 1e30, + "am": 0, "amlogit": 2.0, "logZ": 2.0, "corr": 1, + "tk": [[0, 2.0], [1, -1.0], [2, -2.0], [3, -3.0]]} + done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256, + "completed_items": 2, "completed_targets": 3} + return [header, item1_header, row1, row2, item2_header, row3, done] + + def _reject(self, mutate, records=None): + """Apply `mutate` to a deep copy of the golden records and assert + the mutated artifact is refused.""" + mutated = copy.deepcopy(records if records is not None + else self.golden_records()) + mutate(mutated) + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(mutated), self.CONFIG) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + +class GoldenArtifactAcceptedTests(_GoldenFixture): + def test_valid_artifact_is_accepted_and_pass_line_is_exact(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(self.golden_records()), + self.CONFIG) + result = ABLATE.validate_ablate_evidence( + manifest, evidence, config) + self.assertEqual(result, { + "manifest_sha256": self.MANIFEST_SHA256, + "items": 2, "targets": 3, + }) + cli = _run_cli(manifest, evidence, config) + self.assertEqual(cli.returncode, 0, cli.stderr.decode(errors="replace")) + self.assertEqual(cli.stderr, b"") + # print() terminates the line with the platform's newline, so a + # Windows child hands back CRLF; the pin is on the line's content. + self.assertEqual( + cli.stdout.replace(b"\r\n", b"\n"), + f"[ablate-evidence] PASS manifest={self.MANIFEST_SHA256} " + f"items=2 targets=3\n".encode("ascii")) + + +class TopkProducerCapAboveVocabFourTests(unittest.TestCase): + """The header `topk == min(32, vocab)` check only ever exercises the + "vocab is the binding constraint" side at `_GoldenFixture`'s vocab=4. + This fixture uses vocab=40 (above both 4 and the 32 cap) to pin the + other side: topk must be capped at 32, not left equal to vocab. + """ + + MANIFEST = b"1 2 1 0 0 0 39\n" + CONFIG = (b'{"vocab_size":40,"num_hidden_layers":1,' + b'"first_k_dense_replace":0,"n_routed_experts":1}\n') + MANIFEST_SHA256 = hashlib.sha256(_DOMAIN + MANIFEST).hexdigest() + CONFIG_SHA256 = hashlib.sha256(CONFIG).hexdigest() + + @classmethod + def golden_records(cls, topk=32, tk_count=32): + header = { + "t": "hdr", "schema": "coli-ablate/2", "vocab": 40, "topk": topk, + "n_layers": 1, "first_dense": 0, "n_experts": 1, + "config_sha256": cls.CONFIG_SHA256, + "manifest_sha256": cls.MANIFEST_SHA256, + "expected_items": 1, "expected_targets": 1, + } + item1_header = {"t": "ah", "item": 1, "mode": 0, "ncells": 0, + "T": 2, "n_prompt": 1, "cells": []} + row = {"t": "lg", "item": 1, "pos": 0, "gold": 39, + "nll": 0.0, "glogit": 1.0, "molo": 0.5, "mgn": 0.5, + "am": 39, "amlogit": 1.0, "logZ": 1.3, "corr": 1, + "tk": [[i, -0.01 * i] for i in range(tk_count)]} + done = {"t": "done", "manifest_sha256": cls.MANIFEST_SHA256, + "completed_items": 1, "completed_targets": 1} + return [header, item1_header, row, done] + + def test_topk_capped_at_32_for_vocab_above_the_cap_is_accepted(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(self.golden_records()), + self.CONFIG) + result = ABLATE.validate_ablate_evidence( + manifest, evidence, config) + self.assertEqual(result, { + "manifest_sha256": self.MANIFEST_SHA256, + "items": 1, "targets": 1, + }) + + def test_topk_left_uncapped_at_vocab_above_32_is_rejected(self): + # vocab=40 > 32, so header topk must be 32 -- not 40 (== vocab). + records = self.golden_records(topk=40, tk_count=40) + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(records), self.CONFIG) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + +class EvidenceFramingTests(_GoldenFixture): + """`validate_ablate_evidence`'s canonical-LF-JSONL framing check.""" + + def _reject_raw(self, evidence_raw): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + manifest, evidence, config = _write( + root, self.MANIFEST, evidence_raw, self.CONFIG) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + def test_empty_evidence_rejected(self): + self._reject_raw(b"") + + def test_evidence_missing_trailing_newline_rejected(self): + self._reject_raw(_serialize(self.golden_records())[:-1]) + + def test_evidence_with_carriage_return_rejected(self): + self._reject_raw(_serialize(self.golden_records()).replace( + b"\n", b"\r\n", 1)) + + def test_evidence_with_nul_byte_rejected(self): + self._reject_raw(_serialize(self.golden_records()) + b"\0") + + +class HeaderRecordTests(_GoldenFixture): + CASES = ( + ("missing_key", lambda r: r[0].pop("topk")), + ("extra_key", lambda r: r[0].__setitem__("extra", 1)), + ("wrong_t", lambda r: r[0].__setitem__("t", "nope")), + ("wrong_schema", lambda r: r[0].__setitem__( + "schema", "coli-ablate/1")), + ("topk_wrong_type", lambda r: r[0].__setitem__("topk", "4")), + ("config_sha256_wrong_type", lambda r: r[0].__setitem__( + "config_sha256", 1)), + ("config_sha256_not_hex", lambda r: r[0].__setitem__( + "config_sha256", "z" * 64)), + ("manifest_sha256_wrong_type", lambda r: r[0].__setitem__( + "manifest_sha256", None)), + ("manifest_sha256_not_hex", lambda r: r[0].__setitem__( + "manifest_sha256", "0" * 63 + "g")), + ("vocab_out_of_range", lambda r: r[0].__setitem__("vocab", 0)), + ("n_layers_out_of_range", lambda r: r[0].__setitem__( + "n_layers", 0)), + ("first_dense_out_of_range", lambda r: r[0].__setitem__( + "first_dense", 99)), + ("n_experts_out_of_range", lambda r: r[0].__setitem__( + "n_experts", 0)), + ("expected_items_out_of_range", lambda r: r[0].__setitem__( + "expected_items", 0)), + ("expected_targets_out_of_range", lambda r: r[0].__setitem__( + "expected_targets", 0)), + ("topk_not_producer_exact", lambda r: r[0].__setitem__("topk", 3)), + ("vocab_identity_mismatch", lambda r: ( + r[0].__setitem__("vocab", 5), r[0].__setitem__("topk", 5))), + ("n_layers_identity_mismatch", lambda r: r[0].__setitem__( + "n_layers", 2)), + ("first_dense_identity_mismatch", lambda r: r[0].__setitem__( + "first_dense", 0)), + ("n_experts_identity_mismatch", lambda r: r[0].__setitem__( + "n_experts", 6)), + ("config_sha256_identity_mismatch", lambda r: r[0].__setitem__( + "config_sha256", "0" * 64)), + ("manifest_sha256_binding_mismatch", lambda r: r[0].__setitem__( + "manifest_sha256", "1" * 64)), + ("expected_items_binding_mismatch", lambda r: r[0].__setitem__( + "expected_items", 99)), + ("expected_targets_binding_mismatch", lambda r: r[0].__setitem__( + "expected_targets", 99)), + ) + + def test_header_field_checks(self): + for name, mutate in self.CASES: + with self.subTest(name=name): + self._reject(mutate) + + def test_topk_not_producer_exact_even_when_every_row_agrees_with_it(self): + # Isolates the header-level topk==min(32,vocab) check from the + # per-row "len(tk) == header['topk']" shape check: here every row's + # tk list is ALSO shrunk to match the wrong topk, so only the + # header-level producer-exactness check can catch the artifact. + def mutate(records): + records[0]["topk"] = 2 + for record in records: + if record.get("t") == "lg": + record["tk"] = record["tk"][:2] + self._reject(mutate) + + +class ItemHeaderRecordTests(_GoldenFixture): + def test_missing_item_header_rejected(self): + self._reject(lambda r: r.__delitem__(slice(1, None))) + + def test_item_header_not_a_dict_rejected(self): + self._reject(lambda r: r.__setitem__(1, 5)) + + def test_item_header_missing_key_rejected(self): + self._reject(lambda r: r[1].pop("ncells")) + + def test_item_header_extra_key_rejected(self): + self._reject(lambda r: r[1].__setitem__("extra", 1)) + + def test_item_header_field_wrong_type_rejected(self): + self._reject(lambda r: r[1].__setitem__("item", "1")) + + def test_item_header_cells_not_a_list_rejected(self): + self._reject(lambda r: r[4].__setitem__("cells", {})) + + def test_item_header_cell_wrong_shape_rejected(self): + self._reject(lambda r: r[4].__setitem__("cells", [[1, 2]])) + + def test_item_header_cell_element_wrong_type_rejected(self): + self._reject(lambda r: r[4].__setitem__( + "cells", [[1, 2, "x"]])) + + def test_item_header_mismatch_vs_manifest_rejected(self): + self._reject(lambda r: r[1].__setitem__("T", 99)) + + +class TargetRowRecordTests(_GoldenFixture): + def test_missing_target_row_rejected(self): + self._reject(lambda r: r.__delitem__(slice(2, None))) + + def test_row_not_a_dict_rejected(self): + self._reject(lambda r: r.__setitem__(2, 5)) + + def test_row_missing_key_rejected(self): + self._reject(lambda r: r[2].pop("corr")) + + def test_row_extra_key_rejected(self): + self._reject(lambda r: r[2].__setitem__("extra", 1)) + + def test_row_wrong_t_rejected(self): + self._reject(lambda r: r[2].__setitem__("t", "nope")) + + def test_row_item_mismatch_rejected(self): + self._reject(lambda r: r[2].__setitem__("item", 99)) + + def test_row_pos_mismatch_rejected(self): + self._reject(lambda r: r[2].__setitem__("pos", 5)) + + def test_row_gold_mismatch_rejected(self): + self._reject(lambda r: r[2].__setitem__("gold", 0)) + + def test_row_am_wrong_type_rejected(self): + self._reject(lambda r: r[2].__setitem__("am", "1")) + + def test_row_am_out_of_range_rejected(self): + self._reject(lambda r: r[2].__setitem__("am", 4)) + + def test_row_corr_wrong_type_rejected(self): + self._reject(lambda r: r[2].__setitem__("corr", "1")) + + def test_row_corr_out_of_range_rejected(self): + self._reject(lambda r: r[2].__setitem__("corr", 2)) + + NUMERIC_FIELDS = ("nll", "glogit", "molo", "mgn", "amlogit", "logZ") + + def test_row_numeric_field_wrong_type_rejected(self): + for field in self.NUMERIC_FIELDS: + with self.subTest(field=field): + self._reject(lambda r, field=field: r[2].__setitem__( + field, "0")) + + def test_row_tk_not_a_list_rejected(self): + self._reject(lambda r: r[2].__setitem__("tk", 5)) + + def test_row_tk_wrong_length_rejected(self): + self._reject(lambda r: r[2].__setitem__( + "tk", r[2]["tk"][:-1])) + + def test_row_tk_pair_wrong_shape_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, 0.1, 9])) + + def test_row_tk_pair_id_wrong_type_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, ["0", 0.1])) + + def test_row_tk_pair_id_out_of_range_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, [4, 0.1])) + + def test_row_tk_pair_val_wrong_type_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, [0, "0.1"])) + + def test_row_tk_duplicate_ids_rejected(self): + self._reject(lambda r: r[2]["tk"].__setitem__(0, list(r[2]["tk"][1]))) + + +class CrossFieldInvariantTests(_GoldenFixture): + """Invariants the engine's per-record emitter, `ablate_logit_record` + (and the row-writer `ablate_logit_line` it calls), guarantees for + every row it emits. + + - `nll >= 0`: `nll` is `-target_lp` (`ablate_logit_record`'s own + `gnll=-target_lp`), and `target_lp` is `delta - logse` where + `delta = lo[target] - r.max <= 0` (target's logit minus the row + max) and `logse = log(sum_i exp(lo[i]-max)) >= log(1) = 0` (the + max's own term contributes exp(0)=1 to that sum) -- the row-level + helpers this emitter builds on (`logprob_row_checked`/ + `logprob_from_row_checked`). So `target_lp <= 0` always, hence + `nll >= 0` always. + - `corr == (am == gold)`: the emitter passes an `argmax==gold` + comparison directly as the `corr` argument to `ablate_logit_line`, + and `am` is that same argmax -- `corr` is never anything but that + comparison's result. + - `amlogit >= glogit`: `amlogit` is the row's own maximum logit and + `glogit` is one particular entry of that same row, so it can + never exceed the row's own maximum. + + Top-k ordering is deliberately NOT enforced: `tk` is unsorted on the + wire by design (`logit_topk_select`, documented as "deliberately not + a lowest-token-id tie rule"). + """ + + def test_nll_negative_rejected(self): + self._reject(lambda r: r[2].__setitem__("nll", -0.1)) + + def test_nll_zero_accepted(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + records = self.golden_records() + records[2]["nll"] = 0.0 + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(records), self.CONFIG) + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + def test_corr_true_when_am_not_gold_rejected(self): + # am=1 == gold=1 in the golden row, so corr must be 1; forcing 0 + # while leaving am/gold untouched breaks the agreement. + self._reject(lambda r: r[2].__setitem__("corr", 0)) + + def test_corr_false_when_am_equals_gold_rejected(self): + # am=0 != gold=2 in row2 (index 3), so corr must be 0; forcing 1 + # breaks the agreement the other way. + self._reject(lambda r: r[3].__setitem__("corr", 1)) + + def test_amlogit_below_glogit_rejected(self): + self._reject(lambda r: r[2].__setitem__("amlogit", 0.5)) + + def test_amlogit_equal_to_glogit_accepted(self): + # row1 already has amlogit == glogit == 1.0 (am == gold there); + # confirm the boundary itself -- not just values strictly above + # it -- is accepted. + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + records = self.golden_records() + self.assertEqual(records[2]["amlogit"], records[2]["glogit"]) + manifest, evidence, config = _write( + root, self.MANIFEST, _serialize(records), self.CONFIG) + ABLATE.validate_ablate_evidence(manifest, evidence, config) + + +class TerminalRecordTests(_GoldenFixture): + def test_missing_terminal_record_rejected(self): + self._reject(lambda r: r.__delitem__(slice(6, None))) + + def test_terminal_not_a_dict_rejected(self): + self._reject(lambda r: r.__setitem__(6, 5)) + + def test_terminal_missing_key_rejected(self): + self._reject(lambda r: r[6].pop("completed_items")) + + def test_terminal_extra_key_rejected(self): + self._reject(lambda r: r[6].__setitem__("extra", 1)) + + def test_terminal_completed_items_out_of_range_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_items", 0)) + + def test_terminal_completed_targets_out_of_range_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_targets", 0)) + + def test_terminal_wrong_t_rejected(self): + self._reject(lambda r: r[6].__setitem__("t", "nope")) + + def test_terminal_manifest_sha256_mismatch_rejected(self): + self._reject(lambda r: r[6].__setitem__("manifest_sha256", "1" * 64)) + + def test_terminal_completed_items_mismatch_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_items", 1)) + + def test_terminal_completed_targets_mismatch_rejected(self): + self._reject(lambda r: r[6].__setitem__("completed_targets", 1)) + + def test_trailing_record_after_terminal_rejected(self): + self._reject(lambda r: r.append(dict(r[6]))) + + +class TruncationReplayAndMismatchBiteTests(_GoldenFixture): + """Bite-style table close to the source's own producer-invariant + checks, rebuilt on this module's literal golden fixture instead of + an engine-produced one.""" + + def test_named_mutations_all_refuse(self): + cases = ( + ("missing_done", lambda r: r.__delitem__(6)), + ("truncated_last_row", lambda r: r.__setitem__( + 5, {"t": "lg", "item": 2})), + ("replayed_item_header", lambda r: r.insert(2, dict(r[1]))), + ("duplicate_done", lambda r: r.append(dict(r[6]))), + ("missing_target_row", lambda r: r.__delitem__(3)), + ("wrong_gold_downstream", lambda r: r[3].__setitem__( + "gold", 0)), + ("header_digest_forged", lambda r: r[0].__setitem__( + "manifest_sha256", "2" * 64)), + ) + for name, mutate in cases: + with self.subTest(name=name): + self._reject(mutate) + + +class ManifestProofFramingTests(unittest.TestCase): + """`_manifest_proof`'s non-canonical-text framing checks.""" + + ARGS = (4, 4, 1, 5) # vocab, n_layers, first_dense, n_experts + + def _reject(self, raw): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._manifest_proof(raw, *self.ARGS) + + def test_empty_manifest_rejected(self): + self._reject(b"") + + def _accept(self, raw): + return ABLATE._manifest_proof(raw, *self.ARGS) + + # The engine accepts a manifest saved with CRLF endings and one whose last + # line has no terminator, and digests the canonical form of either. A + # checker that refused them would reject files the producer really ran. + def test_manifest_missing_trailing_newline_accepted(self): + self._accept(b"1 3 1 0 0 0 1 2") + + def test_manifest_with_crlf_endings_accepted(self): + self._accept(b"1 3 1 0 0 0 1 2\r\n") + + def test_all_three_framings_bind_the_same_digest(self): + canonical = self._accept(b"1 3 1 0 0 0 1 2\n")["sha256"] + self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2\r\n")["sha256"], canonical) + self.assertEqual(self._accept(b"1 3 1 0 0 0 1 2")["sha256"], canonical) + + def test_carriage_return_inside_a_record_rejected(self): + self._reject(b"1 3 1 0\r 0 0 1 2\n") + + def test_empty_line_inside_a_manifest_rejected(self): + self._reject(b"1 3 1 0 0 0 1 2\n\n2 2 1 0 0 0 1\n") + + def test_manifest_with_nul_byte_rejected(self): + self._reject(b"1 3 1 0 0 0 1 2\n\0") + + def test_manifest_non_ascii_line_rejected(self): + self._reject("1 3 1 0 0 0 1 é\n".encode("utf-8")) + + def test_manifest_control_byte_breaks_grammar_not_framing(self): + # A vertical tab embedded mid-line is not a canonical digit/space + # byte; it must be caught by the integer-grammar check, not + # silently absorbed as a line boundary bytes.splitlines() would + # not treat it as one either way (see the dedicated probe below). + self._reject(b"1 3\x0b1 0 0 0 1 2\n") + + def test_duplicate_item_id_across_lines_rejected(self): + self._reject(b"1 2 1 0 0 0 1\n1 2 1 0 0 0 1\n") + + def test_records_are_split_on_newline_only(self): + # The module now splits the canonical form on b"\n" rather than + # calling splitlines(), so no other byte can become a record + # boundary. bytes.splitlines() would additionally break on \r, + # which the canonical form no longer contains but which a future + # edit could reintroduce; splitting explicitly removes the + # question. These bytes must therefore stay inside one record and + # be caught by the integer-grammar check. + for value in (0x0B, 0x0C, 0x1C, 0x1D, 0x1E): + with self.subTest(byte=hex(value)): + raw = b"1 3 1 0 0 0 1" + bytes([value]) + b"2\n" + self._reject(raw) + + +class CanonicalManifestDigestTests(unittest.TestCase): + """The canonical rule, pinned against the engine by a literal digest. + + `c/tests/test_ablate_mode.c` asserts the same 64 characters for the same + manifest content. Two implementations that each only agreed with + themselves would both pass their own suites while disagreeing in the + field; a literal known answer on both sides is what rules that out. + """ + + RECORD = b"0 3 2 0 0 1 2 3\n" + KNOWN = "c63a48c375b14ca60f26c7e3c5dd36b5929ffaf669a45511c93deee6e8bbd5ed" + + def test_known_answer_matches_the_engine(self): + digest = hashlib.sha256( + ABLATE.DOMAIN + engine_evidence.canonical_manifest_bytes(self.RECORD) + ).hexdigest() + self.assertEqual(digest, self.KNOWN) + + def test_every_accepted_framing_reaches_the_known_answer(self): + for raw in (self.RECORD, b"0 3 2 0 0 1 2 3\r\n", b"0 3 2 0 0 1 2 3"): + with self.subTest(raw=raw): + digest = hashlib.sha256( + ABLATE.DOMAIN + + engine_evidence.canonical_manifest_bytes(raw)).hexdigest() + self.assertEqual(digest, self.KNOWN) + + def test_canonicalization_refuses_what_the_engine_refuses(self): + for raw in (b"", b"\n", b"a\n\nb\n", b"a\rb\n", b"a\0b\n"): + with self.subTest(raw=raw): + with self.assertRaises(engine_evidence.ManifestFormError): + engine_evidence.canonical_manifest_bytes(raw) + + +class ManifestProofFieldBoundaryTests(unittest.TestCase): + """Per-field/per-cell/per-token bounds `_manifest_proof` enforces. + + Table built from `test_manifest_fixed_width_and_topology_c_python_parity`'s + Python-side expectations (each `expected` value here is the same + literal that method asserted, not something this module computed): + that method also cross-checked each case against a C test binary + this module does not build, so it is not this module's oracle to + carry (flagged separately, not absorbed here). + """ + + def test_boundary_table(self): + i32 = ABLATE._INT32_MAX + i64 = ABLATE._INT64_MAX + sixteen = " ".join(f"{layer} 0 -1" for layer in range(1, 17)) + cases = ( + ("baseline_min", b"0 2 1 0 0 0 1\n", 4, 4, 1, 8, True), + ("fewer_than_five_fields", b"1 2\n", 4, 4, 1, 8, False), + ("item_max", f"{i64} 2 1 0 0 0 1\n".encode(), + 4, 4, 1, 8, True), + ("item_max_plus_1", f"{i64 + 1} 2 1 0 0 0 1\n".encode(), + 4, 4, 1, 8, False), + ("item_min_minus_1", b"-1 2 1 0 0 0 1\n", 4, 4, 1, 8, False), + ("T_min", b"7 2 1 0 0 0 1\n", 4, 4, 1, 8, True), + ("T_below_min", b"7 1 1 0 0 0\n", 4, 4, 1, 8, False), + ("T_max_incomplete", f"7 {i32} 1 0 0\n".encode(), + 4, 4, 1, 8, False), + ("T_max_plus_1", f"7 {i32 + 1} 1 0 0\n".encode(), + 4, 4, 1, 8, False), + ("prompt_max_incomplete", f"7 {i32} {i32 - 1} 0 0\n".encode(), + 4, 4, 1, 8, False), + ("prompt_max_plus_1", f"7 {i32} {i32 + 1} 0 0\n".encode(), + 4, 4, 1, 8, False), + ("prompt_min_minus_1", b"7 2 0 0 0 0 1\n", + 4, 4, 1, 8, False), + ("mode_max", b"7 2 1 3 1 1 2 3 0 1\n", + 4, 4, 1, 8, True), + ("mode_max_plus_1", b"7 2 1 4 1 1 2 -1 0 1\n", + 4, 4, 1, 8, False), + ("cells_max", f"7 2 1 1 16 {sixteen} 0 1\n".encode(), + 4, 17, 1, 8, True), + ("cells_max_plus_1", b"7 2 1 1 17 0 1\n", + 4, 17, 1, 8, False), + ("nonbaseline_zero", b"7 2 1 1 0 0 1\n", + 4, 4, 1, 8, False), + ("dense_layer", b"7 2 1 1 1 0 2 -1 0 1\n", + 4, 4, 1, 8, False), + ("layer_min", b"7 2 1 1 1 0 2 -1 0 1\n", + 4, 4, 0, 8, True), + ("layer_upper", b"7 2 1 1 1 3 2 -1 0 1\n", + 4, 4, 1, 8, True), + ("layer_engine_max", b"7 2 1 1 1 127 2 -1 0 1\n", + 4, 128, 0, 8, True), + ("layer_engine_max_plus_1", b"7 2 1 1 1 128 2 -1 0 1\n", + 4, 128, 0, 8, False), + ("source_upper", b"7 2 1 1 1 1 7 -1 0 1\n", + 4, 4, 1, 8, True), + ("source_min", b"7 2 1 1 1 1 0 -1 0 1\n", + 4, 4, 1, 8, True), + ("source_engine_max", b"7 2 1 1 1 1 4095 -1 0 1\n", + 4, 4, 1, 4096, True), + ("source_engine_max_plus_1", b"7 2 1 1 1 1 4096 -1 0 1\n", + 4, 4, 1, 4096, False), + ("target_upper", b"7 2 1 3 1 1 2 7 0 1\n", + 4, 4, 1, 8, True), + ("target_self_swap", b"7 2 1 3 1 1 2 2 0 1\n", + 4, 4, 1, 8, False), + ("target_signed_min", + f"7 2 1 3 1 1 2 {ABLATE._INT32_MIN} 0 1\n".encode(), + 4, 4, 1, 8, False), + ("target_engine_max", b"7 2 1 3 1 1 0 4095 0 1\n", + 4, 4, 1, 4096, True), + ("target_engine_max_plus_1", b"7 2 1 3 1 1 0 4096 0 1\n", + 4, 4, 1, 4096, False), + ("target_max_plus_1", + f"7 2 1 3 1 1 2 {i32 + 1} 0 1\n".encode(), + 4, 4, 1, 8, False), + ("duplicate_source", b"7 2 1 1 2 1 2 -1 1 2 -1 0 1\n", + 4, 4, 1, 8, False), + ("token_min", b"7 2 1 0 0 0 0\n", 1, 4, 1, 8, True), + ("token_upper", b"7 2 1 0 0 0 16777215\n", + 1 << 24, 4, 1, 8, True), + ("token_max_plus_1", b"7 2 1 0 0 0 16777216\n", + 1 << 24, 4, 1, 8, False), + ("vocab_max", b"7 2 1 0 0 0 1\n", + 1 << 24, 4, 1, 8, True), + ("vocab_max_plus_1", b"7 2 1 0 0 0 1\n", + (1 << 24) + 1, 4, 1, 8, False), + ("leading_zero_rejected", b"07 2 1 0 0 0 1\n", + 4, 4, 1, 8, False), + ("plus_sign_rejected", b"+7 2 1 0 0 0 1\n", + 4, 4, 1, 8, False), + ("double_space_rejected", b"7 2 1 0 0 0 1\n", + 4, 4, 1, 8, False), + ("trailing_space_rejected", b"7 2 1 0 0 0 1 \n", + 4, 4, 1, 8, False), + ) + for (name, raw, vocab, layers, first_dense, experts, + expected) in cases: + with self.subTest(name=name): + if expected: + ABLATE._manifest_proof( + raw, vocab, layers, first_dense, experts) + else: + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._manifest_proof( + raw, vocab, layers, first_dense, experts) + + +class ConfigIdentityTests(unittest.TestCase): + CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + + def test_engine_text_size_rejects_non_int_length(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._checked_engine_text_size(True, "config") + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._checked_engine_text_size(1.0, "config") + + def test_engine_byte_limit_is_inclusive_and_enforced_both_sides(self): + engine_limit = 256 << 20 + self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit) + self.assertEqual( + ABLATE._checked_engine_text_size(engine_limit, "config"), + engine_limit) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._checked_engine_text_size(engine_limit + 1, "config") + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + ablate_config = root / "ablate-config.json" + ablate_config.write_bytes(self.CONFIG) + with mock.patch.object( + ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)): + identity = ABLATE._config_identity(ablate_config) + self.assertEqual(identity["vocab"], 4) + self.assertEqual( + identity["config_sha256"], + hashlib.sha256(self.CONFIG).hexdigest()) + ablate_config.write_bytes(self.CONFIG + b" ") + with self.assertRaisesRegex( + ABLATE.AblateEvidenceError, "256 MiB"): + ABLATE._config_identity(ablate_config) + + def _reject(self, raw): + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / "config.json" + path.write_bytes(raw) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._config_identity(path) + + def test_empty_config_rejected(self): + self._reject(b"") + + def test_invalid_json_config_rejected(self): + self._reject(b"{not json}\n") + + def test_non_object_root_rejected(self): + self._reject(b"[1,2,3]\n") + + def test_vocab_size_missing_rejected(self): + self._reject(b'{"num_hidden_layers":4,"first_k_dense_replace":1,' + b'"n_routed_experts":5}\n') + + def test_vocab_size_out_of_range_rejected(self): + self._reject(b'{"vocab_size":0,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + + def test_num_hidden_layers_out_of_range_rejected(self): + self._reject(b'{"vocab_size":4,"num_hidden_layers":0,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + + def test_n_routed_experts_out_of_range_rejected(self): + self._reject(b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":0}\n') + + def test_first_k_dense_replace_out_of_range_rejected(self): + self._reject(b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":5,"n_routed_experts":5}\n') + + +class JsonHelperTests(unittest.TestCase): + def test_duplicate_json_key_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"a":1,"a":2}', "record 1") + + def test_nan_constant_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"nll":NaN}', "record 1") + + def test_infinity_constant_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"nll":Infinity}', "record 1") + + def test_negative_infinity_constant_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b'{"nll":-Infinity}', "record 1") + + def test_non_ascii_bytes_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b"\xff", "record 1") + + def test_malformed_json_rejected(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._json_record(b"{not json}", "record 1") + + +class FixedWidthHelperTests(unittest.TestCase): + def test_bounded_int_rejects_bool_disguised_as_int(self): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._bounded_int(True, "x", 1, 10) + + def test_int64_max_is_the_literal_signed_64_bit_bound(self): + # Pinned by literal, not derived, so a future refactor of the + # module's own (1 << 63) - 1 expression cannot silently drift. + self.assertEqual(ABLATE._INT64_MAX, 9223372036854775807) + + def test_fixed_width_helpers_and_derived_count_boundaries(self): + for value in (ABLATE._INT64_MIN, ABLATE._INT64_MAX): + self.assertEqual(ABLATE._manifest_i64(str(value), 1), value) + for value in (ABLATE._INT64_MIN - 1, ABLATE._INT64_MAX + 1): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._manifest_i64(str(value), 1) + for value in (ABLATE._INT32_MIN, ABLATE._INT32_MAX): + self.assertEqual(ABLATE._bounded_int( + value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX), value) + for value in (ABLATE._INT32_MIN - 1, ABLATE._INT32_MAX + 1): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._bounded_int( + value, "int32", ABLATE._INT32_MIN, ABLATE._INT32_MAX) + self.assertEqual( + ABLATE._count_add(0, ABLATE._INT64_MAX, "count"), + ABLATE._INT64_MAX) + self.assertEqual( + ABLATE._count_add(ABLATE._INT64_MAX, 0, "count"), + ABLATE._INT64_MAX) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._count_add(ABLATE._INT64_MAX, 1, "count") + for label in ("expected_items", "expected_targets", + "completed_items", "completed_targets"): + self.assertEqual( + ABLATE._bounded_int(1, label, 1, ABLATE._INT64_MAX), 1) + self.assertEqual(ABLATE._bounded_int( + ABLATE._INT64_MAX, label, 1, ABLATE._INT64_MAX), + ABLATE._INT64_MAX) + for value in (0, ABLATE._INT64_MAX + 1): + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._bounded_int(value, label, 1, ABLATE._INT64_MAX) + + def test_retained_long_max_plus_one_artifact_is_incomplete(self): + manifest_raw = b"9223372036854775808 2 1 0 0 0 1\n" + digest = hashlib.sha256(_DOMAIN + manifest_raw).hexdigest() + self.assertEqual( + digest, + "988a1cf2ddc812f38138e51eecfebb2ba0c9980e31b4c7183396716f114d6538") + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + config_raw = (b'{"vocab_size":2,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":8}\n') + records = ( + {"t": "hdr", "schema": "coli-ablate/2", "vocab": 2, + "topk": 2, "n_layers": 4, "first_dense": 1, + "n_experts": 8, + "config_sha256": hashlib.sha256(config_raw).hexdigest(), + "manifest_sha256": digest, + "expected_items": 1, "expected_targets": 1}, + {"t": "ah", "item": 9223372036854775808, "mode": 0, + "ncells": 0, "T": 2, "n_prompt": 1, "cells": []}, + {"t": "lg", "item": 9223372036854775808, "pos": 0, + "gold": 1, "nll": 0, "glogit": 0, "molo": 0, + "mgn": 0, "am": 1, "amlogit": 0, "logZ": 0, + "corr": 1, "tk": [[1, 0], [0, -1]]}, + {"t": "done", "manifest_sha256": digest, + "completed_items": 1, "completed_targets": 1}, + ) + manifest, evidence, config = _write( + root, manifest_raw, _serialize(records), config_raw) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE.validate_ablate_evidence(manifest, evidence, config) + cli = _run_cli(manifest, evidence, config) + self.assertNotEqual(cli.returncode, 0) + self.assertIn(b"[ablate-evidence] INCOMPLETE:", cli.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_check_data_logprob_gaps.py b/c/tests/test_check_data_logprob_gaps.py new file mode 100644 index 000000000..b210cecef --- /dev/null +++ b/c/tests/test_check_data_logprob_gaps.py @@ -0,0 +1,964 @@ +"""tests/check_data_logprob_gaps.py must accept only a complete, well-formed +raw engine-stdout transcript for one opted-in request and reject every +other input: an unparsed or mismatched startup preamble, a legacy 3-field +DATA frame hiding a dropped logprob channel anywhere in the generation, a +numeric tail with the wrong top-k count or an out-of-range/duplicate token +id, malformed framing, a missing or duplicated ACCEPT/DONE, and every other +gap the module's docstring claims to catch. + +Checks enumerated from the source (`check_data_logprob_gaps.py`, read in +full before writing this module) and covered below: + +- `_uint`, `_c17g`, `_fixed_metric`: the noncanonical-ASCII-integer grammar, + the exact finite `%.17g` spelling, and the fixed-decimal-place grammar + with its inclusive bounds. +- `_header_fields` / `_global_header`: non-printable-ASCII and stray-space + rejection on an ordinary protocol header, and the exact field grammar + for every recognized mux-global record (BANNER/LOADED preamble, READY, + STAT, HWINFO, TIERS, EMAP, HITS, PROF). +- `parse_frames`: byte-exact DATA/ECHO payload framing, so a single + malformed frame cannot desynchronize the parse. +- `capture_mode`: the four explicit transcript shapes it names. +- `_numeric_tail` / `check`: every check enumerated in the module + docstring, including the legacy 3-field DATA gap this checker exists to + catch, the `min(topk, vocab)` top-k cap, and out-of-range/duplicate + token ids. + +Every fixture here is a literal transcript built by hand from the module's +documented wire grammar -- no expected value is produced by calling the +checker under test. The transcript-shaped fixtures (`_GapFixture`'s BANNER/ +LOADED/VALID/echo/transcript/telemetry/full_capture/complete_capture +helpers) and the `GapCheckerEvidenceTests` methods that exercise them are +carried over unchanged from the checker's own evidence-consumer suite, +excluding the one method that requires a compiled fixture binary. +""" +import collections +import importlib.util +import pathlib +import random +import subprocess +import sys +import tempfile +import unittest + +_HERE = pathlib.Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location( + "check_data_logprob_gaps", _HERE / "check_data_logprob_gaps.py") +GAPS = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(GAPS) + + +class _GapFixture(unittest.TestCase): + BANNER = ( + b"== GLM C engine (glm_moe_dsa), cache=64 experts/layer | " + b"compute experts@4-bit dense@8-bit | idot: neon-i8mm ==\n") + LOADED = ( + b"loaded in 1.00s | resident dense: 1.00 MB | " + b"layers=78 experts=256 | MTP ACTIVE (draft=1)\n") + VALID = ( + b"ACCEPT 7 1\n" + b"ECHO 7 1 0 nan 0\n" + b"x\n" + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\n" + b"x\n" + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n" + ) + + @staticmethod + def echo(pos): + if pos == 0: + return b"ECHO 7 1 0 nan 0\nx\n" + return (f"ECHO 7 1 {pos} -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\n" + "x\n").encode("ascii") + + @classmethod + def transcript(cls, prompt=1, positions=None, emitted=1, done_prompt=None, + tps="0.10", hit="100.0", rss="10.00", flag=0): + if positions is None: + positions = list(range(prompt)) + if done_prompt is None: + done_prompt = prompt + parts = [f"ACCEPT 7 {prompt}\n".encode("ascii")] + parts.extend(cls.echo(pos) for pos in positions) + parts.extend([ + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n", + (f"DONE 7 STAT {emitted} {tps} {hit} {rss} " + f"{done_prompt} {flag}\n").encode("ascii"), + ]) + return b"".join(parts) + + def problems(self, blob, topk=2, vocab=3): + frames, framing = GAPS.parse_frames(blob) + data, echo, problems, _mode, _forms = GAPS.check( + frames, framing, "7", topk, vocab) + return data, echo, problems + + def full_problems(self, blob, topk=2, vocab=3): + """Like problems(), but also returns capture_mode and the numeric + form(s) observed -- for tests that need those two new signals.""" + frames, framing = GAPS.parse_frames(blob) + return GAPS.check(frames, framing, "7", topk, vocab) + + @staticmethod + def telemetry(cpu=b"AMD Ryzen 9", cores=b"16"): + return b"".join(( + b"HWINFO " + cores + b" 128.0 64.0 2 48.0 " + cpu + + b"|CUDA device x2\n", + b"TIERS 128 256 1024 48.00 32.50\n", + b"EMAP 2 2 00010203\n", + b"HITS 2 2 0f\n", + b"PROF 1.250 1 1 0.100 0.200 0.300 0.400 0.500 7\n", + )) + + @classmethod + def full_capture(cls): + startup = ( + b"\x01\x01READY\x01\x01\n" + b"STAT 0 0.00 0.0 10.00\n" + b"HWINFO 16 128.0 64.0 2 48.0 AMD Ryzen 9|CUDA device x2\n" + b"TIERS 128 256 1024 48.00 32.50\n" + b"EMAP 2 2 00010203\n" + ) + other = ( + b"ACCEPT 8 1\n" + b"ECHO 8 1 0 nan 0\nq\n" + b"DATA 8 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nq\n" + + cls.telemetry(cpu=b"AMD Ryzen 9") + + b"DONE 8 STAT 1 0.10 100.0 10.00 1 0\n" + ) + target = ( + b"ACCEPT 7 1\n" + b"ECHO 7 1 0 nan 0\nx\n" + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n" + + cls.telemetry(cpu=b"AMD Ryzen 9") + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n" + ) + return startup + other + target + + @classmethod + def complete_capture(cls, loaded=None): + return cls.BANNER + (loaded if loaded is not None else cls.LOADED) + cls.full_capture() + + +class GapCheckerEvidenceTests(_GapFixture): + """Ported unchanged from the checker's own evidence-consumer suite, + excluding the one method that scores a compiled fixture binary's + `%.17g` corpus (needs a build, not available here).""" + + def test_complete_request_passes(self): + for prompt in (1, 3): + with self.subTest(prompt=prompt): + data, echo, problems = self.problems(self.transcript(prompt)) + self.assertEqual((data, echo), (1, prompt)) + self.assertEqual(problems, []) + + def test_full_raw_mux_capture_passes_without_filtering(self): + data, echo, problems = self.problems(self.full_capture()) + self.assertEqual((data, echo), (1, 1)) + self.assertEqual(problems, []) + frames, _ = GAPS.parse_frames(self.full_capture()) + self.assertEqual(GAPS.capture_mode(frames), "ready-suffix") + + def test_complete_process_capture_passes_without_filtering(self): + data, echo, problems = self.problems(self.complete_capture()) + self.assertEqual((data, echo), (1, 1)) + self.assertEqual(problems, []) + frames, _ = GAPS.parse_frames(self.complete_capture()) + self.assertEqual(GAPS.capture_mode(frames), "full-process") + + for state, draft in ((b"ACTIVE", 0), (b"ACTIVE", 1), + (b"absent", 0), (b"absent", 2), + (b"DISABLED (multiplexed serve)", 0)): + loaded = ( + b"loaded in 1.00s | resident dense: 1.00 MB | " + b"layers=78 experts=256 | MTP " + state + + b" (draft=" + str(draft).encode("ascii") + b")\n") + _, _, problems = self.problems(self.complete_capture(loaded)) + self.assertEqual(problems, [], (state, draft, problems)) + + def test_complete_process_preamble_lifecycle_is_exact(self): + base = self.complete_capture() + suffix = self.full_capture() + cases = ( + suffix, + base.replace(self.BANNER, b"", 1), + base.replace(self.LOADED, b"", 1), + self.LOADED + self.BANNER + suffix, + self.BANNER + self.BANNER + self.LOADED + suffix, + self.BANNER + self.LOADED + self.LOADED + suffix, + b"unknown preamble\n" + base, + base.replace(b"idot: neon-i8mm", b"idot: fabricated", 1), + base.replace(b"loaded in 1.00s", b"loaded in 1.0s", 1), + ) + # The first case is the separately supported READY-suffix mode. + self.assertEqual(self.problems(cases[0])[2], []) + for blob in cases[1:]: + with self.subTest(blob=blob[:180]): + _, _, problems = self.problems(blob) + self.assertTrue(problems) + + def test_load_state_ranges_and_metrics_are_exact_in_full_capture(self): + base = self.LOADED + overflow = b"9" * 400 + b".00" + cases = ( + base.replace(b"ACTIVE (draft=1)", + b"DISABLED (multiplexed serve) (draft=1)"), + base.replace(b"ACTIVE", b"fabricated"), + base.replace(b"draft=1", b"draft=64"), + base.replace(b"layers=78", b"layers=0"), + base.replace(b"layers=78", b"layers=129"), + base.replace(b"experts=256", b"experts=0"), + base.replace(b"experts=256", b"experts=4097"), + base.replace(b"loaded in 1.00s", b"loaded in -0.01s"), + base.replace(b"loaded in 1.00s", b"loaded in nan s"), + base.replace(b"loaded in 1.00s", b"loaded in infs"), + base.replace(b"loaded in 1.00s", b"loaded in " + overflow + b"s"), + base.replace(b"loaded in 1.00s", b"loaded in 1.0s"), + base.replace(b"resident dense: 1.00 MB", + b"resident dense: -0.01 MB"), + base.replace(b"resident dense: 1.00 MB", + b"resident dense: nan MB"), + base.replace(b"resident dense: 1.00 MB", + b"resident dense: " + overflow + b" MB"), + base.replace(b"resident dense: 1.00 MB", + b"resident dense: 1.0 MB"), + ) + for loaded in cases: + with self.subTest(loaded=loaded): + _, _, problems = self.problems(self.complete_capture(loaded)) + self.assertTrue(any("malformed LOADED" in p for p in problems), + problems) + + for layers, experts in ((1, 1), (128, 4096)): + loaded = base.replace(b"layers=78", f"layers={layers}".encode()) + loaded = loaded.replace(b"experts=256", f"experts={experts}".encode()) + self.assertEqual(self.problems(self.complete_capture(loaded))[2], []) + + def test_emap_and_hits_producer_domains_are_exact(self): + for byte in (b"00", b"20", b"40", b"60", b"80", b"a0"): + _, problems = GAPS._global_header(b"EMAP 1 1 " + byte, 0) + self.assertEqual(problems, [], byte) + for byte in (b"21", b"61", b"a1"): + _, problems = GAPS._global_header(b"EMAP 1 1 " + byte, 0) + self.assertTrue(any("heat" in p for p in problems), problems) + for byte in (b"c0", b"ff"): + _, problems = GAPS._global_header(b"EMAP 1 1 " + byte, 0) + self.assertTrue(any("tier" in p for p in problems), problems) + + _, problems = GAPS._global_header(b"EMAP 0 2 ", 0) + self.assertEqual(problems, []) + _, problems = GAPS._global_header(b"EMAP 0 2 00", 0) + self.assertTrue(any("payload length" in p for p in problems), problems) + + for line in (b"HITS 2 2 0f", b"HITS 1 8 ff"): + _, problems = GAPS._global_header(line, 0) + self.assertEqual(problems, [], line) + for line in (b"HITS 2 2 1f", b"HITS 2 2 ff"): + _, problems = GAPS._global_header(line, 0) + self.assertTrue(any("padding" in p for p in problems), problems) + + def test_global_numeric_fields_never_collide_with_request_id(self): + capture = self.full_capture().replace( + b"HWINFO 16 128.0", b"HWINFO 7 128.0") + capture = capture.replace( + b"TIERS 128 256 1024", b"TIERS 7 256 1024") + capture = capture.replace(b"EMAP 2 2 00010203", b"EMAP 7 1 00010203040506") + capture = capture.replace(b"HITS 2 2 0f", b"HITS 7 1 7f") + data, echo, problems = self.problems(capture) + self.assertEqual((data, echo), (1, 1)) + self.assertEqual(problems, []) + + def test_malformed_or_misordered_global_records_never_pass(self): + base = self.full_capture() + cases = ( + base.replace(b"\x01\x01READY\x01\x01", b"READY", 1), + base.replace(b"STAT 0 0.00 0.0 10.00", b"STAT 0 0.0 0.0 10.00", 1), + base.replace(b"HWINFO 16 128.0", b"HWINFO 016 128.0", 1), + base.replace(b"HWINFO 16 128.0", b"HWINFO 16 128.0", 1), + base.replace(b"TIERS 128 256", b"TIERS 1_28 256", 1), + base.replace(b"EMAP 2 2 00010203", b"EMAP 2 2 0001020F", 1), + base.replace(b"HITS 2 2 0f", b"HITS 2 2 000f", 1), + base.replace(b"PROF 1.250 1 1", b"PROF 1.25 1 1", 1), + base.replace(b"PROF 1.250 1 1", b"PROF 1.250 01 1", 1), + base.replace(b"PROF 1.250 1 1", b"PROF 1.250 1 1_0", 1), + base.replace(b"\x01\x01READY\x01\x01\n", b"", 1), + base.replace(b"STAT 0 0.00 0.0 10.00\n", b"", 1), + base.replace( + b"\x01\x01READY\x01\x01\nSTAT 0 0.00 0.0 10.00\n", + b"STAT 0 0.00 0.0 10.00\n\x01\x01READY\x01\x01\n", 1), + base.replace( + b"\x01\x01READY\x01\x01\n", + b"\x01\x01READY\x01\x01\n\x01\x01READY\x01\x01\n", 1), + base.replace( + b"STAT 0 0.00 0.0 10.00\n", + b"STAT 0 0.00 0.0 10.00\nSTAT 0 0.00 0.0 10.00\n", 1), + ) + for blob in cases: + with self.subTest(blob=blob[:100]): + _, _, problems = self.problems(blob) + self.assertTrue(problems) + + def test_request_id_zero_refuses_before_record_matching(self): + frames, framing = GAPS.parse_frames(self.full_capture()) + data, echo, problems, mode, forms = GAPS.check( + frames, framing, "0", 2, 3) + self.assertEqual((data, echo), (0, 0)) + self.assertEqual(problems, ["requested id must be positive"]) + self.assertEqual(forms, frozenset()) + + with tempfile.TemporaryDirectory() as tmp: + capture = pathlib.Path(tmp) / "capture.raw" + capture.write_bytes(self.full_capture()) + proc = subprocess.run( + [sys.executable, str(pathlib.Path(GAPS.__file__)), str(capture), + "--id", "0", "--topk", "2", "--vocab", "3"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + self.assertEqual(proc.returncode, 2) + self.assertIn(b"requested id must be positive", proc.stderr) + + def test_uint_primitive_and_coordinated_underscore_bite(self): + for token, value in ((b"0", 0), (b"1", 1), (b"10", 10), + (b"2147483647", 2147483647)): + with self.subTest(token=token): + self.assertEqual(GAPS._uint(token, "fixture"), value) + for token in (b"+1", b"01", b"1_0", b" 1", b"1 ", b"1\t0", + "١".encode("utf-8")): + with self.subTest(token=token): + with self.assertRaises(ValueError): + GAPS._uint(token, "fixture") + + parts = [b"ACCEPT 7 1_0\n"] + parts.extend(self.echo(pos) for pos in range(10)) + parts.extend(( + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n", + b"DONE 7 STAT 1 0.10 100.0 10.00 1_0 0\n", + )) + data, echo, problems = self.problems(b"".join(parts)) + self.assertEqual((data, echo), (1, 10)) + self.assertTrue(any("invalid ACCEPT prompt length" in p for p in problems)) + self.assertTrue(any("malformed DONE stats" in p for p in problems)) + self.assertFalse(any("ECHO positions" in p or "DONE prompt count" in p + for p in problems), problems) + + def test_malformed_framing_never_passes(self): + bad = [ + b"ACCEPT 7 1\nDATA 7 nope -2.7000000000000002 0\n", + b"ACCEPT 7 1\nDATA 7 5 -2.7000000000000002 0\nx\n", + b"ACCEPT 7 1\nDATA 7 1 -2.7000000000000002 0\nx", + b"ACCEPT 7 1", # missing header newline + ] + for blob in bad: + with self.subTest(blob=blob): + _, framing_problems = GAPS.parse_frames(blob) + self.assertTrue(framing_problems) + + def test_invalid_topk_tables_never_pass(self): + headers = [ + b"DATA 7 1 -2.7000000000000002 2 -1 -2.7000000000000002 1 -11.123455999999999", # out-of-range id + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 0 -11.123455999999999", # duplicate id + b"DATA 7 1 0.125 2 0 -2.7000000000000002 1 -11.123455999999999", # positive target lp + b"DATA 7 1 -2.7000000000000002 2 0 0.125 1 -11.123455999999999", # positive top-k lp + b"DATA 7 1 nan 2 0 -2.7000000000000002 1 -11.123455999999999", # pending nonfinite policy + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 3 -11.123455999999999", # id == vocab + ] + for header in headers: + blob = (b"ACCEPT 7 1\n" + self.echo(0) + header + b"\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + with self.subTest(header=header): + _, _, problems = self.problems(blob) + self.assertTrue(problems) + + def test_missing_lifecycle_or_partial_denominator_never_passes(self): + no_accept = self.VALID.split(b"\n", 1)[1] + no_done = self.VALID.rsplit(b"DONE", 1)[0] + wrong_done = self.VALID.replace(b"DONE 7 STAT 1", b"DONE 7 STAT 2") + error = self.VALID.replace( + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0", b"ERROR 7 ENGINE") + cases = ( + (no_accept, "expected exactly one ACCEPT"), + (no_done, "expected exactly one DONE"), + (wrong_done, "DONE emitted 2 != observed DATA 1"), + (error, "target request returned ERROR"), + ) + for blob, expected in cases: + with self.subTest(blob=blob): + _, _, problems = self.problems(blob) + self.assertTrue(any(expected in problem for problem in problems), problems) + + def test_unknown_targeted_kind_and_echo_after_data_never_pass(self): + unknown = self.VALID.replace( + b"DONE 7 STAT", b"MYSTERY 7 value\nDONE 7 STAT") + echo_after = ( + b"ACCEPT 7 1\n" + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\n" + b"x\n" + b"ECHO 7 1 0 nan 0\n" + b"x\n" + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n" + ) + for blob in (unknown, echo_after): + with self.subTest(blob=blob): + _, _, problems = self.problems(blob) + self.assertTrue(problems) + + def test_echo_denominator_and_order_are_exact(self): + cases = ([], [0], [0, 1], [0, 2], [0, 1, 1], [1, 0, 2], [0, 1, 2, 3]) + for positions in cases: + with self.subTest(positions=positions): + _, _, problems = self.problems(self.transcript(3, positions)) + self.assertTrue(any("ECHO positions" in p for p in problems), problems) + + def test_done_denominator_and_prompt_joins_bite_independently(self): + zero_data = self.VALID.replace( + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n", b"").replace( + b"DONE 7 STAT 1", b"DONE 7 STAT 0") + data, _, problems = self.problems(zero_data) + self.assertEqual(data, 0) + self.assertEqual( + [p for p in problems if p.startswith("no DATA frames")], + ["no DATA frames found for this id -- wrong id, or the run produced nothing"]) + _, _, problems = self.problems(self.transcript(1, emitted=2)) + self.assertEqual([p for p in problems if p.startswith("DONE emitted")], + ["DONE emitted 2 != observed DATA 1"]) + _, _, problems = self.problems(self.transcript(1, done_prompt=2)) + self.assertEqual([p for p in problems if p.startswith("DONE prompt")], + ["DONE prompt count 2 != ACCEPT 1"]) + + def test_done_domains_and_fixed_grammar_bite_independently(self): + cases = ( + {"tps": "-0.10"}, {"hit": "-0.1"}, {"hit": "100.1"}, + {"rss": "-0.01"}, {"tps": "0.1"}, {"hit": "0.00"}, + {"rss": "10.0"}, {"tps": "+0.10"}, {"tps": "01.00"}, + {"tps": "1e+00"}, {"tps": "nan"}, {"tps": "inf"}, + {"done_prompt": 0}, {"flag": 2}, + ) + for kwargs in cases: + with self.subTest(kwargs=kwargs): + _, _, problems = self.problems(self.transcript(1, **kwargs)) + matches = [p for p in problems if p.startswith("malformed DONE stats")] + self.assertEqual(len(matches), 1, problems) + + _, _, problems = self.problems(self.transcript( + 1, tps="-0.00", hit="-0.0", rss="-0.00")) + self.assertEqual(problems, []) + + def test_noncanonical_ascii_numeric_grammar_never_passes(self): + base = self.VALID + cases = ( + base.replace(b"ACCEPT 7 1", b"ACCEPT 7 +1"), + base.replace(b"ACCEPT 7 1", b"ACCEPT 7 1_0"), + base.replace(b"ECHO 7 1 0", b"ECHO 7 1 +0"), + base.replace(b"DATA 7 1", b"DATA 7 +1"), + base.replace(b" -2.7000000000000002 2 ", b" -0_125 2 ", 1), + base.replace(b" 1 -11.123455999999999", b" 01 -11.123455999999999"), + base.replace(b"DATA 7 1 ", b"DATA 7 1 "), + base.replace(b"DATA 7 1 ", b"DATA\t7 1 "), + base.replace(b"1 -11.123455999999999\nx\n", b"1 -11.123455999999999 \nx\n"), + base.replace(b"-2.7000000000000002 2", b"-1e-9999 2", 1), + base.replace(b"-2.7000000000000002 2", b"-1.00000000000000000 2", 1), + base.replace(b"-2.7000000000000002 2", b"-1e-9 2", 1), + base.replace(b"-2.7000000000000002 2", b"-1e--09 2", 1), + base.replace(b"DONE 7 STAT 1", b"DONE 7 STAT 01"), + base.replace(b" 1 0\n", b" +1 0\n"), + base.replace(b" 1 0\n", b" 1 00\n"), + base.replace(b"DATA 7 1", "DATA 7 ١".encode("utf-8")), + base.replace(b"DATA 7 1 -2.7000000000000002", b"DATA 7 1 -2.7000000000000002junk"), + ) + for blob in cases: + with self.subTest(blob=blob): + _, _, problems = self.problems(blob) + self.assertTrue(problems) + + +class PreambleGateFailsLoudTests(_GapFixture): + """The preamble gate never silently passes an unparsed BANNER/LOADED + line: `parse_engine_preamble` raising `PreambleError`, or the module's + own defensive `is None` branch, both surface as a named problem that + quotes the offending line.""" + + def test_unparsed_banner_is_a_named_failure_quoting_the_line(self): + bad_banner = self.BANNER.replace(b"idot: neon-i8mm", b"idot: bogus", 1) + kind, problems = GAPS._global_header(bad_banner.rstrip(b"\n"), 0) + self.assertEqual(kind, [b"BANNER"]) + self.assertEqual(len(problems), 1) + self.assertIn("malformed BANNER preamble", problems[0]) + self.assertIn(repr(bad_banner.rstrip(b"\n")), problems[0]) + + def test_unparsed_loaded_is_a_named_failure_quoting_the_line(self): + bad_loaded = self.LOADED.replace(b"MTP ACTIVE", b"MTP fabricated", 1) + kind, problems = GAPS._global_header(bad_loaded.rstrip(b"\n"), 0) + self.assertEqual(kind, [b"LOADED"]) + self.assertEqual(len(problems), 1) + self.assertIn("malformed LOADED preamble", problems[0]) + self.assertIn(repr(bad_loaded.rstrip(b"\n")), problems[0]) + + def test_preamble_returning_none_is_defensively_named_not_silently_passed(self): + # `_global_header` treats a None `parse_engine_preamble` result the + # same as a raised PreambleError: it is unreachable through the + # real BANNER/LOADED prefixes (they always parse or raise), so this + # pins the defensive branch directly by forcing that return value. + original = GAPS.parse_engine_preamble + GAPS.parse_engine_preamble = lambda text: None + try: + kind, problems = GAPS._global_header( + self.BANNER.rstrip(b"\n"), 0) + finally: + GAPS.parse_engine_preamble = original + self.assertEqual(kind, [b"BANNER"]) + self.assertEqual(len(problems), 1) + self.assertIn("malformed BANNER preamble", problems[0]) + self.assertIn(repr(self.BANNER.rstrip(b"\n")), problems[0]) + + def test_full_capture_with_unparsed_preamble_fails_the_whole_check(self): + bogus = self.complete_capture().replace( + b"idot: neon-i8mm", b"idot: bogus", 1) + _, _, problems = self.problems(bogus) + self.assertTrue(any("malformed BANNER preamble" in p for p in problems), + problems) + + +class LegacyDataFrameGapTests(_GapFixture): + """A legacy 3-field DATA frame anywhere in the generation is the exact + gap this checker exists to catch, and it must fail loudly even when + it is not the last generated token.""" + + def test_legacy_three_field_data_frame_mid_generation_fails(self): + blob = ( + b"ACCEPT 7 1\n" + b"ECHO 7 1 0 nan 0\nx\n" + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n" # token 1: full channel + b"DATA 7 1\nx\n" # token 2: legacy gap + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n" # token 3: full channel + b"DONE 7 STAT 3 0.10 100.0 10.00 1 0\n" + ) + data, echo, problems = self.problems(blob) + self.assertEqual(data, 3) + self.assertTrue(any( + "GAP: legacy 3-field DATA frame #2" in p for p in problems), + problems) + + def test_wholly_dropped_data_frame_mid_generation_fails_via_done_count(self): + # The severer form of the same defect class: one generated token's + # DATA frame never reached stdout at all (not even degraded to 3 + # fields). Only the DONE-emitted-vs-observed-DATA cross-check can + # name this -- a checker with no concept of a DONE frame would + # pass it silently. + blob = ( + b"ACCEPT 7 1\n" + b"ECHO 7 1 0 nan 0\nx\n" + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n" # token 1 + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 1 -11.123455999999999\nx\n" # token 3 (token 2 missing) + b"DONE 7 STAT 3 0.10 100.0 10.00 1 0\n" + ) + data, echo, problems = self.problems(blob) + self.assertEqual(data, 2) + self.assertIn("DONE emitted 3 != observed DATA 2", problems) + + +class TopkCapAndTokenIdBoundsTests(_GapFixture): + """The numeric tail enforces k == min(--topk, --vocab) and + 0 <= tid < vocab, reporting the first offending frame.""" + + def test_k_above_expected_but_within_wire_cap_never_passes(self): + # vocab=40, --topk=20 => expected_k = min(20, 40) = 20; the frame + # advertises k=25, which is <=32 (the wire's own hard cap) but + # above the expected value for this request. + pairs = " ".join(f"{i} -2.7000000000000002" for i in range(25)) + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + f"DATA 7 1 -2.7000000000000002 25 {pairs}\n".encode("ascii") + b"x\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + _, _, problems = self.problems(blob, topk=20, vocab=40) + self.assertTrue(any( + "DATA top-k 25 != expected 20" in p for p in problems), problems) + + def test_k_above_wire_hard_cap_of_32_never_passes(self): + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 -2.7000000000000002 33 0 -0.1\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + _, _, problems = self.problems(blob, topk=32, vocab=40) + self.assertTrue(any( + "malformed DATA numeric fields" in p for p in problems), problems) + + def test_token_id_equal_to_vocab_never_passes(self): + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 -2.7000000000000002 2 0 -2.7000000000000002 3 -0.25\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + _, _, problems = self.problems(blob, topk=2, vocab=3) + self.assertTrue(any( + "token id 3 outside [0,3)" in p for p in problems), problems) + + def test_token_id_well_above_vocab_never_passes(self): + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 -2.7000000000000002 1 999 -2.7000000000000002\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + _, _, problems = self.problems(blob, topk=1, vocab=40) + self.assertTrue(any( + "token id 999 outside [0,40)" in p for p in problems), problems) + + +class CaptureModeLiteralTests(unittest.TestCase): + """`capture_mode` names the four explicit transcript shapes.""" + + def test_empty_transcript_is_request_only(self): + self.assertEqual(GAPS.capture_mode([]), "request-only") + + def test_accept_first_frame_is_request_only(self): + frames, _ = GAPS.parse_frames(b"ACCEPT 7 1\n") + self.assertEqual(GAPS.capture_mode(frames), "request-only") + + def test_banner_first_frame_is_full_process(self): + frames, _ = GAPS.parse_frames( + b"== GLM C engine (glm_moe_dsa), cache=64 experts/layer | " + b"compute experts@4-bit dense@8-bit | idot: neon-i8mm ==\n" + b"ACCEPT 7 1\n") + self.assertEqual(GAPS.capture_mode(frames), "full-process") + + def test_ready_first_frame_is_ready_suffix(self): + frames, _ = GAPS.parse_frames( + b"\x01\x01READY\x01\x01\nACCEPT 7 1\n") + self.assertEqual(GAPS.capture_mode(frames), "ready-suffix") + + def test_global_after_a_non_global_first_frame_is_invalid(self): + frames, _ = GAPS.parse_frames( + b"ACCEPT 7 1\n\x01\x01READY\x01\x01\n") + self.assertEqual(GAPS.capture_mode(frames), "invalid") + + +class FixedMetricAndUintBoundaryTests(unittest.TestCase): + """`_fixed_metric`'s lower/upper bounds and `_uint`'s maximum + argument are literal, inclusive boundaries.""" + + def test_fixed_metric_lower_bound_is_inclusive(self): + self.assertEqual(GAPS._fixed_metric(b"0.0", 1, "x", lower=0.0), 0.0) + with self.assertRaises(ValueError): + GAPS._fixed_metric(b"-0.1", 1, "x", lower=0.0) + + def test_fixed_metric_upper_bound_is_inclusive(self): + self.assertEqual( + GAPS._fixed_metric(b"100.0", 1, "x", upper=100.0), 100.0) + with self.assertRaises(ValueError): + GAPS._fixed_metric(b"100.1", 1, "x", upper=100.0) + + def test_fixed_metric_place_count_is_exact(self): + with self.assertRaises(ValueError): + GAPS._fixed_metric(b"1.00", 1, "x") + with self.assertRaises(ValueError): + GAPS._fixed_metric(b"1.0", 2, "x") + + def test_uint_maximum_is_inclusive(self): + self.assertEqual(GAPS._uint(b"32", "x", 32), 32) + with self.assertRaises(ValueError): + GAPS._uint(b"33", "x", 32) + + + + +class NumericGrammarFormTests(_GapFixture): + """The numeric grammar accepts both the fixed %.6f form dev's engine + prints today and the earlier %.17g form + (plus an exact nan/inf/-inf spelling), pinned with non-dyadic literal + values (-0.3, -2.7, -11.123456) so the two encodings are actually + distinguishable by their token spelling and not merely by the float + each happens to parse to.""" + + def test_fixed6_form_is_accepted_and_reported(self): + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 -0.300000 2 0 -2.700000 1 -11.123456\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + data, echo, problems, mode, forms = self.full_problems(blob) + self.assertEqual(problems, []) + self.assertEqual(forms, collections.Counter({"fixed6": 3})) + + def test_c17g_form_is_accepted_and_reported(self): + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 -2.7000000000000002 2 0 -11.123455999999999 " + b"1 -0.29999999999999999\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + data, echo, problems, mode, forms = self.full_problems(blob) + self.assertEqual(problems, []) + self.assertEqual(forms, collections.Counter({"c17g": 3})) + + def test_a_token_matching_both_grammars_is_ambiguous_and_still_accepted(self): + # '%.17g' % -1.234567 == '-1.234567', which is ALSO the exact + # six-decimal spelling of that same double: there is no way to + # tell, from the token alone, which engine format produced it. + # An earlier version of this check treated a tail containing both + # an unambiguous fixed6 token and an unambiguous c17g token as a + # "mixed forms" error; that rule was removed because per-token + # classification is inherently ambiguous and the rule produced + # false rejections on real %.17g transcripts (some of whose + # tokens are, coincidentally, exact six-decimal spellings too). + # Nothing about mixing forms is rejected anymore -- only a token + # matching NEITHER grammar is. + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 -1.234567 2 0 -2.7000000000000002 1 -0.300000\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + data, echo, problems, mode, forms = self.full_problems(blob) + self.assertEqual(problems, []) + self.assertEqual(forms, collections.Counter( + {"ambiguous": 1, "c17g": 1, "fixed6": 1})) + + def test_special_nan_inf_tokens_are_syntactically_valid_then_flagged(self): + # "nan"/"inf"/"-inf" are exact libc renderings either format's + # snprintf can emit for a non-finite double: syntactically valid + # under both grammars, but still fail the finite/non-positive + # semantic check that applies regardless of which form carried it. + for special in (b"nan", b"inf", b"-inf"): + with self.subTest(special=special): + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 " + special + b" 1 0 -0.300000\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + _, _, problems, _, forms = self.full_problems(blob, topk=1) + self.assertTrue(any( + "target logprob is not finite/non-positive" in p + for p in problems), problems) + self.assertIn("special", forms) + + def test_token_matching_neither_form_never_passes(self): + blob = ( + b"ACCEPT 7 1\n" + self.echo(0) + + b"DATA 7 1 -0.3 2 0 -2.7 1 -11.123456\nx\n" + + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + _, _, problems = self.problems(blob) + self.assertTrue(any( + "malformed DATA numeric fields" in p for p in problems), problems) + + +class ByteOffsetTests(unittest.TestCase): + """Every reported problem cites the byte offset of + the OFFENDING frame's own header line, not the frame that follows it, + and numeric-tail problems (previously offset-less) now carry one too. + """ + + def test_malformed_global_offset_cites_its_own_line_not_the_next_one(self): + first = b"\x01\x01READY\x01\x01\n" + bad_stat = b"STAT 0 0.0 0.0 10.00\n" # "0.0" should be "0.00" + blob = first + bad_stat + b"ACCEPT 7 1\n" + _, problems = GAPS.parse_frames(blob) + self.assertEqual(len(problems), 1) + self.assertIn(f"byte {len(first)}", problems[0]) + self.assertNotIn(f"byte {len(first) + len(bad_stat)}", problems[0]) + + def test_header_fields_offset_cites_its_own_line_not_the_next_one(self): + first = b"ACCEPT 7 1\n" + bad = b" ACCEPT 8 1\n" # leading space: noncanonical header + blob = first + bad + _, problems = GAPS.parse_frames(blob) + self.assertEqual(len(problems), 1) + self.assertIn(f"byte {len(first)}", problems[0]) + self.assertNotIn(f"byte {len(first) + len(bad)}", problems[0]) + + def test_numeric_tail_problem_cites_the_data_frames_own_header(self): + prefix = b"ACCEPT 7 1\n" + b"ECHO 7 1 0 nan 0\nx\n" + # positive target lp: a semantic failure inside _numeric_tail. + bad_data = b"DATA 7 1 0.300000 2 0 -0.300000 1 -0.300000\n" + blob = prefix + bad_data + b"x\n" + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n" + frames, framing = GAPS.parse_frames(blob) + _, _, problems, _, _ = GAPS.check(frames, framing, "7", 2, 3) + offset = len(prefix) + self.assertTrue(any( + f"byte {offset}" in p and + "target logprob is not finite/non-positive" in p + for p in problems), problems) + + def test_offset_of_a_non_first_offending_frame_is_hand_verified(self): + # Three frames precede the offending one: ACCEPT (11 bytes: the + # 10-character line "ACCEPT 7 1" plus its newline), an ECHO at + # position 0 (17 bytes: "ECHO 7 1 0 nan 0" plus its newline), and + # that ECHO's one-byte "x" payload plus its terminator (2 bytes). + # 11 + 17 + 2 = 30 is where the malformed DONE header starts -- + # a hand count, not a value read back from the module under test. + line0 = b"ACCEPT 7 1\n" + line1 = b"ECHO 7 1 0 nan 0\n" + payload1 = b"x\n" + self.assertEqual(len(line0), 11) + self.assertEqual(len(line1), 17) + self.assertEqual(len(payload1), 2) + bad_done = b"DONE 7 STAT 1 0.10 100.0 10.00 1\n" # missing 1 field + blob = line0 + line1 + payload1 + bad_done + frames, framing = GAPS.parse_frames(blob) + self.assertEqual(len(frames), 3) + self.assertEqual(frames[2][2], 30) + _, _, problems, _, _ = GAPS.check(frames, framing, "7", 2, 3) + self.assertTrue(any( + "malformed DONE frame at byte 30" in p for p in problems), problems) + + +class PreambleGateWiredIntoMainTests(unittest.TestCase): + """capture_mode() is actually wired into check(), so + a transcript opening with an unrecognized (garbage) line fails through + main()'s own CLI path, not only via a function called directly; and + the resolved capture mode is always reported, never silent.""" + + def test_garbage_preamble_fails_via_main_cli_path(self): + blob = ( + b"GARBAGE NOT A REAL PREAMBLE\n" + b"ACCEPT 7 1\n" + b"ECHO 7 1 0 nan 0\nx\n" + b"DATA 7 1 -0.300000 2 0 -0.300000 1 -0.300000\nx\n" + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n") + with tempfile.TemporaryDirectory() as tmp: + capture = pathlib.Path(tmp) / "capture.raw" + capture.write_bytes(blob) + proc = subprocess.run( + [sys.executable, str(pathlib.Path(GAPS.__file__)), str(capture), + "--id", "7", "--topk", "2", "--vocab", "3"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + self.assertEqual(proc.returncode, 1) + self.assertIn(b"unrecognized frame opens the transcript", proc.stdout) + + def test_capture_mode_and_numeric_form_are_reported_in_main_summary(self): + with tempfile.TemporaryDirectory() as tmp: + capture = pathlib.Path(tmp) / "capture.raw" + capture.write_bytes(_GapFixture.VALID) + proc = subprocess.run( + [sys.executable, str(pathlib.Path(GAPS.__file__)), str(capture), + "--id", "7", "--topk", "2", "--vocab", "3"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + self.assertEqual(proc.returncode, 0) + self.assertIn(b"capture_mode=request-only", proc.stdout) + self.assertIn(b"numeric form tally: c17g=3", proc.stdout) + + def test_invalid_capture_mode_is_named_via_check(self): + # A stray global record with no BANNER/READY lead frame at all. + blob = b"STAT 0 0.00 0.0 10.00\nACCEPT 7 1\n" + frames, framing = GAPS.parse_frames(blob) + self.assertEqual(GAPS.capture_mode(frames), "invalid") + _, _, problems, mode, _ = GAPS.check(frames, framing, "7", 2, 3) + self.assertEqual(mode, "invalid") + self.assertTrue(any("invalid capture mode" in p for p in problems), problems) + + + +class EngineFormatCorpusTests(_GapFixture): + """A synthesized corpus in the engine's real wire format: every + numeric token below is spelled exactly as C's `snprintf(..., "%.6f + %d", ...)` (dev's shipped `logprob_tail()`) would spell it -- not + hand-picked round numbers, but 210 distinct, non-dyadic values, so no + single lucky literal is doing the proving. It must pass at head, and + every one of its numeric tokens is independently confirmed to be + something the strict %.17g-only grammar this checker's numeric + parsing predates would have rejected outright. + """ + + N_FRAMES = 210 + VOCAB = 300 + TOPK = 2 + + @classmethod + def _corpus(cls, seed=20260904): + rng = random.Random(seed) + parts = [b"ACCEPT 7 1\n", b"ECHO 7 1 0 nan 0\nx\n"] + tokens = [] + for _ in range(cls.N_FRAMES): + target_lp = -abs(rng.uniform(1e-6, 20.0)) + target_token = f"{target_lp:.6f}".encode("ascii") + tokens.append(target_token) + pair_ids = rng.sample(range(cls.VOCAB), cls.TOPK) + pieces = [target_token, str(cls.TOPK).encode("ascii")] + for tid in pair_ids: + token_lp = -abs(rng.uniform(1e-6, 20.0)) + token_lp_token = f"{token_lp:.6f}".encode("ascii") + tokens.append(token_lp_token) + pieces.append(str(tid).encode("ascii")) + pieces.append(token_lp_token) + parts.append(b"DATA 7 1 " + b" ".join(pieces) + b"\nx\n") + parts.append( + f"DONE 7 STAT {cls.N_FRAMES} 0.10 100.0 10.00 1 0\n".encode("ascii")) + return b"".join(parts), tokens + + def test_engine_format_corpus_of_210_frames_passes(self): + blob, tokens = self._corpus() + self.assertGreaterEqual(len(tokens), self.N_FRAMES) + frames, framing = GAPS.parse_frames(blob) + data, echo, problems, mode, forms = GAPS.check( + frames, framing, "7", self.TOPK, self.VOCAB) + self.assertEqual(problems, []) + self.assertEqual(data, self.N_FRAMES) + # Every token here is a genuine %.6f emission, so none can be + # classified as c17g-only; some nonetheless land on an exact + # %.17g spelling too (the false-positive "mixed forms" case a + # since-removed rejection used to misfire on) and are tallied + # "ambiguous" rather than "fixed6" -- informational only, and + # this test's PASS/problems assertions above already prove that + # tally has no bearing on the verdict. + self.assertEqual(forms["c17g"], 0) + self.assertGreater(forms["ambiguous"], 0) + self.assertEqual(sum(forms.values()), len(tokens)) + + def test_engine_format_corpus_values_are_non_dyadic(self): + # A dyadic value (an exact binary fraction, e.g. 0.125) would make + # the fixed6/c17g distinction uninteresting for that token, since + # both forms could spell it exactly. Confirm the corpus avoids + # that by construction: none of its %.6f tokens round-trips + # through Python's own dyadic-fraction check. + _, tokens = self._corpus() + dyadic = 0 + for token in tokens: + value = float(token) + # A double is dyadic (exactly binary-fraction-representable at + # 6 decimal places) iff multiplying by 1e6 and rounding loses + # nothing AND the resulting numerator's lowest set bits divide + # evenly -- simpler and just as decisive here: a dyadic value + # would print identically under %.17g and %.6f once trailing + # zeros are accounted for, which none of these do (see the + # next test) -- this test instead confirms none is a "clean" + # few-bits-of-mantissa value like *.0, *.5, *.25, *.125. + frac = abs(value) - int(abs(value)) + eighths = frac * 8 + if abs(eighths - round(eighths)) < 1e-9: + dyadic += 1 + self.assertEqual(dyadic, 0) + + def test_engine_format_corpus_would_fail_a_c17g_only_grammar(self): + # Every token in the corpus is a %.6f spelling. Whether any ONE + # such token also happens to be its own double's shortest %.17g + # round-trip spelling is unpredictable per-token (it depends on + # that double's neighborhood, not on the fact that it came from + # %.6f) -- but the module docstring's actual claim is about the + # TRANSCRIPT, not any single token: one rejected token anywhere + # is enough to fail the whole check, since `_numeric_tail` bails + # out of the entire frame the moment its numeric parse raises. + # Confirm most tokens are rejected outright by the standalone + # %.17g parser this module still carries (`_c17g`, kept for the + # earlier engine's own emitted form), and that at least one + # rejection lands inside a real DATA frame of the corpus -- + # which is what actually dooms the whole transcript under a + # %.17g-only grammar. + blob, tokens = self._corpus() + rejected = 0 + for token in tokens: + try: + GAPS._c17g(token, "corpus token") + except ValueError: + rejected += 1 + self.assertGreater(rejected, len(tokens) // 2, (rejected, len(tokens))) + + frame_rejected = False + for line in blob.split(b"\n"): + if not line.startswith(b"DATA "): + continue + fields = line.split(b" ") + for field in fields[3:]: + try: + GAPS._c17g(field, "corpus token") + except ValueError: + frame_rejected = True + break + if frame_rejected: + break + self.assertTrue( + frame_rejected, + "expected at least one DATA frame in the corpus to contain a " + "token the %.17g-only parser refuses") + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_check_native_mtp_witness.py b/c/tests/test_check_native_mtp_witness.py new file mode 100644 index 000000000..b81fd78c3 --- /dev/null +++ b/c/tests/test_check_native_mtp_witness.py @@ -0,0 +1,1553 @@ +"""Drive the hash-bound native-MTP witness capture/validate tool against an +injected stand-in for the direct engine launch (no real model, no real +binary process is ever started by this module). +""" +import contextlib +import importlib.util +import io +import json +import pathlib +import sys +import tempfile +import types +import unittest +from unittest import mock + + +HERE = pathlib.Path(__file__).resolve().parent + +_spec = importlib.util.spec_from_file_location( + "check_native_mtp_witness_under_test", HERE / "check_native_mtp_witness.py") +WITNESS = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(WITNESS) + + +@unittest.skipIf( + sys.platform == "win32", + "the witness requires POSIX stat semantics (fstat/lstat identity) and " + "refuses on Windows; WindowsRefusalTests covers that refusal") +class NativeMtpWitnessTests(unittest.TestCase): + REQUEST_ID = 7 + TOPK = 2 + VOCAB = 3 + INPUT = b"SUBMIT 7 0 1 2 0 1 0 logprobs=2\nx\n" + STDOUT = ( + b"== GLM C engine (glm_moe_dsa), cache=64 experts/layer | " + b"compute experts@4-bit dense@8-bit | idot: neon-i8mm ==\n" + b"loaded in 1.00s | resident dense: 1.00 MB | " + b"layers=78 experts=256 | MTP ACTIVE (draft=1)\n" + b"\x01\x01READY\x01\x01\n" + b"STAT 0 0.00 0.0 10.00\n" + b"HWINFO 16 128.0 64.0 0 0.0 AMD Ryzen 9|\n" + b"TIERS 0 256 1024 0.00 32.50\n" + b"EMAP 2 2 00010203\n" + b"ACCEPT 7 1\n" + b"ECHO 7 1 0 nan 0\nx\n" + b"DATA 7 1 -0.125 2 1 -0.125 0 -2.125\nx\n" + b"HWINFO 16 128.0 64.0 0 0.0 AMD Ryzen 9|\n" + b"TIERS 0 256 1024 0.00 32.50\n" + b"EMAP 2 2 00010203\n" + b"HITS 1 2 00\n" + b"PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n" + b"DONE 7 STAT 1 0.10 100.0 10.00 1 0\n" + ) + STDERR = ( + b"[MTP] active: native speculative decoding (draft=1)\n" + b"[stop] 1 stop tokens: 2\n" + b"[MTP] single-slot serve: speculation active (draft=1)\n" + b"[mtpdbg] draft0=1 verified=1 HIT\n" + b"[mtpemit] request=7 ordinal=0 token=1\n" + ) + + @classmethod + def expected_environment(cls, snapshot, draft=1, ctx=4096): + return { + "SNAP": str(pathlib.Path(snapshot).resolve(strict=True)), + "SERVE": "1", "SERVE_BATCH": "1", + "KV_SLOTS": "1", "DRAFT": str(draft), "CTX": str(ctx), + "MTP_DEBUG": "1", "KVSAVE": "0", "USAGE_SAVE": "0", + "COLI_NO_OMP_TUNE": "1", "LANG": "C", "LC_ALL": "C", + } + + @classmethod + def stdout_for_draft(cls, draft): + return cls.STDOUT.replace(b"MTP ACTIVE (draft=1)", + f"MTP ACTIVE (draft={draft})".encode("ascii")) + + @classmethod + def stderr_for_draft(cls, draft): + return cls.STDERR.replace( + b"(draft=1)", f"(draft={draft})".encode("ascii")) + + @classmethod + def submit(cls, payload, maximum=2, topk=None): + if topk is None: + topk = cls.TOPK + header = (f"SUBMIT {cls.REQUEST_ID} 0 {len(payload)} {maximum} " + f"0 1 0 logprobs={topk}\n").encode("ascii") + return header + payload + b"\n" + + @staticmethod + def write_snapshot_manifest(snapshot, container, payloads=None): + snapshot = pathlib.Path(snapshot) + container = pathlib.Path(container) + if payloads is None: + payloads = {"tokenizer.json": b"{}\n"} + records = [] + for relative, raw in sorted(payloads.items()): + target = snapshot.joinpath(*relative.split("/")) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(raw) + records.append( + f"{WITNESS._sha256_bytes(raw)} {relative}\n".encode("ascii")) + container.write_bytes(b"".join(records)) + + def make_fixture(self, root, draft=1, ctx=4096): + root = pathlib.Path(root) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + container = root / "container-manifest.sha256" + request = root / "request-source.raw" + run = root / "run" + binary.write_bytes(b"test-binary") + self.write_snapshot_manifest(snapshot, container) + request.write_bytes(self.INPUT) + + def fake_run(argv, **kwargs): + kwargs["stdout"].write(self.stdout_for_draft(draft)) + kwargs["stderr"].write(self.stderr_for_draft(draft)) + return types.SimpleNamespace(returncode=0) + + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), str(run), + self.REQUEST_ID, self.TOPK, self.VOCAB, draft=draft, ctx=ctx, + run_fn=fake_run) + paths = { + "binary": binary, + "container": container, + "environment": run / "environment.json", + "input": run / "request.raw", + "status": run / "engine_status.txt", + "stdout": run / "engine_stdout.raw", + "stderr": run / "engine_stderr.raw", + } + return snapshot, paths, run / "binding.json" + + def rebind(self, binding, paths): + record = json.loads(pathlib.Path(binding).read_bytes()) + for name, item in record["artifacts"].items(): + raw = pathlib.Path(paths[name]).read_bytes() + item["size"] = len(raw) + item["sha256"] = WITNESS._sha256_bytes(raw) + record["payloads"] = WITNESS._payload_records( + pathlib.Path(paths["stdout"]).read_bytes()) + record["binding_id"] = WITNESS._binding_id(record) + pathlib.Path(binding).write_bytes(WITNESS._canonical_json(record)) + + @staticmethod + def rewrite_binding(binding, transform): + record = json.loads(pathlib.Path(binding).read_bytes()) + transform(record) + record["binding_id"] = WITNESS._binding_id(record) + pathlib.Path(binding).write_bytes(WITNESS._canonical_json(record)) + + @staticmethod + def rewrite_environment(path, transform): + environment = json.loads(path.read_text()) + transform(environment) + path.write_bytes(WITNESS._canonical_json(environment)) + + def test_complete_compound_witness_passes(self): + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + result = WITNESS.validate_binding(binding) + self.assertEqual(result["request_id"], self.REQUEST_ID) + self.assertEqual(result["accepted_tokens"], [1]) + self.assertEqual(result["replay_verdict"], "COMPLETE") + self.assertEqual(result["provenance_verdict"], "INCOMPLETE") + self.assertRegex(result["binding_id"], r"^[0-9a-f]{64}$") + stdout = io.StringIO() + stderr = io.StringIO() + with contextlib.redirect_stdout(stdout), \ + contextlib.redirect_stderr(stderr): + self.assertEqual(WITNESS.main(["validate", str(binding)]), 1) + self.assertIn("[native-mtp] REPLAY ", stdout.getvalue()) + self.assertNotIn("PASS", stdout.getvalue() + stderr.getvalue()) + self.assertEqual( + stderr.getvalue(), + "[native-mtp] INCOMPLETE: offline replay cannot establish " + "common-run provenance\n") + + def test_cr1_live_only_pass_and_offline_replays_stay_incomplete(self): + def assert_offline_incomplete(binding): + result = WITNESS.validate_binding(binding) + self.assertEqual(result["replay_verdict"], "COMPLETE") + self.assertEqual(result["provenance_verdict"], "INCOMPLETE") + stdout = io.StringIO() + stderr = io.StringIO() + with contextlib.redirect_stdout(stdout), \ + contextlib.redirect_stderr(stderr): + self.assertEqual(WITNESS.main(["validate", str(binding)]), 1) + combined = stdout.getvalue() + stderr.getvalue() + self.assertNotIn("PASS", combined) + self.assertIn("[native-mtp] REPLAY ", stdout.getvalue()) + self.assertIn("offline replay cannot establish common-run provenance", + stderr.getvalue()) + + with tempfile.TemporaryDirectory() as tmp_a, \ + tempfile.TemporaryDirectory() as tmp_b: + _, paths_a, binding_a = self.make_fixture(tmp_a) + _, paths_b, _ = self.make_fixture(tmp_b) + + outcome_path = pathlib.Path(tmp_a) / "run" / "capture_outcome.json" + outcome_raw = outcome_path.read_bytes() + outcome = json.loads(outcome_raw) + binding_raw = pathlib.Path(binding_a).read_bytes() + self.assertEqual(set(outcome), { + "schema", "verdict", "binding_id", "binding_sha256", + "request_id", "accepted_tokens", "run_root", + }) + self.assertEqual(outcome["schema"], WITNESS.CAPTURE_OUTCOME_SCHEMA) + self.assertEqual(outcome["verdict"], "PASS") + self.assertEqual(outcome["binding_sha256"], + WITNESS._sha256_bytes(binding_raw)) + self.assertEqual(outcome["accepted_tokens"], [1]) + self.assertEqual(outcome_raw, WITNESS._canonical_json(outcome)) + + # Clean and fully self-hashed replay are still non-promotable. + assert_offline_incomplete(binding_a) + self.rebind(binding_a, paths_a) + assert_offline_incomplete(binding_a) + + # Rebinding changed but semantically irrelevant binary bytes cannot + # turn detached evidence into a provenance PASS. + paths_a["binary"].write_bytes(b"rebound-binary") + self.rebind(binding_a, paths_a) + assert_offline_incomplete(binding_a) + + # A structurally coherent A/B splice, followed by complete artifact + # and binding rehash, remains only an offline replay. + paths_b["stderr"].write_bytes( + paths_b["stderr"].read_bytes() + b"[run] capture-b\n") + paths_a["stderr"].write_bytes(paths_b["stderr"].read_bytes()) + self.rebind(binding_a, paths_a) + assert_offline_incomplete(binding_a) + + def test_environment_guards_bite_independently(self): + mutations = ( + ("serve", lambda env: env.__setitem__("SERVE", "0"), "SERVE"), + ("batch", lambda env: env.__setitem__("SERVE_BATCH", "0"), + "SERVE_BATCH"), + ("slots", lambda env: env.__setitem__("KV_SLOTS", "2"), "KV_SLOTS"), + ("debug", lambda env: env.__setitem__("MTP_DEBUG", "0"), + "MTP_DEBUG"), + ("kvsave", lambda env: env.__setitem__("KVSAVE", "1"), "KVSAVE"), + ("missing_usage_save", lambda env: env.pop("USAGE_SAVE"), + "USAGE_SAVE"), + ("enabled_usage_save", + lambda env: env.__setitem__("USAGE_SAVE", "1"), "USAGE_SAVE"), + ("lang", lambda env: env.__setitem__("LANG", "C.UTF-8"), "LANG"), + ("locale", lambda env: env.__setitem__("LC_ALL", "C.UTF-8"), + "LC_ALL"), + ("missing_draft", lambda env: env.pop("DRAFT"), "DRAFT"), + ("zero_draft", lambda env: env.__setitem__("DRAFT", "0"), "DRAFT"), + ("large_draft", lambda env: env.__setitem__("DRAFT", "64"), "DRAFT"), + ("grammar", lambda env: env.__setitem__("GRAMMAR", "/g"), + "exact CPU witness map"), + ("schema", lambda env: env.__setitem__("SCHEMA", "/s"), + "exact CPU witness map"), + ("corpus", lambda env: env.__setitem__("COLI_DRAFT_CORPUS", "/c"), + "exact CPU witness map"), + ("snap", lambda env: env.__setitem__("SNAP", "/wrong"), "SNAP"), + ("missing_ctx", lambda env: env.pop("CTX"), "CTX"), + ("zero_ctx", lambda env: env.__setitem__("CTX", "0"), "CTX"), + ) + for name, mutation, expected in mutations: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + snapshot, paths, binding = self.make_fixture(tmp) + self.rewrite_environment(paths["environment"], mutation) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, expected): + WITNESS.validate_binding(binding) + + def test_ctx_int32_boundaries_and_canonical_grammar(self): + for ctx in (1, WITNESS._INT32_MAX): + with self.subTest(ctx=ctx), tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp, ctx=ctx) + self.assertEqual( + WITNESS.validate_binding(binding)["accepted_tokens"], [1]) + + for ctx in (str(WITNESS._INT32_MAX + 1), "01"): + with self.subTest(ctx=ctx), tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + self.rewrite_environment( + paths["environment"], + lambda env, value=ctx: env.__setitem__("CTX", value)) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, "CTX"): + WITNESS.validate_binding(binding) + + def test_exact_omp_recipe_variants_bite_independently(self): + mutations = ( + ("remove_kill_switch", lambda env: env.pop("COLI_NO_OMP_TUNE")), + ("alter_kill_switch", + lambda env: env.__setitem__("COLI_NO_OMP_TUNE", "0")), + ("self_reexec_sentinel", + lambda env: env.__setitem__("COLI_OMP_TUNED", "1")), + ("omp_tuning", lambda env: env.__setitem__("OMP_NUM_THREADS", "2")), + ("gomp_tuning", + lambda env: env.__setitem__("GOMP_CPU_AFFINITY", "0")), + ("kmp_tuning", lambda env: env.__setitem__("KMP_AFFINITY", "none")), + ("cuda_backend", lambda env: env.__setitem__("COLI_CUDA", "1")), + ("metal_backend", lambda env: env.__setitem__("COLI_METAL", "1")), + ) + for name, mutation in mutations: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + self.rewrite_environment(paths["environment"], mutation) + self.rebind(binding, paths) + with self.assertRaises(WITNESS.WitnessError): + WITNESS.validate_binding(binding) + + def test_configured_and_loaded_draft_join_and_boundaries(self): + for configured, loaded in ((2, 1), (1, 2)): + with self.subTest(configured=configured, loaded=loaded), \ + tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + self.rewrite_environment( + paths["environment"], + lambda env, value=configured: env.__setitem__( + "DRAFT", str(value))) + paths["stdout"].write_bytes(self.stdout_for_draft(loaded)) + self.rebind(binding, paths) + with self.assertRaisesRegex( + WITNESS.WitnessError, + "configured DRAFT does not equal loaded"): + WITNESS.validate_binding(binding) + + for draft in (1, 63): + with self.subTest(draft=draft), tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp, draft=draft) + self.assertEqual( + WITNESS.validate_binding(binding)["accepted_tokens"], [1]) + + def test_input_guards_bite_independently(self): + mutations = ( + ("temperature", self.INPUT.replace(b" 0 1 0 logprobs", b" 1 1 0 logprobs")), + ("slot", self.INPUT.replace(b"SUBMIT 7 0", b"SUBMIT 7 1")), + ("grammar", self.INPUT.replace(b" 0 logprobs", b" 1 logprobs")), + ("stop", self.INPUT + b"STOP 7\n"), + ("cancel", self.INPUT + b"CANCEL 7\n"), + ("second", self.INPUT + self.INPUT), + ) + for name, content in mutations: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + snapshot, paths, binding = self.make_fixture(tmp) + paths["input"].write_bytes(content) + self.rebind(binding, paths) + with self.assertRaises(WITNESS.WitnessError): + WITNESS.validate_binding(binding) + + def test_prompt_payload_boundaries_and_nul_bite_independently(self): + for size, should_pass in ((0, False), (1, True), + (16 * 1024 * 1024, True), + (16 * 1024 * 1024 + 1, False)): + with self.subTest(size=size), tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["input"].write_bytes(self.submit(b"x" * size)) + self.rebind(binding, paths) + if should_pass: + self.assertEqual( + WITNESS.validate_binding(binding)["accepted_tokens"], [1]) + else: + with self.assertRaisesRegex(WITNESS.WitnessError, "1..16 MiB"): + WITNESS.validate_binding(binding) + + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["input"].write_bytes(self.submit(b"\0")) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, "contains NUL"): + WITNESS.validate_binding(binding) + + def test_maximum_and_vocabulary_int32_boundaries_bite_independently(self): + values, _ = WITNESS._validate_input( + self.submit(b"x", maximum=1), self.REQUEST_ID, self.TOPK) + self.assertEqual(values["maximum"], 1) + with self.assertRaisesRegex(WITNESS.WitnessError, "maximum"): + WITNESS._validate_input( + self.submit(b"x", maximum=0), self.REQUEST_ID, self.TOPK) + + for maximum, should_pass in ((0, False), (1, True), + (WITNESS._INT32_MAX, True), + (WITNESS._INT32_MAX + 1, False)): + with self.subTest(maximum=maximum), \ + tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["input"].write_bytes(self.submit(b"x", maximum)) + self.rebind(binding, paths) + if should_pass: + WITNESS.validate_binding(binding) + else: + with self.assertRaisesRegex(WITNESS.WitnessError, "maximum"): + WITNESS.validate_binding(binding) + + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["input"].write_bytes(self.submit(b"x", maximum=1, topk=1)) + paths["stdout"].write_bytes( + self.STDOUT.replace( + b"DATA 7 1 -0.125 2 1 -0.125 0 -2.125\nx\n", + b"DATA 7 1 -0.125 1 0 -0.125\nx\n")) + paths["stderr"].write_bytes( + self.STDERR.replace( + b"[stop] 1 stop tokens: 2", + b"[stop] 0 stop tokens:").replace( + b"draft0=1 verified=1 HIT", + b"draft0=0 verified=0 HIT").replace( + b"token=1", b"token=0")) + self.rebind(binding, paths) + self.rewrite_binding( + binding, + lambda record: record["request"].update( + {"topk": 1, "vocab": 1})) + WITNESS.validate_binding(binding) + + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + self.rewrite_binding( + binding, + lambda record: record["request"].__setitem__("vocab", 0)) + with self.assertRaisesRegex( + WITNESS.WitnessError, "outside their domains"): + WITNESS.validate_binding(binding) + + for vocab, should_pass in ((WITNESS._INT32_MAX, True), + (WITNESS._INT32_MAX + 1, False)): + with self.subTest(vocab=vocab), tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + self.rewrite_binding( + binding, + lambda record, value=vocab: record["request"].__setitem__( + "vocab", value)) + if should_pass: + WITNESS.validate_binding(binding) + else: + with self.assertRaisesRegex( + WITNESS.WitnessError, "outside their domains"): + WITNESS.validate_binding(binding) + + def test_emitted_data_count_cannot_exceed_bound_maximum(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["input"].write_bytes(self.submit(b"x", maximum=1)) + self.rebind(binding, paths) + self.assertEqual( + WITNESS.validate_binding(binding)["accepted_tokens"], [1]) + data = b"DATA 7 1 -0.125 2 1 -0.125 0 -2.125\nx\n" + stdout = self.STDOUT.replace(data, data + data, 1) + stdout = stdout.replace( + b"DONE 7 STAT 1 0.10", b"DONE 7 STAT 2 0.10", 1) + paths["stdout"].write_bytes(stdout) + self.rebind(binding, paths) + with self.assertRaisesRegex( + WITNESS.WitnessError, "emitted DATA count exceeds"): + WITNESS.validate_binding(binding) + + def test_stdout_requires_active_positive_native_mtp_and_one_request(self): + replacements = ( + (b"MTP ACTIVE (draft=1)", b"MTP ACTIVE (draft=0)"), + (b"MTP ACTIVE (draft=1)", b"MTP absent (draft=2)"), + (b"MTP ACTIVE (draft=1)", b"MTP DISABLED (multiplexed serve) (draft=0)"), + ) + for old, new in replacements: + with self.subTest(new=new), tempfile.TemporaryDirectory() as tmp: + snapshot, paths, binding = self.make_fixture(tmp) + paths["stdout"].write_bytes(self.STDOUT.replace(old, new)) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, + "active positive-depth"): + WITNESS.validate_binding(binding) + + with tempfile.TemporaryDirectory() as tmp: + snapshot, paths, binding = self.make_fixture(tmp) + other = b"ACCEPT 8 1\n" + paths["stdout"].write_bytes( + self.STDOUT.replace(b"ACCEPT 7 1\n", other + b"ACCEPT 7 1\n")) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, "second request"): + WITNESS.validate_binding(binding) + + def test_stderr_witness_guards_bite_independently(self): + mutations = ( + ("unequal", self.STDERR.replace( + b"draft0=1 verified=1 HIT", b"draft0=1 verified=0 miss").replace( + b"[mtpemit] request=7 ordinal=0 token=1\n", b""), + "no equal non-stop"), + ("lying_hit", self.STDERR.replace( + b"draft0=1 verified=1 HIT", b"draft0=1 verified=0 HIT"), + "contradicts"), + ("stop", self.STDERR.replace( + b"[stop] 1 stop tokens: 2", b"[stop] 1 stop tokens: 1").replace( + b"[mtpemit] request=7 ordinal=0 token=1\n", b""), + "no equal non-stop"), + ("missing", self.STDERR.replace( + b"[mtpdbg] draft0=1 verified=1 HIT\n", b"").replace( + b"[mtpemit] request=7 ordinal=0 token=1\n", b""), + "no mtpdbg"), + ("malformed", self.STDERR.replace(b"draft0=1", b"draft0=01"), + "malformed mtpdbg"), + ("reordered", self.STDERR.replace( + b"[stop] 1 stop tokens: 2\n" + b"[MTP] single-slot serve: speculation active (draft=1)\n" + b"[mtpdbg] draft0=1 verified=1 HIT\n", + b"[mtpdbg] draft0=1 verified=1 HIT\n" + b"[stop] 1 stop tokens: 2\n" + b"[MTP] single-slot serve: speculation active (draft=1)\n"), + "precedes the same-run stop set"), + ("grammar", self.STDERR + b"[GRAMMAR] request: active\n", + "alternate grammar/corpus"), + ("corpus", self.STDERR + b"[CORPUS] 10 ids from x\n", + "alternate grammar/corpus"), + ) + for name, content, expected in mutations: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + snapshot, paths, binding = self.make_fixture(tmp) + paths["stderr"].write_bytes(content) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, expected): + WITNESS.validate_binding(binding) + + def test_mtpemit_identity_ordinal_and_data_join_bite_independently(self): + data = b"DATA 7 1 -0.125 2 1 -0.125 0 -2.125\nx\n" + emit = b"[mtpemit] request=7 ordinal=0 token=1\n" + hit = b"[mtpdbg] draft0=1 verified=1 HIT\n" + mutations = ( + ("hit_token", None, self.STDERR.replace( + emit, b"[mtpemit] request=7 ordinal=0 token=0\n"), + "does not match its qualifying HIT"), + ("data_target", self.STDOUT.replace( + data, b"DATA 7 1 -0.125 2 0 -0.125 1 -2.125\nx\n"), + None, "does not own its DATA target"), + ("missing", None, self.STDERR.replace(emit, b""), + "engine does not emit the accepted-token witness line"), + ("partial_binding", None, self.STDERR.replace( + emit, emit + b"[mtpdbg] draft0=0 verified=0 HIT\n"), + "missing its mtpemit/DATA binding"), + ("duplicate", None, self.STDERR.replace(emit, emit + hit + emit), + "duplicate/replayed"), + ("out_of_range", None, self.STDERR.replace( + emit, b"[mtpemit] request=7 ordinal=1 token=1\n"), + "has no DATA row"), + ("wrong_request", None, self.STDERR.replace( + emit, b"[mtpemit] request=8 ordinal=0 token=1\n"), + "does not match the bound request"), + ("malformed", None, self.STDERR.replace( + emit, b"[mtpemit] request=7 ordinal=00 token=1\n"), + "malformed mtpemit"), + ) + for name, stdout, stderr, expected in mutations: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + if stdout is not None: + paths["stdout"].write_bytes(stdout) + if stderr is not None: + paths["stderr"].write_bytes(stderr) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, expected): + WITNESS.validate_binding(binding) + + def test_nonzero_direct_engine_status_never_passes(self): + with tempfile.TemporaryDirectory() as tmp: + snapshot, paths, binding = self.make_fixture(tmp) + paths["status"].write_bytes(b"9\n") + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, "not exact zero"): + WITNESS.validate_binding(binding) + + def test_every_artifact_binding_bites_independently(self): + for name in ("binary", "container", "environment", "input", "status", + "stdout", "stderr"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + raw = paths[name].read_bytes() + self.assertTrue(raw) + paths[name].write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + with self.assertRaisesRegex(WITNESS.WitnessError, + f"{name} artifact SHA-256 mismatch"): + WITNESS.validate_binding(binding) + + def test_empty_container_artifact_is_not_a_witness(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["container"].write_bytes(b"") + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, "nonempty"): + WITNESS.validate_binding(binding) + + def test_empty_container_refuses_before_child_launch(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + container.write_bytes(b"") + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run = mock.Mock(side_effect=AssertionError("child launched")) + with self.assertRaisesRegex(WITNESS.WitnessError, "nonempty"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(root / "run"), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=run) + run.assert_not_called() + + def test_snapshot_manifest_grammar_and_inventory_bite(self): + digest = b"0" * 64 + letter_digest = b"a" * 64 + malformed = ( + b"", b"payload 00\n", letter_digest.upper() + b" file\n", + digest + b" file\n", digest + b" /absolute\n", + digest + b" ../escape\n", digest + b" a//b\n", + digest + b" .\n", digest + b" C:/absolute\n", + digest + b" a\\b\n", digest + b" file\r\n", + digest + b" b\n" + digest + b" a\n", + digest + b" a\n" + digest + b" a\n", + ) + for raw in malformed: + with self.subTest(raw=raw), self.assertRaises(WITNESS.WitnessError): + WITNESS._parse_snapshot_manifest(raw) + + for mutation in ("modify", "add", "remove"): + with self.subTest(mutation=mutation), \ + tempfile.TemporaryDirectory() as tmp: + snapshot, _, binding = self.make_fixture(tmp) + payload = snapshot / "tokenizer.json" + if mutation == "modify": + payload.write_bytes(b"changed\n") + elif mutation == "add": + (snapshot / "model.safetensors").write_bytes(b"extra") + else: + payload.unlink() + with self.assertRaises(WITNESS.WitnessError): + WITNESS.validate_binding(binding) + + def test_snapshot_symlink_and_unrelated_manifest_refuse(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + outside = root / "outside" + outside.write_bytes(b"outside") + try: + (snapshot / "linked").symlink_to(outside) + except OSError: + pass # Windows without symlink privilege: remaining bites still run. + else: + container = root / "container.sha256" + container.write_bytes( + f"{WITNESS._sha256_bytes(b'outside')} linked\n".encode("ascii")) + with self.assertRaisesRegex(WITNESS.WitnessError, "symlink"): + WITNESS._apply_snapshot_manifest( + snapshot, container.read_bytes()) + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + unrelated = ( + f"{WITNESS._sha256_bytes(b'not the payload')} tokenizer.json\n" + ).encode("ascii") + with self.assertRaisesRegex(WITNESS.WitnessError, "SHA-256 mismatch"): + WITNESS._apply_snapshot_manifest(snapshot, unrelated) + + def test_snapshot_must_remain_stable_during_capture(self): + for mutation in ("modify", "add", "remove", "modify_restore"): + with self.subTest(mutation=mutation), \ + tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + payload = snapshot / "tokenizer.json" + original = payload.read_bytes() + + def mutating_run(argv, **kwargs): + if mutation in ("modify", "modify_restore"): + payload.write_bytes(b"changed") + if mutation == "modify_restore": + payload.write_bytes(original) + elif mutation == "add": + (snapshot / "extra.bin").write_bytes(b"extra") + else: + payload.unlink() + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + return types.SimpleNamespace(returncode=0) + + with self.assertRaises(WITNESS.WitnessError): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(root / "run"), self.REQUEST_ID, self.TOPK, + self.VOCAB, run_fn=mutating_run) + self.assertFalse((root / "run" / "binding.json").exists()) + + def test_usage_persistence_is_disabled_for_absent_and_existing_file(self): + for existing in (False, True): + with self.subTest(existing=existing), \ + tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + payloads = {"tokenizer.json": b"{}\n"} + if existing: + payloads[".coli_usage"] = b"bound usage profile\n" + self.write_snapshot_manifest(snapshot, container, payloads) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + manifest_raw = container.read_bytes() + inventory_before, identity_before = \ + WITNESS._apply_snapshot_manifest(snapshot, manifest_raw) + usage_path = snapshot / ".coli_usage" + usage_before = usage_path.read_bytes() if existing else None + + def usage_writing_run(argv, **kwargs): + if kwargs["env"].get("USAGE_SAVE") != "0": + usage_path.write_bytes(b"mutated usage profile\n") + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + return types.SimpleNamespace(returncode=0) + + result = WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(root / "run"), self.REQUEST_ID, self.TOPK, + self.VOCAB, run_fn=usage_writing_run) + self.assertEqual(result["accepted_tokens"], [1]) + self.assertEqual( + json.loads((root / "run" / "environment.json").read_bytes())[ + "USAGE_SAVE"], "0") + inventory_after, identity_after = \ + WITNESS._apply_snapshot_manifest(snapshot, manifest_raw) + self.assertEqual(inventory_after, inventory_before) + self.assertEqual(identity_after, identity_before) + if existing: + self.assertEqual(usage_path.read_bytes(), usage_before) + else: + self.assertFalse(usage_path.exists()) + self.assertEqual( + WITNESS.validate_binding(root / "run" / "binding.json")[ + "accepted_tokens"], [1]) + + def test_capture_uses_one_resolved_snapshot_identity_through_aliases(self): + for redirect in (False, True): + with self.subTest(redirect=redirect), \ + tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + real_parent = root / "real-parent" + real_parent.mkdir() + snapshot = real_parent / "snapshot" + snapshot.mkdir() + alternate_parent = root / "alternate-parent" + alternate_parent.mkdir() + (alternate_parent / "snapshot").mkdir() + alias_parent = root / "snapshot-parent" + alias_parent.symlink_to(real_parent, target_is_directory=True) + aliased_snapshot = alias_parent / "snapshot" + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + resolved = snapshot.resolve(strict=True) + observed = {} + + def alias_run(argv, **kwargs): + observed["snap"] = kwargs["env"]["SNAP"] + if redirect: + alias_parent.unlink() + alias_parent.symlink_to( + alternate_parent, target_is_directory=True) + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + return types.SimpleNamespace(returncode=0) + + result = WITNESS.capture_bundle( + str(binary), str(aliased_snapshot), str(container), + str(request), str(root / "run"), self.REQUEST_ID, + self.TOPK, self.VOCAB, run_fn=alias_run) + self.assertEqual(result["accepted_tokens"], [1]) + self.assertEqual(observed["snap"], str(resolved)) + environment = json.loads( + (root / "run" / "environment.json").read_bytes()) + binding = json.loads( + (root / "run" / "binding.json").read_bytes()) + self.assertEqual(environment["SNAP"], str(resolved)) + self.assertEqual(binding["snapshot"], str(resolved)) + self.assertEqual( + WITNESS.validate_binding(root / "run" / "binding.json")[ + "accepted_tokens"], [1]) + + def test_snapshot_inventory_binding_and_old_schema_refuse(self): + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + self.rewrite_binding( + binding, + lambda record: record["snapshot_inventory"].__setitem__( + "sha256", "0" * 64)) + with self.assertRaisesRegex(WITNESS.WitnessError, + "bound snapshot inventory"): + WITNESS.validate_binding(binding) + + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + self.rewrite_binding( + binding, + lambda record: record.__setitem__( + "schema", "colibri-b1-native-mtp-witness/1")) + with self.assertRaisesRegex(WITNESS.WitnessError, "unknown binding schema"): + WITNESS.validate_binding(binding) + + def test_binary_and_container_must_remain_stable_during_capture(self): + for changed_name in ("binary", "container"): + with self.subTest(changed_name=changed_name), \ + tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary-before") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run_dir = root / "run" + + def replacing_run(argv, **kwargs): + target = binary if changed_name == "binary" else container + replacement = root / f"{changed_name}.replacement" + replacement.write_bytes( + f"{changed_name}-after".encode("ascii")) + replacement.replace(target) + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + return types.SimpleNamespace(returncode=0) + + with self.assertRaisesRegex( + WITNESS.WitnessError, + f"{changed_name} changed during direct capture"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=replacing_run) + self.assertFalse((run_dir / "binding.json").exists()) + + def test_capture_owned_paths_cannot_be_replaced_before_binding(self): + owned = { + "environment": "environment.json", "input": "request.raw", + "stdout": "engine_stdout.raw", "stderr": "engine_stderr.raw", + } + for name, filename in owned.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run_dir = root / "run" + + def replacing_run(argv, **kwargs): + kwargs["stdout"].write( + b"invalid child stdout\n" if name == "stdout" + else self.STDOUT) + kwargs["stderr"].write( + b"invalid child stderr\n" if name == "stderr" + else self.STDERR) + replacement = run_dir / f"{filename}.replacement" + replacements = { + "environment": WITNESS._canonical_json( + self.expected_environment(snapshot)), + "input": self.INPUT, "stdout": self.STDOUT, + "stderr": self.STDERR, + } + self.assertFalse((run_dir / filename).exists()) + replacement.write_bytes(replacements[name]) + replacement.replace(run_dir / filename) + return types.SimpleNamespace(returncode=0) + + with self.assertRaisesRegex( + WITNESS.WitnessError, + f"{name} capture path was precreated"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=replacing_run) + self.assertFalse((run_dir / "binding.json").exists()) + + def test_child_path_reopen_cannot_rewrite_anonymous_capture_bytes(self): + owned = { + "input": ("request.raw", self.INPUT[:-2] + b"y\n"), + "stdout": ("engine_stdout.raw", self.STDOUT), + "stderr": ("engine_stderr.raw", self.STDERR), + } + for name, (filename, replacement_raw) in owned.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run_dir = root / "run" + observed = {} + + def rewriting_run(argv, **kwargs): + observed["stdin"] = kwargs["stdin"].read() + kwargs["stdout"].write( + b"invalid child stdout\n" if name == "stdout" + else self.STDOUT) + kwargs["stderr"].write( + b"invalid child stderr\n" if name == "stderr" + else self.STDERR) + target = run_dir / filename + self.assertFalse(target.exists()) + with open(target, "wb") as stream: + stream.write(replacement_raw) + return types.SimpleNamespace(returncode=0) + + with self.assertRaisesRegex( + WITNESS.WitnessError, + f"{name} capture path was precreated"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=rewriting_run) + self.assertEqual(observed["stdin"], self.INPUT) + self.assertFalse((run_dir / "binding.json").exists()) + + def test_child_cannot_precreate_status_capture(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run_dir = root / "run" + + def precreating_run(argv, **kwargs): + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + (run_dir / "engine_status.txt").write_bytes(b"0\n") + return types.SimpleNamespace(returncode=0) + + with self.assertRaisesRegex( + WITNESS.WitnessError, "status capture path was precreated"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=precreating_run) + self.assertFalse((run_dir / "binding.json").exists()) + + def test_validator_reads_each_artifact_once_into_an_immutable_map(self): + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + calls = [] + original = WITNESS._read_artifact + + def recording_read(path, label): + calls.append(label) + return original(path, label) + + record = WITNESS._load_binding(binding) + with mock.patch.object(WITNESS, "_read_artifact", + side_effect=recording_read): + blobs = WITNESS._validate_artifacts(record) + self.assertEqual(sorted(calls), sorted(record["artifacts"])) + self.assertEqual(len(calls), len(set(calls))) + with self.assertRaises(TypeError): + blobs["stdout"] = b"replacement" + + def _assert_semantic_caller_uses_shared_bytes(self, name): + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + record = WITNESS._load_binding(binding) + blobs = WITNESS.MappingProxyType({ + key: f"shared-{key}-bytes".encode("ascii") + for key in ("binary", "container", "environment", "input", + "status", "stdout", "stderr") + }) + with mock.patch.object(WITNESS, "_validate_artifacts", + return_value=blobs), \ + mock.patch.object( + WITNESS, "_apply_snapshot_manifest", + return_value=(record["snapshot_inventory"], ())) as snapshot_apply, \ + mock.patch.object( + WITNESS, "_validate_environment", + return_value={"DRAFT": "1"}) as environment, \ + mock.patch.object( + WITNESS, "_validate_input", + return_value=({"maximum": 1}, b"x")) as input_call, \ + mock.patch.object(WITNESS, "_validate_status") as status, \ + mock.patch.object( + WITNESS, "_validate_stdout", + return_value=({"draft": 1}, [{"target": b"-1", "topk": {1: b"-1"}}])) as stdout, \ + mock.patch.object( + WITNESS, "_validate_stderr", return_value=[1]) as stderr: + WITNESS.validate_binding(binding) + calls = { + "environment": environment, "input": input_call, + "status": status, "stdout": stdout, "stderr": stderr, + "container": snapshot_apply, + } + arg_index = 1 if name == "container" else 0 + self.assertIs(calls[name].call_args.args[arg_index], blobs[name]) + + def test_environment_caller_uses_shared_bytes(self): + self._assert_semantic_caller_uses_shared_bytes("environment") + + def test_input_caller_uses_shared_bytes(self): + self._assert_semantic_caller_uses_shared_bytes("input") + + def test_status_caller_uses_shared_bytes(self): + self._assert_semantic_caller_uses_shared_bytes("status") + + def test_stdout_caller_uses_shared_bytes(self): + self._assert_semantic_caller_uses_shared_bytes("stdout") + + def test_stderr_caller_uses_shared_bytes(self): + self._assert_semantic_caller_uses_shared_bytes("stderr") + + def test_snapshot_caller_uses_shared_container_bytes(self): + self._assert_semantic_caller_uses_shared_bytes("container") + + def test_common_binding_id_payloads_and_distinct_streams_bite(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + record = json.loads(binding.read_text()) + record["binding_id"] = "0" * 64 + binding.write_bytes(WITNESS._canonical_json(record)) + with self.assertRaisesRegex(WITNESS.WitnessError, "binding_id"): + WITNESS.validate_binding(binding) + + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + record = json.loads(binding.read_text()) + record["payloads"][0]["sha256"] = "0" * 64 + record["binding_id"] = WITNESS._binding_id(record) + binding.write_bytes(WITNESS._canonical_json(record)) + with self.assertRaisesRegex(WITNESS.WitnessError, "payload hashes"): + WITNESS.validate_binding(binding) + + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + record = json.loads(binding.read_text()) + record["artifacts"]["stderr"] = dict(record["artifacts"]["stdout"]) + record["binding_id"] = WITNESS._binding_id(record) + binding.write_bytes(WITNESS._canonical_json(record)) + with self.assertRaisesRegex(WITNESS.WitnessError, "paths must be distinct"): + WITNESS.validate_binding(binding) + + def test_complementary_runs_cannot_mint_a_post_hoc_witness(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + + def attempt(label, stdout, stderr): + run_dir = root / label + + def fake_run(argv, **kwargs): + kwargs["stdout"].write(stdout) + kwargs["stderr"].write(stderr) + return types.SimpleNamespace(returncode=0) + + with self.assertRaises(WITNESS.WitnessError): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=fake_run) + self.assertTrue((run_dir / "binding.json").is_file()) + with self.assertRaises(WITNESS.WitnessError): + WITNESS.validate_binding(run_dir / "binding.json") + return run_dir + + run_a = attempt( + "run-a", self.STDOUT, + self.STDERR.replace(b"[mtpdbg] draft0=1 verified=1 HIT\n", b"")) + run_b = attempt( + "run-b", + self.STDOUT.replace( + b"loaded in 1.00s | resident dense: 1.00 MB | " + b"layers=78 experts=256 | MTP ACTIVE (draft=1)\n", b""), + self.STDERR) + + (run_b / "engine_stdout.raw").write_bytes( + (run_a / "engine_stdout.raw").read_bytes()) + with self.assertRaisesRegex(WITNESS.WitnessError, "stdout artifact"): + WITNESS.validate_binding(run_b / "binding.json") + + spliced = root / "spliced-binding.json" + old_record_argv = [ + "record", "--binary", str(binary), "--snapshot", str(snapshot), + "--container-manifest", str(container), "--environment", + str(run_a / "environment.json"), "--input", + str(run_a / "request.raw"), "--status", + str(run_a / "engine_status.txt"), "--stdout", + str(run_a / "engine_stdout.raw"), "--stderr", + str(run_b / "engine_stderr.raw"), "--id", "7", "--topk", "2", + "--vocab", "3", "--output", str(spliced), + ] + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr), \ + self.assertRaises(SystemExit) as stopped: + WITNESS.main(old_record_argv) + self.assertEqual(stopped.exception.code, 2) + self.assertIn("invalid choice: 'record'", stderr.getvalue()) + self.assertFalse(spliced.exists()) + + def test_capture_recipe_uses_snap_env_distinct_streams_and_child_status(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run_dir = root / "run" + observed = {} + canaries = { + "AWS_SECRET_ACCESS_KEY": "credential-canary", + "HTTPS_PROXY": "proxy-canary", + "DYLD_INSERT_LIBRARIES": "/dynamic-loader-canary", + "RUST_LOG": "unrelated-runtime-canary", + } + + def fake_run(argv, **kwargs): + observed["argv"] = argv + observed.update(kwargs) + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + return types.SimpleNamespace(returncode=0) + + with mock.patch.dict(WITNESS.os.environ, canaries, clear=False): + result = WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=fake_run) + self.assertEqual(result["accepted_tokens"], [1]) + self.assertEqual(result["provenance_verdict"], "PASS") + self.assertRegex(result["outcome_sha256"], r"^[0-9a-f]{64}$") + self.assertEqual(observed["argv"], [str(binary.resolve())]) + self.assertNotIn(str(snapshot), observed["argv"]) + expected = self.expected_environment(snapshot) + self.assertEqual(observed["env"], expected) + serialized = json.loads((run_dir / "environment.json").read_bytes()) + self.assertEqual(serialized, expected) + for key in canaries: + self.assertNotIn(key, observed["env"]) + self.assertNotIn(key, serialized) + self.assertIsNot(observed["stdout"], observed["stderr"]) + self.assertFalse(observed["shell"]) + self.assertFalse(observed["check"]) + self.assertEqual((run_dir / "engine_status.txt").read_bytes(), b"0\n") + for name in ("environment.json", "request.raw", "engine_status.txt", + "engine_stdout.raw", "engine_stderr.raw", "binding.json"): + self.assertEqual((run_dir / name).parent, run_dir) + binding_raw = (run_dir / "binding.json").read_bytes() + outcome = json.loads((run_dir / "capture_outcome.json").read_bytes()) + self.assertEqual(outcome["binding_sha256"], + WITNESS._sha256_bytes(binding_raw)) + + def test_live_capture_refuses_binding_write_revalidation_failures(self): + original_write = WITNESS._write_binding + + def corrupt(path, record): + pathlib.Path(path).write_bytes(b"{}\n") + + def truncated(path, record): + pathlib.Path(path).write_bytes(WITNESS._canonical_json(record)[:-1]) + + def removed(path, record): + original_write(path, record) + pathlib.Path(path).unlink() + + def replaced(path, record): + other = json.loads(json.dumps(record)) + other["request"]["id"] += 1 + other["binding_id"] = WITNESS._binding_id(other) + pathlib.Path(path).write_bytes(WITNESS._canonical_json(other)) + + for label, writer in { + "corrupt": corrupt, "short": truncated, + "removed": removed, "replaced": replaced}.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run_dir = root / "run" + + def fake_run(argv, **kwargs): + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + return types.SimpleNamespace(returncode=0) + + with mock.patch.object(WITNESS, "_write_binding", + side_effect=writer), \ + self.assertRaisesRegex( + WITNESS.WitnessError, + "unavailable|exact byte revalidation"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=fake_run) + self.assertFalse((run_dir / "capture_outcome.json").exists()) + + def test_capture_recipe_records_child_failure_not_writer_success(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.INPUT) + run_dir = root / "run" + + def failed_run(argv, **kwargs): + kwargs["stdout"].write(self.STDOUT) + kwargs["stderr"].write(self.STDERR) + return types.SimpleNamespace(returncode=9) + + with self.assertRaisesRegex(WITNESS.WitnessError, "not exact zero"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=failed_run) + self.assertEqual((run_dir / "engine_status.txt").read_bytes(), b"9\n") + self.assertFalse((run_dir / "capture_outcome.json").exists()) + + + def test_banner_only_or_bare_accept_alone_is_not_a_witness(self): + # A configuration banner alone, or a bare acceptance marker alone, + # never yields a passing compound witness -- both refused + # independently. + cases = ( + ("banner_only", self.STDOUT.split(b"\n", 2)[0] + b"\n" + + self.STDOUT.split(b"\n", 2)[1] + b"\n"), + ("accept_alone", self.STDOUT.split(b"\n", 2)[0] + b"\n" + + self.STDOUT.split(b"\n", 2)[1] + b"\n" + b"ACCEPT 7 1\n"), + ) + for name, stdout in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["stdout"].write_bytes(stdout) + self.rebind(binding, paths) + # Whichever guard fires first (the positive DATA/ECHO + # denominator, or the downstream mtpemit/DATA-row binding + # once stderr is checked), the compound witness is refused. + with self.assertRaises(WITNESS.WitnessError): + WITNESS.validate_binding(binding) + + def test_duplicate_binding_json_keys_refuse(self): + # A duplicate key can never re-serialize to its own raw bytes, + # so the canonical-byte round-trip check refuses it -- a + # dedicated duplicate-key rejection would be unreachable dead + # code layered on top of that guarantee. + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + raw = pathlib.Path(binding).read_bytes() + assert raw.startswith(b'{"artifacts"') + doubled = b'{"schema":"stray",' + raw[1:] + pathlib.Path(binding).write_bytes(doubled) + with self.assertRaisesRegex( + WITNESS.WitnessError, "noncanonical binding JSON bytes"): + WITNESS.validate_binding(binding) + + def test_unparsed_loaded_banner_is_a_named_failure(self): + # The loaded-banner line is checked through the engine-evidence + # module's exact grammar; a line that merely resembles it but + # fails that grammar is a named WitnessError, never a silent pass. + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["stdout"].write_bytes(self.STDOUT.replace( + b"MTP ACTIVE (draft=1)", b"MTP SOMETHING-ELSE (draft=1)")) + self.rebind(binding, paths) + with self.assertRaisesRegex( + WITNESS.WitnessError, + "malformed LOADED preamble|load record is invalid"): + WITNESS.validate_binding(binding) + + + def test_engine_witness_unsupported_when_mtpemit_absent(self): + # A capture whose stderr never prints the per-emission witness + # line at all -- even though a genuine, non-stop native-MTP HIT + # is proposed -- is refused with a distinct status naming the + # limitation, never a bare failure and never a false success. + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["stderr"].write_bytes(self.STDERR.replace( + b"[mtpemit] request=7 ordinal=0 token=1\n", b"")) + self.rebind(binding, paths) + with self.assertRaisesRegex( + WITNESS.EngineWitnessUnsupported, + "engine does not emit the accepted-token witness line"): + WITNESS.validate_binding(binding) + with tempfile.TemporaryDirectory() as tmp: + _, _, binding = self.make_fixture(tmp) + self.assertEqual( + WITNESS.validate_binding(binding)["accepted_tokens"], [1]) + + def test_engine_witness_unsupported_is_distinguishable_at_the_cli(self): + # The EngineWitnessUnsupported outcome must never be mistaken for + # an ordinary INCOMPLETE witness at the CLI boundary: it needs its + # own exit status and its own stderr prefix, not the generic ones + # every other WitnessError/OSError shares. + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["stderr"].write_bytes(self.STDERR.replace( + b"[mtpemit] request=7 ordinal=0 token=1\n", b"")) + self.rebind(binding, paths) + stdout = io.StringIO() + stderr = io.StringIO() + with contextlib.redirect_stdout(stdout), \ + contextlib.redirect_stderr(stderr): + status = WITNESS.main(["validate", str(binding)]) + self.assertEqual(status, 3) + self.assertNotEqual(status, 1) + self.assertEqual(stdout.getvalue(), "") + self.assertTrue( + stderr.getvalue().startswith("[native-mtp] UNSUPPORTED: ")) + self.assertNotIn("INCOMPLETE", stderr.getvalue()) + self.assertIn( + "engine does not emit the accepted-token witness line", + stderr.getvalue()) + + def test_prof_and_hits_globals_are_recognized_but_optional(self): + without = self.STDOUT.replace(b"HITS 1 2 00\n", b"").replace( + b"PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n", b"") + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["stdout"].write_bytes(without) + self.rebind(binding, paths) + self.assertEqual( + WITNESS.validate_binding(binding)["accepted_tokens"], [1]) + frames, framing = WITNESS.GAPS.parse_frames(self.STDOUT) + self.assertFalse(framing) + kinds = {fields[0] for fields, _payload, _offset in frames} + self.assertIn(b"HITS", kinds) + self.assertIn(b"PROF", kinds) + + def test_manifest_and_walk_order_agree_on_file_beside_same_named_dir(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + (snapshot / "model.json").write_bytes(b"{}\n") + (snapshot / "model").mkdir() + (snapshot / "model" / "inner.bin").write_bytes(b"weights\n") + payloads = { + "model.json": b"{}\n", + "model/inner.bin": b"weights\n", + } + # A sorted sha256sum manifest orders by full relative-path text, + # not by bare directory-entry name: "model.json" < "model/ + # inner.bin" because "." (0x2e) sorts before "/" (0x2f), even + # though a directory walk sorted per-directory by entry name + # visits the "model" directory before the "model.json" file. + ordered = sorted(payloads) + self.assertEqual(ordered, ["model.json", "model/inner.bin"]) + container = root / "container.sha256" + container.write_bytes("".join( + f"{WITNESS._sha256_bytes(payloads[path])} {path}\n" + for path in ordered).encode("ascii")) + inventory, _ = WITNESS._apply_snapshot_manifest( + snapshot, container.read_bytes()) + self.assertEqual(inventory["files"], 2) + + def test_two_accepted_tokens_refused_by_both_subcommands(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + snapshot = root / "snapshot" + snapshot.mkdir() + binary = root / "colibri" + binary.write_bytes(b"binary") + container = root / "container.sha256" + self.write_snapshot_manifest(snapshot, container) + request = root / "submit.raw" + request.write_bytes(self.submit(b"x", maximum=4)) + run_dir = root / "run" + + data = b"DATA 7 1 -0.125 2 1 -0.125 0 -2.125\nx\n" + stdout = self.STDOUT.replace(data, data + data, 1).replace( + b"DONE 7 STAT 1 0.10", b"DONE 7 STAT 2 0.10", 1) + hit_emit = ( + b"[mtpdbg] draft0=1 verified=1 HIT\n" + b"[mtpemit] request=7 ordinal=0 token=1\n") + stderr = self.STDERR.replace( + hit_emit, + hit_emit + b"[mtpdbg] draft0=1 verified=1 HIT\n" + b"[mtpemit] request=7 ordinal=1 token=1\n") + + def fake_run(argv, **kwargs): + kwargs["stdout"].write(stdout) + kwargs["stderr"].write(stderr) + return types.SimpleNamespace(returncode=0) + + with self.assertRaisesRegex(WITNESS.WitnessError, + "exactly one accepted token"): + WITNESS.capture_bundle( + str(binary), str(snapshot), str(container), str(request), + str(run_dir), self.REQUEST_ID, self.TOPK, self.VOCAB, + run_fn=fake_run) + binding = run_dir / "binding.json" + self.assertTrue(binding.is_file()) + with self.assertRaisesRegex(WITNESS.WitnessError, + "exactly one accepted token"): + WITNESS.validate_binding(binding) + + def test_missing_done_frame_refused(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + stdout = self.STDOUT[:self.STDOUT.index(b"DONE 7 STAT")] + paths["stdout"].write_bytes(stdout) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, + "expected exactly one DONE"): + WITNESS.validate_binding(binding) + + def test_duplicate_accept_frame_refused(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + stdout = self.STDOUT.replace( + b"ACCEPT 7 1\n", b"ACCEPT 7 1\nACCEPT 7 1\n", 1) + paths["stdout"].write_bytes(stdout) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, + "expected exactly one ACCEPT"): + WITNESS.validate_binding(binding) + + def test_done_emitted_count_mismatch_refused(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + stdout = self.STDOUT.replace( + b"DONE 7 STAT 1 0.10", b"DONE 7 STAT 2 0.10", 1) + paths["stdout"].write_bytes(stdout) + self.rebind(binding, paths) + with self.assertRaisesRegex( + WITNESS.WitnessError, + "DONE emitted .* != observed DATA"): + WITNESS.validate_binding(binding) + + def test_positive_or_noncanonical_logprob_refused(self): + cases = ( + ("positive", b"-0.125 2 1 -0.125 0 -2.125", + b"0.125 2 1 -0.125 0 -2.125"), + ("noncanonical", b"-0.125 2 1 -0.125 0 -2.125", + b"-0.1250 2 1 -0.125 0 -2.125"), + ) + for name, old, new in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["stdout"].write_bytes(self.STDOUT.replace(old, new, 1)) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, + "not finite/non-positive|" + "matches neither|malformed"): + WITNESS.validate_binding(binding) + + def test_out_of_vocab_token_id_refused(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + stdout = self.STDOUT.replace( + b"DATA 7 1 -0.125 2 1 -0.125 0 -2.125", + b"DATA 7 1 -0.125 2 9 -0.125 0 -2.125", 1) + paths["stdout"].write_bytes(stdout) + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, + r"outside \[0,3\)"): + WITNESS.validate_binding(binding) + + def test_bare_hit_with_no_armed_stop_record_refused(self): + with tempfile.TemporaryDirectory() as tmp: + _, paths, binding = self.make_fixture(tmp) + paths["stderr"].write_bytes(b"[mtpdbg] draft0=1 verified=1 HIT\n") + self.rebind(binding, paths) + with self.assertRaisesRegex(WITNESS.WitnessError, + "expected one armed-stop record"): + WITNESS.validate_binding(binding) + +if __name__ == "__main__": + unittest.main() + + +class WindowsRefusalTests(unittest.TestCase): + """The witness's payload-identity check is POSIX-only and says so. + + Runs on every platform: on POSIX the platform name is patched so the + guard is exercised; on Windows the patch is a no-op and the same + assertion holds against the real platform. + """ + + def test_payload_hashing_refuses_on_windows_by_name(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + (root / "tokenizer.json").write_bytes(b"{}\n") + with mock.patch.object(sys, "platform", "win32"), \ + self.assertRaises(WITNESS.WitnessError) as caught: + WITNESS._hash_snapshot_payload(root, "tokenizer.json") + self.assertIn("POSIX stat semantics", str(caught.exception)) + self.assertIn("Windows is unsupported", str(caught.exception)) + + def test_payload_hashing_works_where_not_refused(self): + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + (root / "tokenizer.json").write_bytes(b"{}\n") + if sys.platform == "win32": + self.skipTest("refusal covered above") + digest, size = WITNESS._hash_snapshot_payload(root, "tokenizer.json") + self.assertEqual(size, 3) + self.assertEqual( + digest, WITNESS.hashlib.sha256(b"{}\n").hexdigest()) + diff --git a/c/tests/test_engine_evidence.py b/c/tests/test_engine_evidence.py new file mode 100644 index 000000000..8bcfa5a6c --- /dev/null +++ b/c/tests/test_engine_evidence.py @@ -0,0 +1,263 @@ +"""tools/engine_evidence.py must parse only exact, in-range preamble text. + +Pins the exact banner/loaded text the module accepts, the field ranges it +enforces on both sides of each bound, and the None-vs-raise split in +parse_engine_preamble, so a future edit to the shared parser cannot +silently loosen or break any of its numeric bounds or its exact-text +matching without a local, fast failure. +""" +import unittest + +from tools.engine_evidence import ( + PreambleError, + parse_engine_banner, + parse_engine_loaded, + parse_engine_preamble, +) + +_BANNER = ( + "== GLM C engine (glm_moe_dsa), cache=8 experts/layer | " + "compute experts@4-bit dense@8-bit | idot: avx2 ==" +) +_LOADED = ( + "loaded in 12.34s | resident dense: 5678.90 MB | layers=32 experts=128 " + "| MTP ACTIVE (draft=4)" +) + + +def _banner(**subs): + text = _BANNER + for old, new in subs.items(): + assert old in text, old + text = text.replace(old, new, 1) + return text + + +def _loaded(**subs): + text = _LOADED + for old, new in subs.items(): + assert old in text, old + text = text.replace(old, new, 1) + return text + + +class ParseEngineBannerTest(unittest.TestCase): + def test_exact_banner_returns_typed_fields(self): + fields = parse_engine_banner(_BANNER) + self.assertEqual(fields, { + "kind": "BANNER", "cap": 8, "expert_bits": 4, "dense_bits": 8, + "kernel": "avx2", + }) + + def test_non_string_raises(self): + with self.assertRaises(PreambleError): + parse_engine_banner(None) + + def test_unrecognized_text_raises(self): + with self.assertRaises(PreambleError): + parse_engine_banner("not a banner at all") + + def test_unknown_kernel_raises(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"idot: avx2": "idot: sse4"})) + + def test_trailing_text_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_BANNER + " extra") + + # -- cap: [1, 2**31-1] -- + + def test_cap_lower_bound_accepted(self): + fields = parse_engine_banner(_banner(**{"cache=8": "cache=1"})) + self.assertEqual(fields["cap"], 1) + + def test_cap_lower_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"cache=8": "cache=0"})) + + def test_cap_upper_bound_accepted(self): + fields = parse_engine_banner( + _banner(**{"cache=8": "cache=2147483647"})) + self.assertEqual(fields["cap"], 2147483647) + + def test_cap_upper_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"cache=8": "cache=2147483648"})) + + # -- expert_bits: [1, 16] -- + + def test_expert_bits_lower_bound_accepted(self): + fields = parse_engine_banner(_banner(**{"experts@4-bit": "experts@1-bit"})) + self.assertEqual(fields["expert_bits"], 1) + + def test_expert_bits_lower_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"experts@4-bit": "experts@0-bit"})) + + def test_expert_bits_upper_bound_accepted(self): + fields = parse_engine_banner(_banner(**{"experts@4-bit": "experts@16-bit"})) + self.assertEqual(fields["expert_bits"], 16) + + def test_expert_bits_upper_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"experts@4-bit": "experts@17-bit"})) + + # -- dense_bits: [1, 16] -- + + def test_dense_bits_lower_bound_accepted(self): + fields = parse_engine_banner(_banner(**{"dense@8-bit": "dense@1-bit"})) + self.assertEqual(fields["dense_bits"], 1) + + def test_dense_bits_lower_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"dense@8-bit": "dense@0-bit"})) + + def test_dense_bits_upper_bound_accepted(self): + fields = parse_engine_banner(_banner(**{"dense@8-bit": "dense@16-bit"})) + self.assertEqual(fields["dense_bits"], 16) + + def test_dense_bits_upper_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"dense@8-bit": "dense@17-bit"})) + + # -- leading-zero handling (no field allows a leading zero on a + # multi-digit value; a leading zero makes the whole line unrecognized, + # not merely out of range) -- + + def test_leading_zero_digit_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_banner(_banner(**{"cache=8": "cache=007"})) + + def test_no_leading_zero_digit_accepted(self): + fields = parse_engine_banner(_banner(**{"cache=8": "cache=7"})) + self.assertEqual(fields["cap"], 7) + + +class ParseEngineLoadedTest(unittest.TestCase): + def test_exact_loaded_returns_typed_fields(self): + fields = parse_engine_loaded(_LOADED) + self.assertEqual(fields, { + "kind": "LOADED", "load_s": 12.34, "resident_mb": 5678.90, + "layers": 32, "experts": 128, "mtp": "ACTIVE", "draft": 4, + }) + + def test_non_string_raises(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(1234) + + def test_unrecognized_text_raises(self): + with self.assertRaises(PreambleError): + parse_engine_loaded("not a load record") + + # -- layers: [1, 128] -- + + def test_layers_lower_bound_accepted(self): + fields = parse_engine_loaded(_loaded(**{"layers=32": "layers=1"})) + self.assertEqual(fields["layers"], 1) + + def test_layers_lower_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded(**{"layers=32": "layers=0"})) + + def test_layers_upper_bound_accepted(self): + fields = parse_engine_loaded(_loaded(**{"layers=32": "layers=128"})) + self.assertEqual(fields["layers"], 128) + + def test_layers_upper_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded(**{"layers=32": "layers=129"})) + + # -- experts: [1, 4096] -- + + def test_experts_lower_bound_accepted(self): + fields = parse_engine_loaded(_loaded(**{"experts=128": "experts=1"})) + self.assertEqual(fields["experts"], 1) + + def test_experts_lower_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded(**{"experts=128": "experts=0"})) + + def test_experts_upper_bound_accepted(self): + fields = parse_engine_loaded(_loaded(**{"experts=128": "experts=4096"})) + self.assertEqual(fields["experts"], 4096) + + def test_experts_upper_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded(**{"experts=128": "experts=4097"})) + + # -- exactly two decimal digits on load_s / resident_mb -- + + def test_two_decimal_places_accepted(self): + fields = parse_engine_loaded(_LOADED) + self.assertEqual(fields["load_s"], 12.34) + + def test_one_decimal_place_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded(**{"12.34s": "12.3s"})) + + def test_three_decimal_places_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded(**{"12.34s": "12.345s"})) + + # -- MTP / draft interaction -- + + def test_absent_mtp_allows_nonzero_draft(self): + fields = parse_engine_loaded( + _loaded(**{"MTP ACTIVE (draft=4)": "MTP absent (draft=5)"})) + self.assertEqual(fields["mtp"], "absent") + self.assertEqual(fields["draft"], 5) + + def test_active_mtp_allows_nonzero_draft(self): + fields = parse_engine_loaded(_LOADED) + self.assertEqual(fields["mtp"], "ACTIVE") + self.assertEqual(fields["draft"], 4) + + def test_draft_upper_bound_accepted(self): + fields = parse_engine_loaded(_loaded(**{"draft=4)": "draft=63)"})) + self.assertEqual(fields["draft"], 63) + + def test_draft_upper_bound_rejected(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded(**{"draft=4)": "draft=64)"})) + + def test_disabled_multiplexed_requires_zero_draft(self): + with self.assertRaises(PreambleError): + parse_engine_loaded(_loaded( + **{"MTP ACTIVE (draft=4)": + "MTP DISABLED (multiplexed serve) (draft=4)"})) + + def test_disabled_multiplexed_with_zero_draft_parses(self): + fields = parse_engine_loaded(_loaded( + **{"MTP ACTIVE (draft=4)": + "MTP DISABLED (multiplexed serve) (draft=0)"})) + self.assertEqual(fields["mtp"], "DISABLED (multiplexed serve)") + self.assertEqual(fields["draft"], 0) + + +class ParseEnginePreambleTest(unittest.TestCase): + def test_dispatches_to_banner(self): + self.assertEqual( + parse_engine_preamble(_BANNER), parse_engine_banner(_BANNER)) + + def test_dispatches_to_loaded(self): + self.assertEqual( + parse_engine_preamble(_LOADED), parse_engine_loaded(_LOADED)) + + def test_unowned_line_returns_none(self): + self.assertIsNone(parse_engine_preamble("some ordinary log line")) + + def test_banner_prefixed_but_malformed_still_raises(self): + with self.assertRaises(PreambleError): + parse_engine_preamble("== GLM C engine but garbled ==") + + def test_loaded_prefixed_but_malformed_still_raises(self): + with self.assertRaises(PreambleError): + parse_engine_preamble("loaded in not a valid record") + + def test_non_string_raises(self): + with self.assertRaises(PreambleError): + parse_engine_preamble(3.14) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_eval_glm.py b/c/tests/test_eval_glm.py new file mode 100644 index 000000000..e8d90908f --- /dev/null +++ b/c/tests/test_eval_glm.py @@ -0,0 +1,979 @@ +"""tools/eval_glm.py must accept only the engine's real SCORE stdout +records and refuse everything else with a named error: an unparsed or +duplicated banner/load preamble, a SCORE line whose numeric token is not +the canonical finite ``%.6f``/``%.17g`` spelling the engine actually +emits, a replayed or out-of-order identity-bound evidence record, and a +foreign stdout line that is neither a preamble nor a SCORE record. It +must also write result rows incrementally (one flush per request, never +buffered until completion) and mark incomplete runs, including a +pre-launch refusal, before the engine is ever started. + +Checks enumerated from the source (`tools/eval_glm.py`, read in full +before writing this module) and covered below, grouped by the function +that performs them: + +- `parse_c17g`: both exact finite spellings the engine actually emits -- + the shipped ``%.6f`` form and the newer ``%.17g`` evidence form -- and + nothing else; this module imports nothing from and shares no code with + `check_data_logprob_gaps.py`, which parses its own independent grammar. +- `parse_score_result` / `_SCORE_RE`: the shipped ``%.6f`` form AND the + ``%.17g`` evidence form (both numeric forms), non-finite/malformed + rejection, and the `` `` metadata bounds. +- `classify_score_stdout` / `ScoreStdoutClassifier`: the banner/load + preamble lifecycle (missing, duplicated, out-of-order, or a SCORE + record before the load record all refuse); every line that is not an + exact banner, an exact load record, or an exact SCORE record refuses + by name -- a foreign line is never silently treated as a score, which + is exactly the defect dev's plain ``line[0] in "-0123456789"`` filter + does not catch (differential bite, below). +- identity-bound evidence mode (``ScoreStdoutClassifier(request_digests)``): + strict ordinal join, digest binding, replay/duplicate/out-of-order/ + extra-record refusal; a stream that mixes identity-bound and legacy + records refuses by name; a legacy-only stream still completes, marked + UNBOUND rather than silently treated as bound. +- `score_request_wire`: strict ASCII/LF request grammar, the per-record + SHA-256 digest, and the inclusive 256 MiB engine text limit shared with + `check_ablate_evidence.py`. +- `completion_error`: the exact zero-exit/complete-count/positive-token + denominator that alone passes. +- `main`: incremental durability (one written+flushed row per completed + request, never buffered until the run ends) and pre-launch INCOMPLETE + marking (no benchmark tasks selected; zero SCORE requests produced; + every choice's context/continuation split is empty) -- the engine is + never launched for any of these; a mid-run crash or interrupt still + leaves the INCOMPLETE marker and terminates the child process; a + partial run still prints the accuracy table over whatever rows landed. + +Deferred (need a live binary this module does not have access to): +- `test_c_emitted_c17g_corpus_is_canonical`, which drives + ``test_logprob_wire --score-c17g-fixture`` (a binary produced by a + different part of this project's build, not present here). +- the ABLATE-block stdout probes (`c_manifest_accepts`/ + `test_complete_production_fixture_is_strict_json`-style checks): out of + scope for this module (owned by `test_check_ablate_evidence.py`). + +No model is run by the committed tests -- every case here drives +`eval_glm.py` against an injected stand-in for the direct engine launch +(a fake ``subprocess.Popen`` returning canned stdout/stderr), never a +real ``./glm`` process. +""" + +import contextlib +import hashlib +import importlib.util +import io +import json +import os +import pathlib +import signal +import sys +import tempfile +import types +import unittest +from unittest import mock + + +HERE = pathlib.Path(__file__).resolve().parent +TOOLS = HERE.parent / "tools" + +_spec = importlib.util.spec_from_file_location( + "eval_glm_under_test", TOOLS / "eval_glm.py") +EVAL = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(EVAL) + +_ablate_spec = importlib.util.spec_from_file_location( + "check_ablate_evidence_under_test", TOOLS / "check_ablate_evidence.py") +ABLATE = importlib.util.module_from_spec(_ablate_spec) +_ablate_spec.loader.exec_module(ABLATE) + + +class EvalGlmEvidenceTests(unittest.TestCase): + BANNER = ( + "== GLM C engine (glm_moe_dsa), cache=64 experts/layer | " + "compute experts@4-bit dense@8-bit | idot: neon-i8mm ==") + CONFIG = (b'{"vocab_size":4,"num_hidden_layers":4,' + b'"first_k_dense_replace":1,"n_routed_experts":5}\n') + + @staticmethod + def loaded(state="ACTIVE", draft=1, layers=78, experts=256, + load="1.00", resident="1.00"): + return (f"loaded in {load}s | resident dense: {resident} MB | " + f"layers={layers} experts={experts} | MTP {state} " + f"(draft={draft})") + + def run_eval_main(self, stdout_records, bind_evidence=True): + class Encoded: + ids = [1, 2] + + class FakeTokenizer: + @staticmethod + def from_file(path): + return FakeTokenizer() + + @staticmethod + def encode(text): + return Encoded() + + request_raw = b"2 2 1 2 1 2\n" + request_digest = hashlib.sha256(request_raw).hexdigest() + bound_records = [] + score_index = 0 + for record in stdout_records: + line = record[:-1] if record.endswith("\n") else record + if bind_evidence and EVAL._SCORE_RE.fullmatch(line): + record = f"SCORE {score_index} {request_digest} {line}\n" + score_index += 1 + bound_records.append(record) + process = types.SimpleNamespace( + returncode=0, stderr=(), stdout=tuple(bound_records), + wait=lambda: 0, poll=lambda: 0, terminate=lambda: None) + with tempfile.TemporaryDirectory() as tmp: + output = pathlib.Path(tmp) / "results.csv" + (pathlib.Path(tmp) / "config.json").write_text( + '{"vocab_size":3}\n') + argv = [ + "eval_glm.py", "--snap", tmp, "--tasks", "smoke", + "--limit", "1", "--glm", "/fake/glm", "--out", + str(output), + ] + tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer) + stderr_buf = io.StringIO() + stdout_buf = io.StringIO() + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \ + mock.patch.object(EVAL.subprocess, "Popen", + return_value=process) as popen, \ + contextlib.redirect_stderr(stderr_buf), \ + contextlib.redirect_stdout(stdout_buf): + rc = EVAL.main() + self.last_popen_kwargs = popen.call_args.kwargs if popen.called else {} + self.last_stderr = stderr_buf.getvalue() + self.last_stdout = stdout_buf.getvalue() + return rc, output.read_text(), popen.call_count + + def test_exact_score_text_survives_csv(self): + text = "-8.5534581234567888" + exact, value, contlen, greedy = EVAL.parse_score_result( + f"{text} 4096 1") + self.assertEqual(exact, text) + self.assertEqual(contlen, 4096) + self.assertEqual(greedy, 1) + self.assertTrue(value < 0) + + out = io.StringIO() + meta = ("task", 3, 2, 4096, 17, 2) + EVAL.write_result_row(out, 9, meta, exact, greedy) + self.assertEqual( + out.getvalue(), + "9,task,3,2,4096,17,2,-8.5534581234567888,1\n") + + def test_shipped_dot6f_form_is_still_accepted(self): + # The tool must accept BOTH the shipped %.6f form the engine + # actually prints today AND the %.17g evidence form. + exact, value, contlen, greedy = EVAL.parse_score_result( + "-8.553458 4096 1") + self.assertEqual(exact, "-8.553458") + self.assertEqual(value, -8.553458) + self.assertEqual((contlen, greedy), (4096, 1)) + + def test_score_request_digest_binds_strict_ascii_bytes_including_lf(self): + requests = ("1 1 1 2", "2 1 0 1 2") + lines, payload, digests = EVAL.score_request_wire(requests, 3) + self.assertEqual(lines, ( + b"1 1 1 2\n", b"2 1 0 1 2\n")) + self.assertEqual(payload, b"".join(lines)) + self.assertEqual( + digests, + tuple(hashlib.sha256(line).hexdigest() for line in lines)) + self.assertNotEqual( + digests[1], + hashlib.sha256(b"2 1 0 1 2").hexdigest()) + bad_requests = ( + "", "two\nlines", "cr\rline", "1 1", "1 1 0", + "1 1 0 1 2", "1 1 0 1 junk", "1 1 0 3", + "1 1 -1 1", "1 1 00 1", "2147483648 1 0 1", + "1 2147483647 0 1", "evidence-μ", + ) + for bad in bad_requests: + with self.subTest(bad=bad): + with self.assertRaises(EVAL.EvidenceError): + EVAL.score_request_wire((bad,), 3) + with self.assertRaises(EVAL.EvidenceError): + EVAL.score_request_wire((), 3) + + def test_nonfinite_and_malformed_scores_refuse(self): + bad = [ + "nan 1 1", "inf 1 1", "-inf 1 1", "-1 1", "-1 1 1 extra", + "not-a-number 1 1", "-1 -1 1", "-1 0 1", "-1 1 2", "1 1 1", + "-1 1 1", "-1\t1 1", "-1 01 1", "-1_0 1 1", + "-1 2147483648 1", "+0 1 0", "-01 1 0", + "-0_125 1 0", "-١ 1 0", "-1e-9999 1 0", + "-1.00000000000000000 1 0", "-1e-9 1 0", + "-1e--09 1 0", "-1e+009 1 0", "-1.25 1 0 junk", + ] + for line in bad: + with self.subTest(line=line): + with self.assertRaises(EVAL.EvidenceError): + EVAL.parse_score_result(line) + + def test_stdout_grammar_refuses_unknown_records(self): + self.assertIsNone(EVAL.classify_score_stdout(self.BANNER + "\n")) + for state, draft in (("ACTIVE", 0), ("ACTIVE", 1), + ("absent", 0), ("absent", 2), + ("DISABLED (multiplexed serve)", 0)): + with self.subTest(state=state, draft=draft): + self.assertIsNone(EVAL.classify_score_stdout( + self.loaded(state, draft) + "\n")) + exact, _, _, _ = EVAL.classify_score_stdout("-1.25 1 0\n") + self.assertEqual(exact, "-1.25") + bad = ( + "\n", "unexpected banner\n", "nan 1 0\n", "inf 1 0\n", + " -1.25 1 0\n", "-1.25 1 0 \n", "-1.25 1 0\t\n", + "-1.25 1 0", "-1.25 1 0\r\n", + "PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n", + "DONE 7 STAT 1 1.00 0.0 1.00 1 0\n", + "== GLM C engine fabricated ==\n", + "== GLM C engine (glm_moe_dsa), cache=0 experts/layer | " + "compute experts@4-bit dense@8-bit | idot: neon ==\n", + "== GLM C engine (glm_moe_dsa), cache=064 experts/layer | " + "compute experts@4-bit dense@8-bit | idot: neon ==\n", + "loaded in 1.0s | resident dense: 1.00 MB | layers=78 experts=256 | " + "MTP ACTIVE (draft=1)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | " + "MTP DISABLED (multiplexed serve) (draft=1)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | " + "MTP unknown (draft=0)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=256 | " + "MTP ACTIVE (draft=64)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=0 experts=256 | " + "MTP ACTIVE (draft=1)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=129 experts=256 | " + "MTP ACTIVE (draft=1)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=0 | " + "MTP ACTIVE (draft=1)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=78 experts=4097 | " + "MTP ACTIVE (draft=1)\n", + "loaded in -0.01s | resident dense: 1.00 MB | layers=78 experts=256 | " + "MTP ACTIVE (draft=1)\n", + "loaded in nan s | resident dense: 1.00 MB | layers=78 experts=256 | " + "MTP ACTIVE (draft=1)\n", + "loaded in 1.0s | resident dense: 1.00 MB | layers=78 experts=256 | " + "MTP ACTIVE (draft=1)\n", + "loaded in 1.00s | resident dense: 1.00 MB | layers=2147483648 experts=256 | " + "MTP ACTIVE (draft=1)\n", + " == GLM C engine (glm_moe_dsa), cache=64 experts/layer | " + "compute experts@4-bit dense@8-bit | idot: neon ==\n", + ) + for line in bad: + with self.subTest(line=line): + with self.assertRaises(EVAL.EvidenceError): + EVAL.classify_score_stdout(line) + + def test_banner_kernels_and_load_boundaries_are_exact(self): + self.assertEqual( + EVAL.parse_engine_banner(self.BANNER)["kernel"], "neon-i8mm") + for kernel in ("avx512-vnni", "avx-vnni", "avx2", "neon-i8mm", + "neon", "vsx", "scalar"): + line = self.BANNER.replace("neon-i8mm", kernel) + self.assertEqual(EVAL.parse_engine_banner(line)["kernel"], kernel) + with self.assertRaises(EVAL.PreambleError): + EVAL.parse_engine_banner(self.BANNER.replace("neon-i8mm", "fabricated")) + + for layers, experts in ((1, 1), (128, 4096)): + parsed = EVAL.parse_engine_loaded(self.loaded( + layers=layers, experts=experts)) + self.assertEqual((parsed["layers"], parsed["experts"]), + (layers, experts)) + + def test_score_stream_owns_banner_load_lifecycle(self): + parser = EVAL.ScoreStdoutClassifier() + self.assertIsNone(parser.classify(self.BANNER + "\n")) + self.assertIsNone(parser.classify(self.loaded("absent", 2) + "\n")) + exact, _, _, _ = parser.classify("-1.25 1 0\n") + self.assertEqual(exact, "-1.25") + parser.finish() + + cases = ( + [self.loaded() + "\n", self.BANNER + "\n"], + [self.BANNER + "\n", "-1.25 1 0\n"], + [self.BANNER + "\n", self.BANNER + "\n"], + [self.BANNER + "\n", self.loaded() + "\n", + self.loaded() + "\n"], + [self.BANNER + "\n"], + ) + for records in cases: + with self.subTest(records=records): + parser = EVAL.ScoreStdoutClassifier() + with self.assertRaises(EVAL.EvidenceError): + for record in records: + parser.classify(record) + parser.finish() + + def test_eval_main_uses_stateful_score_stream(self): + rc, output, launches = self.run_eval_main(( + self.BANNER + "\n", + self.loaded("absent", 2) + "\n", + "-1 2 1\n", "-2 2 0\n", "-3 2 0\n", + )) + self.assertEqual(rc, 0) + self.assertEqual(launches, 1) + self.assertIn("# finished: 3/3", output) + self.assertEqual(self.last_popen_kwargs["env"]["SCORE_EVIDENCE"], "1") + + def test_result_rows_are_written_and_flushed_incrementally(self): + # A run interrupted mid-task must leave a valid partial file -- + # rows are written and flushed per-request, never buffered until + # the run completes. Simulate a mid-run crash by having the fake + # engine's stdout iterator raise after the first scored record; + # the CSV must already contain that row. + request_raw = b"2 2 1 2 1 2\n" + request_digest = hashlib.sha256(request_raw).hexdigest() + + class Encoded: + ids = [1, 2] + + class FakeTokenizer: + @staticmethod + def from_file(path): + return FakeTokenizer() + + @staticmethod + def encode(text): + return Encoded() + + class CrashingStdout: + def __init__(self, lines): + self._lines = list(lines) + + def __iter__(self): + for index, line in enumerate(self._lines): + yield line + if index == 2: # after banner+load+one SCORE row + raise OSError("engine died mid-run") + + stdout_lines = [ + self.BANNER + "\n", self.loaded("absent", 2) + "\n", + f"SCORE 0 {request_digest} -1 2 1\n", + f"SCORE 1 {request_digest} -2 2 0\n", + ] + process = types.SimpleNamespace( + returncode=1, stderr=(), stdout=CrashingStdout(stdout_lines), + wait=lambda: 1, poll=lambda: 1, terminate=lambda: None) + with tempfile.TemporaryDirectory() as tmp: + output = pathlib.Path(tmp) / "results.csv" + (pathlib.Path(tmp) / "config.json").write_text( + '{"vocab_size":3}\n') + argv = [ + "eval_glm.py", "--snap", tmp, "--tasks", "smoke", + "--limit", "1", "--glm", "/fake/glm", "--out", + str(output), + ] + tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer) + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \ + mock.patch.object(EVAL.subprocess, "Popen", + return_value=process), \ + self.assertRaises(OSError): + EVAL.main() + text = output.read_text() + self.assertIn(",-1,1\n", text.replace(".000000", "")) + self.assertNotIn("# finished:", text) + # The crash-path marker itself must be present, not + # just the absence of "# finished:" -- a downstream consumer + # scans for this exact line to know the run never reached a + # complete denominator. + self.assertIn( + "# INCOMPLETE: evaluator terminated before a complete " + "denominator", text) + + def test_child_is_terminated_on_mid_run_interrupt_or_exception(self): + # SIGINT/SIGTERM/any exception mid-run must not leave the + # engine child running. The fake engine here never exits on its + # own -- .poll() keeps returning None (as a real child that + # ignores its stdin being closed would) until .terminate() is + # actually called -- so a passing test proves main() called + # terminate() itself rather than relying on the child to die. + request_raw = b"2 2 1 2 1 2\n" + request_digest = hashlib.sha256(request_raw).hexdigest() + + class Encoded: + ids = [1, 2] + + class FakeTokenizer: + @staticmethod + def from_file(path): + return FakeTokenizer() + + @staticmethod + def encode(text): + return Encoded() + + banner = self.BANNER + "\n" + loaded = self.loaded("absent", 2) + "\n" + + def interrupting_stdout(): + yield banner + yield loaded + yield f"SCORE 0 {request_digest} -1 2 1\n" + raise KeyboardInterrupt("operator pressed Ctrl+C") + + class NeverExitingProcess: + def __init__(self): + self.returncode = None + self.stderr = () + self.stdout = interrupting_stdout() + self.terminated = False + self.terminate_calls = 0 + self.wait_calls = 0 + + def poll(self): + return 0 if self.terminated else None + + def terminate(self): + self.terminated = True + self.terminate_calls += 1 + + def wait(self): + self.wait_calls += 1 + return 0 + + process = NeverExitingProcess() + with tempfile.TemporaryDirectory() as tmp: + output = pathlib.Path(tmp) / "results.csv" + (pathlib.Path(tmp) / "config.json").write_text( + '{"vocab_size":3}\n') + argv = [ + "eval_glm.py", "--snap", tmp, "--tasks", "smoke", + "--limit", "1", "--glm", "/fake/glm", "--out", + str(output), + ] + tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer) + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \ + mock.patch.object(EVAL.subprocess, "Popen", + return_value=process), \ + self.assertRaises(KeyboardInterrupt): + EVAL.main() + self.assertEqual(process.terminate_calls, 1) + self.assertGreaterEqual(process.wait_calls, 1) + + @unittest.skipIf( + sys.platform == "win32", + "os.kill(pid, SIGTERM) on Windows is TerminateProcess with exit code 15: " + "no handler runs, so the POSIX mechanism under test does not exist there " + "(the SIGINT/exception half above covers child cleanup on Windows)") + def test_sigterm_mid_run_terminates_the_child_and_propagates(self): + # The SIGTERM half: a real SIGTERM (not just an ordinary + # Python exception) delivered while the child is running must + # also be converted into child cleanup, not left to Python's + # default SIGTERM handling (which does not run this module's + # `finally` cleanup at all). + request_raw = b"2 2 1 2 1 2\n" + request_digest = hashlib.sha256(request_raw).hexdigest() + + class Encoded: + ids = [1, 2] + + class FakeTokenizer: + @staticmethod + def from_file(path): + return FakeTokenizer() + + @staticmethod + def encode(text): + return Encoded() + + banner = self.BANNER + "\n" + loaded = self.loaded("absent", 2) + "\n" + + def stdout_then_sigterm(): + yield banner + yield loaded + yield f"SCORE 0 {request_digest} -1 2 1\n" + os.kill(os.getpid(), signal.SIGTERM) + # Not reached if the handler fires promptly, as it must. + yield f"SCORE 1 {request_digest} -2 2 0\n" + + class NeverExitingProcess: + def __init__(self): + self.returncode = None + self.stderr = () + self.stdout = stdout_then_sigterm() + self.terminated = False + self.terminate_calls = 0 + self.wait_calls = 0 + + def poll(self): + return 0 if self.terminated else None + + def terminate(self): + self.terminated = True + self.terminate_calls += 1 + + def wait(self): + self.wait_calls += 1 + return 0 + + process = NeverExitingProcess() + previous_handler = signal.getsignal(signal.SIGTERM) + try: + with tempfile.TemporaryDirectory() as tmp: + output = pathlib.Path(tmp) / "results.csv" + (pathlib.Path(tmp) / "config.json").write_text( + '{"vocab_size":3}\n') + argv = [ + "eval_glm.py", "--snap", tmp, "--tasks", "smoke", + "--limit", "1", "--glm", "/fake/glm", "--out", + str(output), + ] + tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer) + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \ + mock.patch.object(EVAL.subprocess, "Popen", + return_value=process), \ + self.assertRaises(EVAL.ChildTerminateRequested): + EVAL.main() + finally: + # Defensive: main()'s own finally already restores the prior + # handler, but never trust a test to leave process-global + # signal state behind if the assertion above ever fails. + signal.signal(signal.SIGTERM, previous_handler) + self.assertEqual(process.terminate_calls, 1) + self.assertGreaterEqual(process.wait_calls, 1) + self.assertEqual(signal.getsignal(signal.SIGTERM), previous_handler) + + def test_identity_bound_score_join_rejects_replay_and_order_mutations(self): + requests = (b"1 1 1 2\n", b"1 1 1 3\n", b"1 1 1 4\n") + digests = tuple(hashlib.sha256(raw).hexdigest() + for raw in requests) + + def record(ordinal, digest_index, score="-1"): + return (f"SCORE {ordinal} {digests[digest_index]} " + f"{score} 1 1\n") + + control = EVAL.ScoreStdoutClassifier(digests) + self.assertIsNone(control.classify(self.BANNER + "\n")) + self.assertIsNone(control.classify(self.loaded("absent", 2) + "\n")) + for index in range(3): + self.assertEqual(control.classify(record(index, index))[0], "-1") + control.finish() + + cases = { + "replay_digest": (record(0, 0), record(1, 0)), + "duplicate_ordinal": (record(0, 0), record(0, 1)), + "out_of_order": (record(1, 1),), + # NOTE: a bare "-1 1 1\n" (no identity prefix) with no prior + # bound record is no longer an error here -- that is the + # legitimate legacy/UNBOUND path, covered by + # test_legacy_engine_completes_unbound below. Mix a legacy + # line into an ALREADY-bound stream instead, which is still + # refused (test_mixed_bound_and_legacy_stream_refuses). + "extra": (record(0, 0), record(1, 1), record(2, 2), + record(3, 2)), + } + for name, records in cases.items(): + with self.subTest(name=name): + parser = EVAL.ScoreStdoutClassifier(digests) + parser.classify(self.BANNER + "\n") + parser.classify(self.loaded("absent", 2) + "\n") + with self.assertRaises(EVAL.EvidenceError): + for value in records: + parser.classify(value) + parser.finish() + + def test_wrong_digest_at_a_correct_ordinal_is_refused_mid_stream(self): + # Isolates digest binding from the record-count completeness + # check above: exactly len(digests) records land at the right + # ordinals (so a count-only bug would stay quiet), but the + # second record's digest belongs to a different request. + requests = (b"1 1 1 2\n", b"1 1 1 3\n", b"1 1 1 4\n") + digests = tuple(hashlib.sha256(raw).hexdigest() + for raw in requests) + + def record(ordinal, digest_index, score="-1"): + return (f"SCORE {ordinal} {digests[digest_index]} " + f"{score} 1 1\n") + + parser = EVAL.ScoreStdoutClassifier(digests) + parser.classify(self.BANNER + "\n") + parser.classify(self.loaded("absent", 2) + "\n") + parser.classify(record(0, 0)) + with self.assertRaisesRegex( + EVAL.EvidenceError, + "digest does not match exact request bytes"): + parser.classify(record(1, 2)) # ordinal 1, wrong digest (index 2) + + def test_legacy_engine_completes_unbound(self): + # An engine that never emits the identity-bound + # "SCORE ..." prefix -- only + # the byte-compatible legacy three-field form -- is not a + # failure. The run completes, every row is written, and the + # result is marked UNBOUND (never silently treated as bound). + rc, output, launches = self.run_eval_main(( + self.BANNER + "\n", + self.loaded("absent", 2) + "\n", + "-1 2 1\n", "-2 2 0\n", "-3 2 0\n", + ), bind_evidence=False) + self.assertEqual(rc, 0) + self.assertEqual(launches, 1) + self.assertIn("# finished: 3/3", output) + self.assertIn("evidence=UNBOUND", output) + self.assertNotIn("evidence=BOUND", output) + # SCORE_EVIDENCE is still set for the child -- harmless to an + # engine that never reads it (confirmed: dev's run_score has no + # getenv("SCORE_EVIDENCE") call at all). + self.assertEqual(self.last_popen_kwargs["env"]["SCORE_EVIDENCE"], "1") + # The UNBOUND stderr announcement must actually be + # printed, not just the output-file marker -- an operator + # watching a live run only sees stderr. + self.assertIn( + "engine does not emit score evidence lines; results are unbound", + self.last_stderr) + + def test_evidence_engine_completes_bound(self): + rc, output, launches = self.run_eval_main(( + self.BANNER + "\n", + self.loaded("absent", 2) + "\n", + "-1 2 1\n", "-2 2 0\n", "-3 2 0\n", + ), bind_evidence=True) + self.assertEqual(rc, 0) + self.assertEqual(launches, 1) + self.assertIn("# finished: 3/3", output) + self.assertIn("evidence=BOUND", output) + self.assertNotIn("evidence=UNBOUND", output) + # A bound run must never print the unbound announcement. + self.assertNotIn("results are unbound", self.last_stderr) + + def test_mixed_bound_and_legacy_stream_refuses(self): + requests = (b"1 1 1 2\n", b"1 1 1 3\n") + digests = tuple(hashlib.sha256(raw).hexdigest() + for raw in requests) + parser = EVAL.ScoreStdoutClassifier(digests) + parser.classify(self.BANNER + "\n") + parser.classify(self.loaded("absent", 2) + "\n") + parser.classify(f"SCORE 0 {digests[0]} -1 1 1\n") # bound + with self.assertRaisesRegex( + EVAL.EvidenceError, + "mixes identity-bound and legacy records"): + parser.classify("-2 1 0\n") # legacy, mid-stream switch + + # Also refused the other way around: legacy first, then bound. + parser2 = EVAL.ScoreStdoutClassifier(digests) + parser2.classify(self.BANNER + "\n") + parser2.classify(self.loaded("absent", 2) + "\n") + parser2.classify("-2 1 0\n") # legacy + with self.assertRaisesRegex( + EVAL.EvidenceError, + "mixes identity-bound and legacy records"): + parser2.classify(f"SCORE 1 {digests[1]} -1 1 1\n") # bound + + def test_digest_bound_classifier_refuses_unknown_lines(self): + # ScoreStdoutClassifier is the class main() actually + # constructs with request_digests -- classify_score_stdout (the + # standalone function, covered by test_stdout_grammar_refuses_ + # unknown_records) is a separate code path main() never calls. + # A foreign line must be refused by the digest-bound classifier + # itself, not merely by the standalone function. + requests = (b"1 1 1 2\n",) + digests = tuple(hashlib.sha256(raw).hexdigest() for raw in requests) + foreign_lines = ( + "PROF 0.001 1 1 0.000 0.000 0.000 0.000 0.000 1\n", + "not a score line at all\n", + "nan 1 0\n", + "1 1 1\n", # positive logprob, shaped like a legacy record + ) + for line in foreign_lines: + with self.subTest(line=line): + parser = EVAL.ScoreStdoutClassifier(digests) + parser.classify(self.BANNER + "\n") + parser.classify(self.loaded("absent", 2) + "\n") + with self.assertRaises(EVAL.EvidenceError): + parser.classify(line) + + def test_eval_main_rejects_lifecycle_and_blank_records(self): + banner = self.BANNER + "\n" + loaded = self.loaded("absent", 2) + "\n" + scores = ("-1 2 1\n", "-2 2 0\n", "-3 2 0\n") + cases = { + "load_before_banner": (loaded, banner) + scores, + "score_before_load": (banner, scores[0], loaded) + scores[1:], + "duplicate_load": (banner, loaded, loaded) + scores, + "missing_load_at_eof": (banner,), + "blank_before_banner": ("\n", banner, loaded) + scores, + "blank_between_preambles": (banner, "\n", loaded) + scores, + "blank_after_scores": (banner, loaded) + scores + ("\n",), + } + for name, records in cases.items(): + with self.subTest(name=name): + rc, output, launches = self.run_eval_main(records) + self.assertEqual(rc, 1) + self.assertEqual(launches, 1) + self.assertIn("# INCOMPLETE:", output) + self.assertNotIn("# finished:", output) + + def test_only_complete_zero_exit_denominator_passes(self): + # NOTE: completion_error's contract changed from the original + # ported oracle -- it now matches dev's own exit-code contract + # exactly (a partial or nonzero-exit-but-nonempty run is no + # longer fatal here; see its docstring), so several of the + # original oracle's assertions below are inverted rather than + # reused verbatim. + self.assertIsNone(EVAL.completion_error(0, 3, 3, 9)) + # A clean exit with zero requests scored is no longer flagged by + # completion_error itself (dev's own contract only fires on a + # NONZERO exit with zero scored; `expected`/`continuation_tokens` + # are not otherwise consulted). + self.assertIsNone(EVAL.completion_error(0, 0, 0, 0)) + self.assertIsNone(EVAL.completion_error(0, 3, 3, 0)) + # A nonzero exit that still scored at least one request is a + # partial run, not fatal. + self.assertIsNone(EVAL.completion_error(2, 7, 7, 7)) + self.assertIsNone(EVAL.completion_error(0, 6, 7, 6)) + # Fatal only when the engine exits nonzero with NOTHING scored... + self.assertIsNotNone(EVAL.completion_error(2, 0, 7, 0)) + self.assertIn("zero requests scored", EVAL.completion_error(2, 0, 7, 0)) + # ...or a stream_error is present regardless of completion count. + self.assertIn("broken", EVAL.completion_error(0, 7, 7, 7, "broken")) + self.assertIn("broken", EVAL.completion_error(0, 0, 7, 0, "broken")) + + def test_empty_selection_refuses_before_engine_launch(self): + with tempfile.TemporaryDirectory() as tmp: + out = pathlib.Path(tmp) / "results.csv" + argv = ["eval_glm.py", "--snap", tmp, "--tasks", "", "--out", str(out)] + with mock.patch.object(sys, "argv", argv), \ + mock.patch.object(EVAL.subprocess, "Popen") as popen: + rc = EVAL.main() + self.assertEqual(rc, 1) + popen.assert_not_called() + text = out.read_text() + self.assertIn("# INCOMPLETE: 0/0; error=no benchmark tasks selected", text) + self.assertNotIn("finished: 0/0", text) + + def test_zero_request_task_refuses_before_engine_launch(self): + class FakeTokenizer: + @staticmethod + def from_file(path): + return object() + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + (root / "empty.jsonl").write_text("") + out = root / "results.csv" + argv = ["eval_glm.py", "--snap", tmp, "--data", tmp, + "--tasks", "empty", "--out", str(out)] + fake = types.SimpleNamespace(Tokenizer=FakeTokenizer) + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict(sys.modules, {"tokenizers": fake}), \ + mock.patch.object(EVAL.subprocess, "Popen", + side_effect=AssertionError("engine launched")) as popen: + rc = EVAL.main() + self.assertEqual(rc, 1) + popen.assert_not_called() + text = out.read_text() + self.assertIn( + "# INCOMPLETE: 0/0; error=selected tasks produced zero SCORE requests", + text) + self.assertNotIn("finished: 0/0", text) + + def test_zero_continuation_choices_refuse_before_engine_launch(self): + class Encoded: + def __init__(self, ids): + self.ids = ids + + class BoundaryTokenizer: + @staticmethod + def from_file(path): + return BoundaryTokenizer() + + @staticmethod + def encode(text): + return Encoded({ + "ctx": [1], "ctxgood": [1, 2], "good": [2], + "ctxvanish": [1], "vanish": [], "": [], + }.get(text, [1])) + + cases = { + "one_empty": [{"ctx": "ctx", "choices": [""], "gold": 0}], + "all_empty": [{"ctx": "ctx", "choices": ["", ""], "gold": 0}], + "boundary_still_empty": [ + {"ctx": "ctx", "choices": ["vanish"], "gold": 0}], + "mixed_positive_zero": [ + {"ctx": "ctx", "choices": ["good", ""], "gold": 0}], + } + tokenizer = BoundaryTokenizer() + for name, docs in cases.items(): + with self.subTest(name=name), \ + self.assertRaisesRegex(EVAL.EvidenceError, + "no positive context/continuation"): + EVAL.build_requests(tokenizer, {"task": docs}) + + tokenizers = types.SimpleNamespace(Tokenizer=BoundaryTokenizer) + for name, docs in cases.items(): + with self.subTest(prelaunch=name), \ + tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + data = root / f"{name}.jsonl" + data.write_text(json.dumps(docs[0]) + "\n") + out = root / "results.csv" + argv = ["eval_glm.py", "--snap", str(root), + "--data", str(root), "--tasks", name, + "--out", str(out)] + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict( + sys.modules, {"tokenizers": tokenizers}), \ + mock.patch.object( + EVAL.subprocess, "Popen", + side_effect=AssertionError( + "engine launched")) as popen: + rc = EVAL.main() + self.assertEqual(rc, 1) + popen.assert_not_called() + text = out.read_text() + self.assertIn("# INCOMPLETE: 0/0; error=", text) + self.assertNotIn("# finished:", text) + + def test_dry_run_does_not_require_a_vocabulary(self): + # dev's own --dry never looked up config.json's + # vocab_size at all -- it only builds and tokenizes requests, + # then stops. This module's vocabulary/digest binding is a + # per-request-wire step for the real engine launch, not a + # plumbing check, so --dry must not depend on it. + class Encoded: + ids = [1, 2] + + class FakeTokenizer: + @staticmethod + def from_file(path): + return FakeTokenizer() + + @staticmethod + def encode(text): + return Encoded() + + with tempfile.TemporaryDirectory() as tmp: + # No config.json at all in the snapshot directory -- a real + # vocabulary lookup would raise EvidenceError immediately. + argv = ["eval_glm.py", "--snap", tmp, "--tasks", "smoke", + "--limit", "1", "--dry"] + tokenizers = types.SimpleNamespace(Tokenizer=FakeTokenizer) + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict(sys.modules, {"tokenizers": tokenizers}), \ + mock.patch.object( + EVAL, "score_snapshot_vocab", + side_effect=AssertionError( + "vocabulary looked up during --dry")) as vocab, \ + mock.patch.object( + EVAL.subprocess, "Popen", + side_effect=AssertionError("engine launched")) as popen: + rc = EVAL.main() + self.assertIsNone(rc) + vocab.assert_not_called() + popen.assert_not_called() + + def test_partial_run_still_reports_the_table_and_exits_zero(self): + # dev's own exit-code contract (coli bench does + # `sys.exit(subprocess.call(cmd, ...))`; diag_harness.py parses + # this tool's stdout table from a subprocess call) exits nonzero + # ONLY when the engine produced nothing at all. A partial run -- + # some but not all requests scored, clean stream -- must still + # print the accuracy table and exit 0; the INCOMPLETE marker is + # additive (alongside "# finished", not instead of it). + rc, output, launches = self.run_eval_main(( + self.BANNER + "\n", + self.loaded("absent", 2) + "\n", + "-1 2 1\n", # only 1 of 3 requests scored + ), bind_evidence=False) + self.assertEqual(rc, 0) + self.assertEqual(launches, 1) + self.assertIn("# finished: 1/3", output) + self.assertIn("# INCOMPLETE: 1/3 requests scored", output) + self.assertIn( + "WARNING: only 1/3 requests scored", self.last_stderr) + self.assertIn("MEAN acc_norm", self.last_stdout) + + def test_python_engine_byte_limits_are_inclusive_and_preallocation(self): + # The 256 MiB inclusive engine-text limit is shared between + # this module and check_ablate_evidence.py; checked on both + # sides of the shared constant/helper. + engine_limit = 256 << 20 + self.assertEqual(ABLATE._ENGINE_TEXT_MAX_BYTES, engine_limit) + self.assertEqual(EVAL._ENGINE_TEXT_MAX_BYTES, engine_limit) + self.assertEqual( + ABLATE._checked_engine_text_size(engine_limit, "config"), + engine_limit) + self.assertEqual( + EVAL._checked_engine_text_size(engine_limit, "SCORE"), + engine_limit) + with self.assertRaises(ABLATE.AblateEvidenceError): + ABLATE._checked_engine_text_size(engine_limit + 1, "config") + with self.assertRaises(EVAL.EvidenceError): + EVAL._checked_engine_text_size(engine_limit + 1, "SCORE") + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + ablate_config = root / "ablate-config.json" + ablate_config.write_bytes(self.CONFIG) + with mock.patch.object( + ABLATE, "_ENGINE_TEXT_MAX_BYTES", len(self.CONFIG)): + identity = ABLATE._config_identity(ablate_config) + self.assertEqual(identity["vocab"], 4) + self.assertEqual( + identity["config_sha256"], + hashlib.sha256(self.CONFIG).hexdigest()) + ablate_config.write_bytes(self.CONFIG + b" ") + with self.assertRaisesRegex( + ABLATE.AblateEvidenceError, "256 MiB"): + ABLATE._config_identity(ablate_config) + + score_config = root / "config.json" + score_raw = b'{"vocab_size":4}\n' + score_config.write_bytes(score_raw) + with mock.patch.object( + EVAL, "_ENGINE_TEXT_MAX_BYTES", len(score_raw)): + self.assertEqual(EVAL.score_snapshot_vocab(root), 4) + score_config.write_bytes(score_raw + b" ") + with self.assertRaisesRegex(EVAL.EvidenceError, "256 MiB"): + EVAL.score_snapshot_vocab(root) + + request = "1 1 0 1" + request_bytes = len((request + "\n").encode("ascii")) + with mock.patch.object( + EVAL, "_ENGINE_TEXT_MAX_BYTES", request_bytes): + _, payload, _ = EVAL.score_request_wire((request,), 4) + self.assertEqual(len(payload), request_bytes) + with mock.patch.object( + EVAL, "_ENGINE_TEXT_MAX_BYTES", request_bytes - 1): + with self.assertRaisesRegex(EVAL.EvidenceError, "256 MiB"): + EVAL.score_request_wire((request,), 4) + + +class DifferentialBiteTests(unittest.TestCase): + """dev's classifier silently accepts a foreign stdout line the new + copy refuses. dev's side is not itself invoked here (no ported dev + module exists in this tree); it is asserted against the same fixture + line via dev's documented filter logic, ported verbatim inline. Only + the new copy's refusal is exercised by calling real code.""" + + FOREIGN_LINE = "1 1 1\n" # shaped like dev's own accepted grammar + + def test_dev_copy_silently_accepts_a_foreign_numeric_line(self): + # dev's inline stdout filter (ported verbatim as the oracle): any + # line starting with a digit or '-' is treated as a SCORE record, + # with no further validation at all. + line = self.FOREIGN_LINE.strip() + self.assertTrue(line and line[0] in "-0123456789") + parts = line.split() + logprob = float(parts[0]) # dev: "try: logprob = float(parts[0])" + # dev accepts this as a real SCORE result -- a false positive: a + # log-likelihood can never be positive, but dev's filter never + # checks the sign (or finiteness, or field count) at all. + self.assertEqual(logprob, 1.0) + + def test_new_copy_refuses_the_same_foreign_line_by_name(self): + with self.assertRaisesRegex( + EVAL.EvidenceError, "SCORE logprob is not finite/non-positive"): + EVAL.classify_score_stdout(self.FOREIGN_LINE) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_pack_python.py b/c/tests/test_pack_python.py index dc0aaaccd..fee5a4fda 100644 --- a/c/tests/test_pack_python.py +++ b/c/tests/test_pack_python.py @@ -166,6 +166,16 @@ def test_the_real_tree_carries_the_case_that_started_this(self): self.assertIn(TOOLS / "iq3xxs_grid.json", paths, "reached only as a data file opened next to iq3_pack.py") + def test_engine_evidence_is_needed_by_the_real_tree(self): + """This branch's own new tool, asserted the same way and for the same + reason: eval_glm.py is launched by coli as a subprocess and imports + engine_evidence, so the module is reachable only across the boundary + this suite exists to defend. It is a second real-tree case rather than + a replacement for the one above -- that one pins the historical bug, + this one pins the edge the branch adds.""" + paths = PACK.needed(HERE.parent) + self.assertIn(TOOLS / "engine_evidence.py", paths) + class PackPythonDataFileTests(unittest.TestCase): """#1359 left a second edge open after the import one was closed: packaging diff --git a/c/tools/check_ablate_evidence.py b/c/tools/check_ablate_evidence.py new file mode 100644 index 000000000..0f8486edf --- /dev/null +++ b/c/tools/check_ablate_evidence.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +"""Validate one complete ABLATE evidence artifact against a config.json.""" + +import argparse +import hashlib +import json +import math +import pathlib +import re +import sys + +# The canonical-manifest rule is shared with the engine and with the other +# evidence checkers, so it lives in one module. Imported both ways because this +# file is run as a script from the engine directory and imported as part of the +# tools package by the tests. +try: + from tools.engine_evidence import ( + ManifestFormError, canonical_manifest_bytes) +except ImportError: # run directly: tools/ is on the path + from engine_evidence import ManifestFormError, canonical_manifest_bytes + + +DOMAIN = b"coli-ablate-manifest/2\n" +_INT = re.compile(r"(?:0|[1-9][0-9]*|-[1-9][0-9]*)") +_SHA256 = re.compile(r"[0-9a-f]{64}") + +# The wire schema (``coli-ablate/2``) is a fixed-width LP64 domain, chosen so +# a producer and a checker on different host ABIs agree byte-for-byte: every +# manifest integer and every completion counter is signed 64-bit, while a +# value the engine narrows to C ``int`` (a layer, an expert, a token id) is +# signed 32-bit. +_INT32_MIN = -(1 << 31) +_INT32_MAX = (1 << 31) - 1 +_INT64_MIN = -(1 << 63) +_INT64_MAX = (1 << 63) - 1 +_ENGINE_VOCAB_MAX = 1 << 24 +_ENGINE_LAYERS_MAX = 128 +_ENGINE_EXPERTS_MAX = 4096 +_ENGINE_TEXT_MAX_BYTES = 256 << 20 + + +class AblateEvidenceError(ValueError): + """The artifact cannot prove a complete ABLATE denominator.""" + + +def _checked_engine_text_size(length, label): + if type(length) is not int or not 0 <= length <= _ENGINE_TEXT_MAX_BYTES: + raise AblateEvidenceError( + f"{label} exceeds the inclusive 256 MiB engine limit") + return length + + +def _reject_constant(value): + raise AblateEvidenceError(f"non-JSON constant: {value}") + + +def _reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise AblateEvidenceError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _json_record(raw, label): + try: + return json.loads( + raw.decode("ascii"), parse_constant=_reject_constant, + object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError, + AblateEvidenceError) as exc: + raise AblateEvidenceError(f"invalid {label} JSON: {exc}") from exc + + +def _bounded_config_bytes(config_path): + path = pathlib.Path(config_path) + with path.open("rb") as source: + source.seek(0, 2) + length = source.tell() + _checked_engine_text_size(length, "external config.json") + source.seek(0) + raw = source.read(_ENGINE_TEXT_MAX_BYTES + 1) + _checked_engine_text_size(len(raw), "external config.json") + return raw + + +def _config_identity(config_path): + raw = _bounded_config_bytes(config_path) + if not raw: + raise AblateEvidenceError("external config.json is empty") + try: + config = json.loads( + raw.decode("utf-8"), parse_constant=_reject_constant, + object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError, + AblateEvidenceError) as exc: + raise AblateEvidenceError(f"external config.json is invalid: {exc}") from exc + if not isinstance(config, dict): + raise AblateEvidenceError("external config.json root is not an object") + try: + identity = { + "vocab": _bounded_int( + config["vocab_size"], "config vocab_size", + 1, _ENGINE_VOCAB_MAX), + "n_layers": _bounded_int( + config["num_hidden_layers"], "config num_hidden_layers", + 1, _ENGINE_LAYERS_MAX), + "n_experts": _bounded_int( + config["n_routed_experts"], "config n_routed_experts", + 1, _ENGINE_EXPERTS_MAX), + } + identity["first_dense"] = _bounded_int( + config["first_k_dense_replace"], + "config first_k_dense_replace", 0, identity["n_layers"]) + except (KeyError, AblateEvidenceError) as exc: + raise AblateEvidenceError( + "external config.json topology is incomplete or invalid") from exc + identity["config_sha256"] = hashlib.sha256(raw).hexdigest() + return identity + + +def _bounded_int(value, label, minimum, maximum): + if type(value) is not int or not minimum <= value <= maximum: + raise AblateEvidenceError( + f"{label} is outside {minimum}..{maximum}") + return value + + +def _manifest_i64(text, line_number): + try: + value = int(text) + except ValueError as exc: + raise AblateEvidenceError( + f"manifest line {line_number} integer is too large") from exc + return _bounded_int( + value, f"manifest line {line_number} integer", + _INT64_MIN, _INT64_MAX) + + +def _count_add(current, increment, label): + _bounded_int(current, label, 0, _INT64_MAX) + _bounded_int(increment, f"{label} increment", 0, _INT64_MAX) + if increment > _INT64_MAX - current: + raise AblateEvidenceError(f"{label} exceeds signed 64-bit domain") + return current + increment + + +def _manifest_proof(raw, vocab, n_layers, first_dense, n_experts): + # The engine accepts a manifest saved with either line ending, and with or + # without a terminator on its last line, and digests the canonical form + # rather than the bytes on disk. Reproduce that here from the shared rule, + # or this checker would reject a file the producer ran and would compute a + # different digest for one it accepted. + try: + raw = canonical_manifest_bytes(raw) + except ManifestFormError as exc: + raise AblateEvidenceError(f"manifest is not canonical text: {exc}") from exc + _bounded_int(vocab, "external vocabulary", 1, _ENGINE_VOCAB_MAX) + _bounded_int(n_layers, "external n_layers", 1, _ENGINE_LAYERS_MAX) + _bounded_int(first_dense, "header first_dense", 0, n_layers) + _bounded_int(n_experts, "external n_experts", 1, _ENGINE_EXPERTS_MAX) + items = [] + seen = set() + item_count = targets = 0 + for line_number, raw_line in enumerate(raw[:-1].split(b"\n"), 1): + try: + text = raw_line.decode("ascii") + except UnicodeDecodeError as exc: + raise AblateEvidenceError( + f"manifest line {line_number} is not ASCII") from exc + parts = text.split(" ") + if (not parts or any(not _INT.fullmatch(part) for part in parts) or + " ".join(parts) != text): + raise AblateEvidenceError( + f"manifest line {line_number} is not canonical integer grammar") + values = [_manifest_i64(part, line_number) for part in parts] + if len(values) < 5: + raise AblateEvidenceError(f"manifest line {line_number} is truncated") + item, length, prompt, mode, cells = values[:5] + if (item < 0 or item in seen or + not 2 <= length <= _INT32_MAX or prompt < 1 or + prompt >= length or mode not in range(4) or + cells not in range(17) or + (mode == 0 and cells != 0) or + (mode != 0 and cells == 0)): + raise AblateEvidenceError( + f"manifest line {line_number} has invalid fields/denominator") + expected = 5 + 3 * cells + length + if len(values) != expected: + raise AblateEvidenceError( + f"manifest line {line_number} has invalid fields/denominator") + triples = [] + source_cells = set() + cursor = 5 + for _ in range(cells): + layer, expert, applied = values[cursor:cursor + 3] + cursor += 3 + if (not first_dense <= layer < n_layers or + not 0 <= expert < n_experts or + not _INT32_MIN <= applied <= _INT32_MAX or + (mode == 3 and + (not 0 <= applied < n_experts or applied == expert)) or + (mode != 3 and applied != -1) or + (layer, expert) in source_cells): + raise AblateEvidenceError( + f"manifest line {line_number} has invalid cell") + source_cells.add((layer, expert)) + triples.append([layer, expert, applied]) + tokens = values[cursor:] + if any(token < 0 or token >= vocab for token in tokens): + raise AblateEvidenceError( + f"manifest line {line_number} has out-of-vocabulary token") + seen.add(item) + positions = range(prompt - 1, length - 1) + row_targets = length - prompt + item_count = _count_add(item_count, 1, "manifest item count") + targets = _count_add(targets, row_targets, "manifest target count") + items.append({ + "item": item, "T": length, "n_prompt": prompt, + "mode": mode, "ncells": cells, "cells": triples, + "positions": positions, "tokens": tuple(tokens), + }) + if item_count <= 0 or targets <= 0: + raise AblateEvidenceError("manifest denominator is not positive") + return { + "sha256": hashlib.sha256(DOMAIN + raw).hexdigest(), + "items": tuple(items), "item_count": item_count, "targets": targets, + } + + +def validate_ablate_evidence(manifest_path, evidence_path, config_path): + identity = _config_identity(config_path) + manifest_raw = pathlib.Path(manifest_path).read_bytes() + evidence_raw = pathlib.Path(evidence_path).read_bytes() + if (not evidence_raw or not evidence_raw.endswith(b"\n") or + b"\r" in evidence_raw or b"\0" in evidence_raw): + raise AblateEvidenceError("evidence is not nonempty canonical LF JSONL") + records = [_json_record(line, f"record {index}") + for index, line in enumerate(evidence_raw.splitlines(), 1)] + if not records: + raise AblateEvidenceError("evidence has no header") + header = records[0] + if (not isinstance(header, dict) or set(header) != { + "t", "schema", "vocab", "topk", "manifest_sha256", + "n_layers", "first_dense", "n_experts", + "config_sha256", + "expected_items", "expected_targets"} or + header.get("t") != "hdr" or header.get("schema") != "coli-ablate/2" or + type(header.get("topk")) is not int or + not isinstance(header.get("config_sha256"), str) or + not _SHA256.fullmatch(header["config_sha256"]) or + not isinstance(header.get("manifest_sha256"), str) or + not _SHA256.fullmatch(header["manifest_sha256"])): + raise AblateEvidenceError("header keys or values are not exact") + try: + _bounded_int(header["vocab"], "header vocabulary", + 1, _ENGINE_VOCAB_MAX) + if header["topk"] != min(32, header["vocab"]): + raise AblateEvidenceError("header topk is not producer-exact") + _bounded_int(header["n_layers"], "header n_layers", + 1, _ENGINE_LAYERS_MAX) + _bounded_int(header["first_dense"], "header first_dense", + 0, header["n_layers"]) + _bounded_int(header["n_experts"], "header n_experts", + 1, _ENGINE_EXPERTS_MAX) + _bounded_int(header["expected_items"], "header expected_items", + 1, _INT64_MAX) + _bounded_int(header["expected_targets"], "header expected_targets", + 1, _INT64_MAX) + except (KeyError, AblateEvidenceError) as exc: + raise AblateEvidenceError( + f"header keys or values are not exact: {exc}") from exc + if any(header[key] != identity[key] for key in ( + "vocab", "n_layers", "first_dense", "n_experts", + "config_sha256")): + raise AblateEvidenceError( + "header does not match the external config identity") + proof = _manifest_proof( + manifest_raw, identity["vocab"], identity["n_layers"], + identity["first_dense"], identity["n_experts"]) + if (header["manifest_sha256"] != proof["sha256"] or + header["expected_items"] != proof["item_count"] or + header["expected_targets"] != proof["targets"]): + raise AblateEvidenceError("header does not bind the source manifest proof") + + cursor = 1 + completed_items = completed_targets = 0 + logit_keys = { + "t", "item", "pos", "gold", "nll", "glogit", "molo", "mgn", + "am", "amlogit", "logZ", "corr", "tk", + } + for expected in proof["items"]: + if cursor >= len(records): + raise AblateEvidenceError("missing item header") + item_header = records[cursor] + cursor += 1 + if (not isinstance(item_header, dict) or set(item_header) != { + "t", "item", "mode", "ncells", "T", "n_prompt", "cells"} or + any(type(item_header.get(key)) is not int for key in ( + "item", "mode", "ncells", "T", "n_prompt")) or + not isinstance(item_header.get("cells"), list) or + any(not isinstance(cell, list) or len(cell) != 3 or + any(type(value) is not int for value in cell) + for cell in item_header["cells"]) or + item_header != {key: expected[key] for key in ( + "item", "mode", "ncells", "T", "n_prompt", "cells")} | + {"t": "ah"}): + raise AblateEvidenceError("item header does not match manifest order") + for position in expected["positions"]: + if cursor >= len(records): + raise AblateEvidenceError("missing target row") + row = records[cursor] + cursor += 1 + if (not isinstance(row, dict) or set(row) != logit_keys or + row.get("t") != "lg" or type(row.get("item")) is not int or + row["item"] != expected["item"] or + type(row.get("pos")) is not int or row["pos"] != position or + type(row.get("gold")) is not int or + row["gold"] != expected["tokens"][position + 1] or + type(row.get("am")) is not int or + row["am"] not in range(header["vocab"]) or + type(row.get("corr")) is not int or row["corr"] not in (0, 1)): + raise AblateEvidenceError("target row identity/fields are invalid") + for field in ("nll", "glogit", "molo", "mgn", "amlogit", "logZ"): + if (type(row.get(field)) not in (int, float) or + not math.isfinite(row[field])): + raise AblateEvidenceError(f"target row {field} is nonfinite") + # These three hold for every row the producer can emit: nll is a + # negated log-probability (always <= 0 before negation); corr is + # defined as the argmax/gold agreement, not sampled separately; + # and amlogit is the row's own max logit, so no field can exceed + # it -- least of all the gold token's own logit. + if row["nll"] < 0: + raise AblateEvidenceError("target row nll is negative") + if row["corr"] != int(row["am"] == row["gold"]): + raise AblateEvidenceError( + "target row corr does not match its am/gold agreement") + if row["amlogit"] < row["glogit"]: + raise AblateEvidenceError( + "target row amlogit is below glogit") + topk = row.get("tk") + if (not isinstance(topk, list) or len(topk) != header["topk"] or + any(not isinstance(pair, list) or len(pair) != 2 or + type(pair[0]) is not int or not 0 <= pair[0] < header["vocab"] or + type(pair[1]) not in (int, float) or not math.isfinite(pair[1]) + for pair in topk) or + len({pair[0] for pair in topk}) != len(topk)): + raise AblateEvidenceError("target row top-k is invalid") + completed_targets = _count_add( + completed_targets, 1, "completed target count") + completed_items = _count_add( + completed_items, 1, "completed item count") + + if cursor >= len(records): + raise AblateEvidenceError("missing terminal completion record") + done = records[cursor] + cursor += 1 + if (not isinstance(done, dict) or set(done) != { + "t", "manifest_sha256", "completed_items", "completed_targets"}): + raise AblateEvidenceError("terminal completion proof is invalid") + try: + _bounded_int(done["completed_items"], "done completed_items", + 1, _INT64_MAX) + _bounded_int(done["completed_targets"], "done completed_targets", + 1, _INT64_MAX) + except (KeyError, AblateEvidenceError) as exc: + raise AblateEvidenceError("terminal completion proof is invalid") from exc + if done != {"t": "done", "manifest_sha256": proof["sha256"], + "completed_items": completed_items, + "completed_targets": completed_targets}: + raise AblateEvidenceError("terminal completion proof is invalid") + if cursor != len(records): + raise AblateEvidenceError("records follow terminal completion proof") + if (completed_items != proof["item_count"] or + completed_targets != proof["targets"]): + raise AblateEvidenceError("completed denominator does not match manifest") + return { + "manifest_sha256": proof["sha256"], "items": completed_items, + "targets": completed_targets, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest") + parser.add_argument("evidence") + parser.add_argument("--config", required=True, + help="independently supplied loaded-model config.json") + args = parser.parse_args(argv) + try: + result = validate_ablate_evidence( + args.manifest, args.evidence, args.config) + except (OSError, AblateEvidenceError) as exc: + print(f"[ablate-evidence] INCOMPLETE: {exc}", file=sys.stderr) + return 1 + print(f"[ablate-evidence] PASS manifest={result['manifest_sha256']} " + f"items={result['items']} targets={result['targets']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/c/tools/engine_evidence.py b/c/tools/engine_evidence.py new file mode 100644 index 000000000..cdafd0ff9 --- /dev/null +++ b/c/tools/engine_evidence.py @@ -0,0 +1,158 @@ +"""Helpers for reading what the engine wrote, and for reproducing what it read. + +Two unrelated jobs live here because both are shared by more than one +checker: parsing the engine's startup preamble lines, and reproducing the +canonical form of a manifest that the engine binds by digest. + +Recognizes the two typed lines the engine prints at startup -- the +"== GLM C engine ..." banner and the following "loaded in ..." record -- +and returns their fields as typed values, used by the evidence checkers +that read raw engine stdout. A line that merely looks like one of these +preambles but fails a field check is a bug worth surfacing loudly, so +parsing raises rather than silently skipping. + +canonical_manifest_bytes() is the other half: the engine accepts a manifest +saved with either line ending and with or without a final newline, and +digests the normalised form rather than the file as it sits on disk. A +checker that hashed the raw file would disagree with the engine about a +manifest both of them accept, so the rule lives here once and both sides +use it. +""" + +import math +import re + + +class PreambleError(ValueError): + """A line resembles an owned engine preamble but is not source-valid.""" + + +_INT32_MAX = 2**31 - 1 +_UINT_TEXT = r"(?:0|[1-9][0-9]*)" +_FIXED2_TEXT = r"(?:0|[1-9][0-9]*)\.[0-9]{2}" +IDOT_KERNELS = ( + "avx512-vnni", "avx-vnni", "avx2", "neon-i8mm", "neon", "vsx", + "scalar", +) + +_BANNER_RE = re.compile( + r"^== GLM C engine \(glm_moe_dsa\), cache=(?P" + _UINT_TEXT + + r") experts/layer \| compute experts@(?P" + _UINT_TEXT + + r")-bit dense@(?P" + _UINT_TEXT + + r")-bit \| idot: (?P" + "|".join(IDOT_KERNELS) + r") ==$") +_LOADED_RE = re.compile( + r"^loaded in (?P" + _FIXED2_TEXT + + r")s \| resident dense: (?P" + _FIXED2_TEXT + + r") MB \| layers=(?P" + _UINT_TEXT + + r") experts=(?P" + _UINT_TEXT + + r") \| MTP (?PACTIVE|absent|DISABLED \(multiplexed serve\)) " + r"\(draft=(?P" + _UINT_TEXT + r")\)$") + + +def parse_engine_banner(line): + """Return typed fields for the exact production "== GLM C engine" banner.""" + if not isinstance(line, str): + raise PreambleError(f"engine banner is not text: {line!r}") + match = _BANNER_RE.fullmatch(line) + if not match: + raise PreambleError(f"not an exact engine banner: {line!r}") + cap, expert_bits, dense_bits = map( + int, match.group("cap", "expert_bits", "dense_bits")) + if not 1 <= cap <= _INT32_MAX: + raise PreambleError(f"engine cache outside [1,{_INT32_MAX}]: {cap}") + if not 1 <= expert_bits <= 16 or not 1 <= dense_bits <= 16: + raise PreambleError( + f"engine compute bits outside [1,16]: {expert_bits}/{dense_bits}") + return { + "kind": "BANNER", "cap": cap, "expert_bits": expert_bits, + "dense_bits": dense_bits, "kernel": match.group("kernel"), + } + + +def parse_engine_loaded(line): + """Return typed fields for the exact "loaded in ..." record that follows the banner.""" + if not isinstance(line, str): + raise PreambleError(f"engine load record is not text: {line!r}") + match = _LOADED_RE.fullmatch(line) + if not match: + raise PreambleError(f"not an exact engine load record: {line!r}") + load_s = float(match.group("load_s")) + resident_mb = float(match.group("resident_mb")) + layers, experts, draft = map( + int, match.group("layers", "experts", "draft")) + mtp = match.group("mtp") + if not math.isfinite(load_s) or not math.isfinite(resident_mb): + raise PreambleError("engine load metrics must be finite") + if load_s < 0 or resident_mb < 0: + raise PreambleError("engine load metrics must be nonnegative") + if not 1 <= layers <= 128: + raise PreambleError(f"engine layers outside [1,128]: {layers}") + if not 1 <= experts <= 4096: + raise PreambleError(f"engine experts outside [1,4096]: {experts}") + if not 0 <= draft <= 63: + raise PreambleError(f"engine draft outside [0,63]: {draft}") + if mtp == "DISABLED (multiplexed serve)" and draft != 0: + raise PreambleError("disabled multiplexed MTP requires draft=0") + return { + "kind": "LOADED", "load_s": load_s, "resident_mb": resident_mb, + "layers": layers, "experts": experts, "mtp": mtp, "draft": draft, + } + + +def parse_engine_preamble(line): + """Dispatch to the banner/loaded parser by prefix, or return None. + + None means the line is not one of the two owned preambles at all (an + ordinary log line); a line that starts like one of them but fails to + parse still raises PreambleError rather than being treated as unowned. + """ + if not isinstance(line, str): + raise PreambleError(f"engine preamble is not text: {line!r}") + if line.startswith("== GLM C engine"): + return parse_engine_banner(line) + if line.startswith("loaded in"): + return parse_engine_loaded(line) + return None + + +class ManifestFormError(ValueError): + """A manifest cannot be reduced to the canonical form the engine binds.""" + + +def canonical_manifest_bytes(raw): + """Return the exact byte stream the engine digests for this manifest. + + The engine reads the file a line at a time, drops the line terminator, + drops one carriage return in front of it if there is one, and digests the + remaining record followed by a single newline. A file saved with CRLF + endings, or without a terminator on its last line, therefore produces the + same digest as the same content saved as plain newline-terminated text -- + which is what a host editor makes it easy to get wrong. + + Everything else is still refused, and refused here rather than later: + an empty file, an empty record, a carriage return inside a record, and an + embedded NUL. Those are not framings of valid content, they are corruption. + """ + if not isinstance(raw, (bytes, bytearray)): + raise ManifestFormError(f"manifest is not bytes: {type(raw).__name__}") + raw = bytes(raw) + if not raw: + raise ManifestFormError("manifest is empty") + if b"\0" in raw: + raise ManifestFormError("manifest contains a NUL byte") + records = raw.split(b"\n") + if records and records[-1] == b"": + records.pop() # the file ended with its terminator + if not records: + raise ManifestFormError("manifest holds no records") + canonical = [] + for number, record in enumerate(records, 1): + if record.endswith(b"\r"): + record = record[:-1] + if not record: + raise ManifestFormError(f"manifest line {number} is empty") + if b"\r" in record: + raise ManifestFormError( + f"manifest line {number} has a carriage return inside it") + canonical.append(record) + return b"\n".join(canonical) + b"\n" diff --git a/c/tools/eval_glm.py b/c/tools/eval_glm.py index 76097cd6e..4e3931085 100644 --- a/c/tools/eval_glm.py +++ b/c/tools/eval_glm.py @@ -21,8 +21,40 @@ --tasks hellaswag,arc_challenge,mmlu --limit 40 --ram 15 # leve di ricerca: passate al motore via env TOPP=0.9 python3 tools/eval_glm.py --snap /path/to/glm52_i4 --data ./bench --tasks mmlu --ram 15 + +Evidence binding (current limitation): this harness always sets +SCORE_EVIDENCE=1 in the child environment and always tries to bind each +SCORE result to the exact request bytes that produced it by digest. Only +an engine build that prints the identity-bound wire form -- "SCORE + " -- can +satisfy that binding; SCORE_EVIDENCE is a plain, unread environment +variable to every other engine build, harmless to set. When the engine +instead prints only the byte-compatible legacy three-field form (" + ", no identity prefix), the run still completes +normally and every row is still written, but the results are UNBOUND -- +marked as such in the output file's summary line and announced once on +stderr -- rather than silently treated as bound. A stream that mixes both +forms in one run is refused with a named error instead of guessed at. """ -import os, sys, subprocess, argparse, random, json, tempfile, time, threading +import argparse +import hashlib +import json +import math +import os +import random +import re +import signal +import subprocess +import sys +import tempfile +import threading +import time + +_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +if _TOOLS_DIR not in sys.path: + sys.path.insert(0, _TOOLS_DIR) +from engine_evidence import (PreambleError, parse_engine_banner, + parse_engine_loaded, parse_engine_preamble) # mini-set OFFLINE per testare la meccanica (NON misura qualita': domande banali) SMOKE = [ @@ -38,6 +70,277 @@ "arc_challenge": {"GLM-5.2 (pubbl.)": None}, } + +class EvidenceError(ValueError): + """A SCORE stream cannot support a complete, finite evidence result.""" + + +class ChildTerminateRequested(BaseException): + """A termination signal (SIGTERM) arrived while the engine child was + running. Raised from a signal handler installed only for the + duration of that child's run, so it unwinds through the same + ``finally`` cleanup as any other mid-run exception (including + Python's own SIGINT-to-KeyboardInterrupt) and the child is never + left running as a zombie/orphan. + """ + + +_INT32_MAX = 2**31 - 1 +_ENGINE_TEXT_MAX_BYTES = 256 << 20 +_UINT_TEXT = r"(?:0|[1-9][0-9]*)" +_C17G_TEXT = (r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?" + r"(?:e[+-](?:0[0-9]|[1-9][0-9]{1,2}))?") +_SCORE_RE = re.compile( + rf"^({_C17G_TEXT}) ({_UINT_TEXT}) ([01])$") +_SCORE_EVIDENCE_RE = re.compile( + rf"^SCORE ({_UINT_TEXT}) ([0-9a-f]{{64}}) " + rf"({_C17G_TEXT}) ({_UINT_TEXT}) ([01])$") + + +def _checked_engine_text_size(length, label): + if type(length) is not int or not 0 <= length <= _ENGINE_TEXT_MAX_BYTES: + raise EvidenceError( + f"{label} exceeds the inclusive 256 MiB engine limit") + return length + + +def parse_c17g(text): + """Parse the canonical C-locale numeric token the engine actually emits. + + The engine has shipped two mutually exclusive SCORE spellings across + its history -- the byte-compatible ``printf("%.6f")`` form dev still + emits today, and the newer opt-in ``printf("%.17g")`` evidence form. + Both are exact, finite, round-trippable spellings of the same + C-locale numeric domain, so both are accepted here (the function name + is kept for the newer form this module's identity checks depend on); + a text that is neither exact spelling -- including any non-canonical + variant such as a non-canonical ``%.17g`` corpus (nan/inf/-inf never + survive: neither spelling is finite-preserving for them) -- is + refused. + """ + if not isinstance(text, str) or not re.fullmatch(_C17G_TEXT, text): + raise EvidenceError(f"not a canonical %.6f/%.17g token: {text!r}") + try: + value = float(text) + except ValueError as exc: + raise EvidenceError(f"malformed %.6f/%.17g token: {text!r}") from exc + if (not math.isfinite(value) or + (format(value, ".17g") != text and format(value, ".6f") != text)): + raise EvidenceError( + f"not an exact finite %.6f/%.17g spelling: {text!r}") + return value + + +def is_score_preamble(line): + """Validate one of the two exact stdout records emitted before SCORE.""" + try: + return parse_engine_preamble(line) is not None + except PreambleError: + return False + + +def parse_score_result(line): + """Return (exact_text, value, contlen, greedy) for one complete SCORE line.""" + match = _SCORE_RE.fullmatch(line) + if not match: + raise EvidenceError(f"not an exact SCORE record: {line!r}") + exact, contlen_text, greedy_text = match.groups() + try: + value = parse_c17g(exact) + contlen = int(contlen_text) + greedy = int(greedy_text) + except ValueError as exc: + raise EvidenceError(f"malformed SCORE fields: {line!r}") from exc + if not math.isfinite(value) or value > 0.0: + raise EvidenceError(f"SCORE logprob is not finite/non-positive: {exact}") + if not 1 <= contlen <= _INT32_MAX or greedy not in (0, 1): + raise EvidenceError(f"invalid SCORE metadata: {line!r}") + return exact, value, contlen, greedy + + +def parse_score_evidence_result(line): + """Return the strict ordinal/digest identity plus the SCORE payload.""" + match = _SCORE_EVIDENCE_RE.fullmatch(line) + if not match: + raise EvidenceError(f"not an exact evidence SCORE record: {line!r}") + ordinal_text, digest, exact, contlen_text, greedy_text = match.groups() + ordinal = int(ordinal_text) + if ordinal > _INT32_MAX: + raise EvidenceError(f"SCORE request ordinal is outside int32: {line!r}") + parsed = parse_score_result(f"{exact} {contlen_text} {greedy_text}") + return ordinal, digest, *parsed + + +def classify_score_stdout(raw_line): + """Accept one exact newline-terminated production stdout record.""" + if (not raw_line.endswith("\n") or raw_line.count("\n") != 1 or + "\r" in raw_line): + raise EvidenceError(f"unterminated/non-canonical stdout record: {raw_line!r}") + line = raw_line[:-1] + if not line: + raise EvidenceError("blank SCORE stdout record") + try: + preamble = parse_engine_preamble(line) + except PreambleError as exc: + raise EvidenceError(str(exc)) from exc + if preamble is not None: + return None + return parse_score_result(line) + + +class ScoreStdoutClassifier: + """Own exactly one banner then one load record before SCORE results. + + Only an engine build that prints the identity-bound wire form + (``SCORE + ``) lets results be BOUND to their originating request by + digest. An engine that prints only the byte-compatible legacy form + (`` ``, no identity prefix) is still a + complete, honest run -- this is not a failure -- but nothing ties any + individual result back to the request that produced it, so the run's + results are UNBOUND. The two forms are never silently conflated: a + stream that starts in one form and switches to the other mid-run + (the classic case would be corrupted/interleaved output) is refused + with a named error rather than guessed at. + + Beyond the banner/load preamble, no other multiplexed-serve global + record (``PROF``, ``HITS``, ``EMAP``, ...) or stderr-only banner + (``[prefill]``, ``[PIN]``, ``[USAGE]``, ...) can ever reach this + classifier: ``run_score`` never prints them to stdout (verified + against the engine source), so any such line arriving here is + refused by name like any other unrecognized record, not specially + recognized or passed through. + """ + + def __init__(self, request_digests=None): + self._state = 0 + self._request_digests = (None if request_digests is None + else tuple(request_digests)) + self._result_index = 0 + self._mode = None # None (no SCORE record yet) | "bound" | "unbound" + + @property + def binding_mode(self): + """"bound", "unbound", or None if no SCORE record was classified.""" + return self._mode + + def classify(self, raw_line): + if (not raw_line.endswith("\n") or raw_line.count("\n") != 1 or + "\r" in raw_line): + raise EvidenceError( + f"unterminated/non-canonical stdout record: {raw_line!r}") + line = raw_line[:-1] + if not line: + raise EvidenceError("blank SCORE stdout record") + try: + if self._state == 0: + parse_engine_banner(line) + self._state = 1 + return None + if self._state == 1: + parse_engine_loaded(line) + self._state = 2 + return None + preamble = parse_engine_preamble(line) + except PreambleError as exc: + raise EvidenceError(str(exc)) from exc + if preamble is not None: + raise EvidenceError(f"duplicate/out-of-order SCORE preamble: {line!r}") + if self._request_digests is None: + return parse_score_result(line) + is_bound_shape = _SCORE_EVIDENCE_RE.fullmatch(line) is not None + is_unbound_shape = not is_bound_shape and _SCORE_RE.fullmatch(line) is not None + if not is_bound_shape and not is_unbound_shape: + raise EvidenceError(f"not an exact SCORE record: {line!r}") + line_mode = "bound" if is_bound_shape else "unbound" + if self._mode is None: + self._mode = line_mode + elif self._mode != line_mode: + raise EvidenceError( + "SCORE stream mixes identity-bound and legacy records: " + f"{line!r}") + if line_mode == "unbound": + self._result_index += 1 + return parse_score_result(line) + ordinal, digest, exact, value, contlen, greedy = \ + parse_score_evidence_result(line) + if self._result_index >= len(self._request_digests): + raise EvidenceError("engine emitted extra evidence SCORE result lines") + if ordinal != self._result_index: + raise EvidenceError( + f"SCORE request ordinal {ordinal} != expected {self._result_index}") + expected_digest = self._request_digests[self._result_index] + if digest != expected_digest: + raise EvidenceError( + f"SCORE request {ordinal} digest does not match exact request bytes") + self._result_index += 1 + return exact, value, contlen, greedy + + def finish(self): + if self._state != 2: + missing = "engine banner" if self._state == 0 else "engine load record" + raise EvidenceError(f"missing {missing} before SCORE EOF") + if (self._mode == "bound" and self._request_digests is not None and + self._result_index != len(self._request_digests)): + raise EvidenceError( + f"only {self._result_index}/{len(self._request_digests)} " + "identity-bound SCORE records") + + +def completion_error(returncode, completed, expected, continuation_tokens, + stream_error=None): + """Return the reason this run has NOTHING trustworthy to report, or + None if it can report a table (even a partial one). + + Matches dev's own exit-code contract exactly, since callers such as + ``coli bench`` (``sys.exit(subprocess.call(cmd, ...))``) and + ``diag_harness.py`` (which parses this tool's own accuracy table from + a subprocess call) depend on it: dev exits nonzero ONLY when the + engine itself produced nothing at all (nonzero exit AND zero + requests scored); a partial run -- some but not all requests scored, + or the engine exiting nonzero after scoring at least one request -- + still prints the accuracy table over whatever landed and exits 0. + Nothing here checks ``expected``/``continuation_tokens`` against a + denominator: by the time this runs, ``expected`` (the request count) + is always positive (an empty selection is refused before the engine + ever launches), and a positive ``completed`` count always carries a + positive token count by construction. + + ``stream_error`` is the one condition dev's own contract has no + analog for: a genuinely corrupted or self-inconsistent SCORE stream + (mixed identity-bound/legacy records, a replayed digest, an + out-of-vocabulary token, ...), which this module's evidence layer can + detect and dev's plain per-line filter cannot. That failure stays + fatal regardless of how many requests completed, because the parsed + numbers themselves are not trustworthy. + """ + if stream_error: + return str(stream_error) + if returncode != 0 and completed == 0: + return f"engine exited {returncode} with zero requests scored" + return None + + +def write_result_row(out_f, req_idx, meta_row, exact_logprob, greedy): + """Write the exact engine token, never a rounded float reconstruction.""" + task, qi, oi, clen, cchars, gold = meta_row + out_f.write(f"{req_idx},{task},{qi},{oi},{clen},{cchars},{gold}," + f"{exact_logprob},{greedy}\n") + + +def prelaunch_incomplete(out_path, reason): + """Refuse vacuous evidence before Popen and durably mark writable output.""" + message = f"EVIDENCE INCOMPLETE before engine launch: {reason}" + print(message, file=sys.stderr) + if out_path: + try: + with open(out_path, "a") as out_f: + out_f.write(f"# INCOMPLETE: 0/0; error={reason}\n") + except OSError as exc: + print(f"cannot mark output {out_path!r} INCOMPLETE: {exc}", + file=sys.stderr) + return 1 + def load_docs(task, data_dir, limit, seed): if task == "smoke": return SMOKE[:limit] if limit else SMOKE @@ -72,14 +375,88 @@ def build_requests(tk, docs_by_task, prefix=""): cl = len(ctx_ids) while cl > 0 and (cl > len(full) or full[:cl] != ctx_ids[:cl]): cl -= 1 cont_ids = full[cl:] - if not cont_ids: # boundary degenere: forza split esplicito - full = ctx_ids + tk.encode(cont).ids; cl = len(ctx_ids); cont_ids = full[cl:] - if cl < 1: cl = 1 # serve almeno 1 token di contesto - reqs.append(f"{cl} {len(full)-cl} " + " ".join(map(str, full))) - meta.append((t, qi, oi, len(full) - cl, max(1, len(cont)), gold)) + if cl < 1 or not cont_ids: # boundary degenere: forza split esplicito + choice_ids = tk.encode(cont).ids + full = ctx_ids + choice_ids + cl = len(ctx_ids) + cont_ids = choice_ids + if cl < 1 or not cont_ids: + raise EvidenceError( + f"{t} question {qi} choice {oi} has no positive " + "context/continuation token denominator") + reqs.append(f"{cl} {len(cont_ids)} " + " ".join(map(str, full))) + meta.append((t, qi, oi, len(cont_ids), max(1, len(cont)), gold)) perq.setdefault((t, qi), []).append(len(meta) - 1) return reqs, meta, perq + +def score_snapshot_vocab(snap): + """Read the engine's independently loaded vocabulary bound from config.""" + path = os.path.join(snap, "config.json") + try: + with open(path, "rb") as source: + source.seek(0, os.SEEK_END) + length = source.tell() + _checked_engine_text_size(length, "SCORE config.json") + source.seek(0) + raw = source.read(_ENGINE_TEXT_MAX_BYTES + 1) + _checked_engine_text_size(len(raw), "SCORE config.json") + config = json.loads(raw.decode("utf-8")) + vocab = config["vocab_size"] + except (OSError, UnicodeDecodeError, json.JSONDecodeError, + KeyError, TypeError) as exc: + raise EvidenceError(f"cannot derive SCORE vocabulary from {path}: {exc}") from exc + if type(vocab) is not int or not 1 <= vocab <= 1 << 24: + raise EvidenceError(f"invalid SCORE vocabulary in {path}: {vocab!r}") + return vocab + + +def score_request_wire(requests, vocab): + """Return strict ASCII/LF records, joined bytes, and per-record SHA-256. + + The C SCORE evidence mode hashes the ``getline`` byte span, including LF; + this helper owns the identical byte domain before the temporary file exists. + """ + if type(vocab) is not int or not 1 <= vocab <= 1 << 24: + raise EvidenceError(f"invalid SCORE vocabulary: {vocab!r}") + lines = [] + continuation_tokens = 0 + image_bytes = 0 + for request in requests: + if not isinstance(request, str) or not request or "\n" in request or "\r" in request: + raise EvidenceError(f"request is not one canonical line: {request!r}") + fields = request.split(" ") + if (" ".join(fields) != request or len(fields) < 4 or + any(not re.fullmatch(_UINT_TEXT, field) for field in fields)): + raise EvidenceError(f"request is not canonical integer grammar: {request!r}") + values = [int(field) for field in fields] + ctxlen, contlen = values[:2] + if (not 1 <= ctxlen <= _INT32_MAX or + not 1 <= contlen <= _INT32_MAX - ctxlen): + raise EvidenceError(f"request lengths are invalid: {request!r}") + total = ctxlen + contlen + tokens = values[2:] + if len(tokens) != total: + raise EvidenceError(f"request token count is invalid: {request!r}") + if any(token >= vocab for token in tokens): + raise EvidenceError(f"request token is outside vocabulary: {request!r}") + if continuation_tokens > (1 << 63) - 1 - contlen: + raise EvidenceError("SCORE continuation denominator exceeds int64") + continuation_tokens += contlen + try: + line = (request + "\n").encode("ascii") + except UnicodeEncodeError as exc: + raise EvidenceError( + f"request is not canonical ASCII: {request!r}") from exc + image_bytes = _checked_engine_text_size( + image_bytes + len(line), "SCORE request image") + lines.append(line) + if not lines or continuation_tokens <= 0 or len(lines) > _INT32_MAX: + raise EvidenceError("SCORE request image has no positive denominator") + frozen = tuple(lines) + return (frozen, b"".join(frozen), + tuple(hashlib.sha256(line).hexdigest() for line in frozen)) + def score_accuracy(tasks, meta, perq, lp): print(f"\n{'task':<18} {'n':>4} {'acc':>7} {'acc_norm':>9}") overall = [] @@ -123,24 +500,45 @@ def main(): score_accuracy(["t"], meta, perq, lp) print("selftest OK" if True else ""); return + tasks = [t.strip() for t in a.tasks.split(",") if t.strip()] + if not tasks: + return prelaunch_incomplete(a.out,"no benchmark tasks selected") + from tokenizers import Tokenizer tk = Tokenizer.from_file(os.path.join(a.snap, "tokenizer.json")) - tasks = [t.strip() for t in a.tasks.split(",") if t.strip()] docs_by_task = {t: load_docs(t, a.data, a.limit, a.seed) for t in tasks} for t, d in docs_by_task.items(): print(f"[{t}] {len(d)} questions", file=sys.stderr) - reqs, meta, perq = build_requests(tk, docs_by_task, detect_prefix(a.snap)) + try: + reqs, meta, perq = build_requests( + tk, docs_by_task, detect_prefix(a.snap)) + except EvidenceError as exc: + return prelaunch_incomplete(a.out, str(exc)) print(f"total requests: {len(reqs)} (answer options)", file=sys.stderr) + if not reqs: + return prelaunch_incomplete(a.out,"selected tasks produced zero SCORE requests") if a.dry: + # Matches dev exactly: --dry stops right after request + # construction, before the vocabulary lookup below -- it never + # needed config.json's vocab_size (a plumbing check has no + # engine, and therefore no vocabulary, to bind requests against). for r in reqs[:3]: print(" example request:", r[:80], "...", file=sys.stderr) print("DRY: request construction and tokenization passed. Engine was not run.", file=sys.stderr); return + try: + score_vocab = score_snapshot_vocab(a.snap) + _, request_payload, request_digests = score_request_wire( + reqs, score_vocab) + except EvidenceError as exc: + return prelaunch_incomplete(a.out, str(exc)) # mkstemp (non mktemp): crea il file atomicamente con permessi 0600, niente # race TOCTOU/symlink su una tmp dir condivisa (CWE-377). fd, req_path = tempfile.mkstemp(suffix=".txt") - with os.fdopen(fd, "w") as f: - f.write("\n".join(reqs) + "\n") - env = dict(os.environ, SNAP=a.snap, SCORE=req_path) + with os.fdopen(fd, "wb") as f: + written = f.write(request_payload) + if written != len(request_payload): + raise EvidenceError("short write while freezing SCORE requests") + env = dict(os.environ, SNAP=a.snap, SCORE=req_path, SCORE_EVIDENCE="1") if a.ram: env["RAM_GB"] = str(a.ram) cmd = [a.glm, str(a.cap)] + a.bits.split() print("running:", " ".join(cmd), file=sys.stderr) @@ -155,55 +553,137 @@ def main(): out_f.write("req_idx,task,qi,oi,contlen,contchars,gold,logprob,greedy\n") out_f.flush() t0 = time.time() - proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, bufsize=1) # line-buffered - lp = [None] * len(reqs) - n_done = 0 - # Drain stderr (engine progress lines) to console live on a background thread - # so the [score N req] heartbeat is visible while stdout is consumed below. - def _drain_stderr(): - for line in proc.stderr: - print(f" [engine] {line.rstrip()}", file=sys.stderr) - threading.Thread(target=_drain_stderr, daemon=True).start() - for line in proc.stdout: - line = line.strip() - if not line or line[0] not in "-0123456789": continue - parts = line.split() - if n_done >= len(reqs): break - try: logprob = float(parts[0]) - except (ValueError, IndexError): continue - lp[n_done] = logprob - greedy = parts[2] if len(parts) > 2 else "?" - t, qi, oi, clen, cchars, gold = meta[n_done] + proc = None + previous_sigterm = None + + def _on_sigterm(signum, frame): + raise ChildTerminateRequested(f"received signal {signum}") + + try: + # A SIGTERM (or Ctrl+C's SIGINT, which Python already converts to + # KeyboardInterrupt on its own) must not leave the engine child + # running as an orphan/zombie -- the handler below converts + # SIGTERM into the same exception path, so both unwind through + # the identical `finally` cleanup that terminates the child. + previous_sigterm = signal.signal(signal.SIGTERM, _on_sigterm) + proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, bufsize=1) # line-buffered + lp = [None] * len(reqs) + n_done = 0 + continuation_tokens = 0 + stream_error = None + stdout_classifier = ScoreStdoutClassifier(request_digests) + # Drain stderr (engine progress lines) to console live on a background thread + # so the [score N req] heartbeat is visible while stdout is consumed below. + def _drain_stderr(): + for line in proc.stderr: + print(f" [engine] {line.rstrip()}", file=sys.stderr) + threading.Thread(target=_drain_stderr, daemon=True).start() + for raw_line in proc.stdout: + if stream_error: + continue # drain fully so the child cannot block + try: + result = stdout_classifier.classify(raw_line) + except EvidenceError as exc: + stream_error = exc + continue + if result is None: + continue + if n_done >= len(reqs): + stream_error = EvidenceError("engine emitted extra SCORE result lines") + continue + try: + exact, logprob, contlen, greedy = result + if contlen != meta[n_done][3]: + raise EvidenceError( + f"request {n_done} contlen {contlen} != expected {meta[n_done][3]}") + except EvidenceError as exc: + stream_error = exc + continue + lp[n_done] = logprob + continuation_tokens += contlen + t, qi, oi, clen, cchars, gold = meta[n_done] + if out_f: + write_result_row(out_f,n_done,meta[n_done],exact,greedy) + out_f.flush() + n_done += 1 + if n_done % 5 == 0 or n_done == len(reqs): + elapsed = time.time() - t0 + rate = n_done / elapsed if elapsed > 0 else 0 + eta = (len(reqs) - n_done) / rate if rate > 0 else 0 + print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | " + f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}", + file=sys.stderr) + if not stream_error: + try: + stdout_classifier.finish() + except EvidenceError as exc: + stream_error = exc + binding_mode = stdout_classifier.binding_mode + proc.wait() + elapsed = time.time() - t0 + # Fatal only in the two cases dev's own contract (and this + # module's own evidence layer) recognize -- see completion_error's + # docstring. Everything else, including a partial request count, + # still reports the accuracy table and exits 0, matching dev. + fatal = completion_error( + proc.returncode,n_done,len(reqs),continuation_tokens,stream_error) + partial = n_done != len(reqs) + evidence_status = "BOUND" if binding_mode == "bound" else "UNBOUND" if out_f: - out_f.write(f"{n_done},{t},{qi},{oi},{clen},{cchars},{gold},{logprob:.6f},{greedy}\n") - out_f.flush() - n_done += 1 - if n_done % 5 == 0 or n_done == len(reqs): - elapsed = time.time() - t0 - rate = n_done / elapsed if elapsed > 0 else 0 - eta = (len(reqs) - n_done) / rate if rate > 0 else 0 - print(f"[progress] {n_done}/{len(reqs)} requests scored | {elapsed:.0f}s elapsed | " - f"{rate:.2f} req/s | ETA {eta:.0f}s | last: {t} q{qi} opt{oi} lp={logprob:.3f}", - file=sys.stderr) - proc.wait() - elapsed = time.time() - t0 - if out_f: - out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, exit={proc.returncode}\n") - out_f.close() - if proc.returncode != 0 and n_done == 0: - print(f"ENGINE ERROR (exit {proc.returncode})", file=sys.stderr); sys.exit(1) - if n_done != len(reqs): - print(f"WARNING: only {n_done}/{len(reqs)} requests scored (engine exited {proc.returncode}); " - f"scoring partial results.", file=sys.stderr) - # Fill any unscored slots with -inf so argmax never picks them - for i in range(len(lp)): - if lp[i] is None: lp[i] = float("-inf") - print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, exit {proc.returncode})", file=sys.stderr) - score_accuracy(tasks, meta, perq, lp) - print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result" - "\n indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)") - os.remove(req_path) + if fatal: + out_f.write(f"# INCOMPLETE: {n_done}/{len(reqs)} in {elapsed:.0f}s, " + f"tokens={continuation_tokens}, exit={proc.returncode}; " + f"error={fatal}\n") + else: + out_f.write(f"# finished: {n_done}/{len(reqs)} in {elapsed:.0f}s, " + f"tokens={continuation_tokens}, exit={proc.returncode}, " + f"evidence={evidence_status}\n") + if partial: + # Additive: the run still finished (exit 0, full + # table below) -- this line only ANNOUNCES that fewer + # than the full request count landed, it does not + # replace the "# finished" line or change the exit + # code dev's own consumers depend on. + out_f.write(f"# INCOMPLETE: {n_done}/{len(reqs)} requests " + f"scored; engine exit={proc.returncode}\n") + out_f.close(); out_f=None + if fatal: + print(f"EVIDENCE INCOMPLETE: {fatal}", file=sys.stderr) + return 1 + if evidence_status == "UNBOUND": + print("engine does not emit score evidence lines; " + "results are unbound", file=sys.stderr) + if partial: + # Same wording and the same exit-0 contract dev used: a + # partial run is a WARNING, not a failure. + print(f"WARNING: only {n_done}/{len(reqs)} requests scored " + f"(engine exited {proc.returncode}); scoring partial " + "results.", file=sys.stderr) + # Fill any unscored slots with -inf so argmax never picks them + # (dev's own fallback for a partial run). + for i in range(len(lp)): + if lp[i] is None: lp[i] = float("-inf") + print(f"(engine: {elapsed:.0f}s, {n_done}/{len(reqs)} scored, " + f"{continuation_tokens} continuation tokens, " + f"exit {proc.returncode})", file=sys.stderr) + score_accuracy(tasks, meta, perq, lp) + print("\nNOTE: compare acc_norm with GLM-5.2's PUBLISHED model-card score. A close result" + "\n indicates that int4 quantization preserved quality. (Fill REFERENCE in tools/eval_glm.py.)") + return 0 + finally: + if previous_sigterm is not None: + signal.signal(signal.SIGTERM, previous_sigterm) + if proc is not None and proc.poll() is None: + # A mid-run exception or termination signal must never leave + # the engine child running: no zombie, no orphan. + proc.terminate() + proc.wait() + if out_f: + out_f.write("# INCOMPLETE: evaluator terminated before a complete denominator\n") + out_f.close() + try: os.remove(req_path) + except FileNotFoundError: pass if __name__ == "__main__": - main() + sys.exit(main() or 0)