Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .flow/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,7 @@ pilot-runs/
.cache/
create-first/
review-fanout/
artifacts/*/pr-cognitive-aid/*.json
artifacts/*/pr-cognitive-aid/.write.lock
# End of auto-managed block. User patterns below this line are preserved.
locks/
349 changes: 349 additions & 0 deletions .flow/artifacts/fn-249-make-pr-measurement/README.md

Large diffs are not rendered by default.

188 changes: 188 additions & 0 deletions .flow/artifacts/fn-249-make-pr-measurement/authored_bytes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Measure omission-only sparse inputs, proving identity with flowctl expansion."""

import copy
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
from unittest.mock import patch

ROOT = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT / "plugins/flow-next/scripts"))
SPEC = importlib.util.spec_from_file_location(
"flowctl", ROOT / "plugins/flow-next/scripts/flowctl.py"
)
flowctl = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = flowctl
SPEC.loader.exec_module(flowctl)
FIELDS = (
"changeType", "additions", "deletions", "diffUrl",
"sourceRefs", "rIds", "taskIds", "attentionClass",
)


def encoded(value):
return flowctl._pr_aid_serialized_text(value).encode("utf-8")


def historical_diff(artifact):
"""Keep flowctl's diff flags/parser, substituting the recorded head for HEAD."""
base, head = artifact["baseSha"], artifact["headSha"]
run_git = flowctl._export_run_git

def at_head(args, **kwargs):
args = [head if arg == "HEAD" else
f"{base}..{head}" if arg == f"{base}..HEAD" else arg
for arg in args]
return run_git(args, **kwargs)

try:
with patch.object(flowctl, "_export_run_git", side_effect=at_head):
return flowctl._pr_aid_live_diff_files(ROOT, base, head), None
except flowctl.PrCognitiveAidValidationError as error:
return None, str(error)


def measure(path, diff_cache, mode="strict"):
if mode not in ("strict", "assumptionsAB"):
raise ValueError(f"unknown measurement mode: {mode}")
raw = (ROOT / path).read_text(encoding="utf-8")
complete = json.loads(raw)
target = encoded(complete)
sparse = copy.deepcopy(complete)
key = (complete["baseSha"], complete["headSha"])
if key not in diff_cache:
diff_cache[key] = historical_diff(complete)
metadata, unavailable = diff_cache[key]
if mode == "assumptionsAB" and metadata is None:
# Assumption A: these values were verified when written, not re-proven here.
metadata = {
row["path"]: (row["changeType"], row["additions"], row["deletions"])
for group in complete["changeWalkthrough"]["groups"]
for row in group["files"]
}
saved = dict.fromkeys((*FIELDS, "wholeRows"), 0)
counts = dict.fromkeys(saved, 0)

def identical(candidate):
errors = []
expanded = flowctl._expand_pr_cognitive_aid_input(
candidate, metadata, _errors=errors
)
if mode == "assumptionsAB":
# Assumption B permits only URLs added where the stored row lacked one.
# Ordered canonical comparison still checks every other leaf and row.
for before_group, after_group in zip(
complete["changeWalkthrough"]["groups"],
expanded["changeWalkthrough"]["groups"], strict=False,
):
for before_row, after_row in zip(before_group["files"], after_group["files"], strict=False):
if "diffUrl" not in before_row:
after_row.pop("diffUrl", None)
return not errors and encoded(expanded) == target

result = {
"path": path, "diskBytes": len(raw.encode("utf-8")),
"completeBytes": len(target), "diffUnavailable": unavailable,
}
if not identical(sparse):
# Carry the original size when this mode cannot prove identity.
result["limit"] = "complete input itself does not expand identically"
errors = []
expanded = flowctl._expand_pr_cognitive_aid_input(
complete, metadata, _errors=errors
)
result["expansionErrors"] = errors
result["addedFields"] = {}
for before_group, after_group in zip(
complete["changeWalkthrough"]["groups"],
expanded["changeWalkthrough"]["groups"], strict=True,
):
for before_row, after_row in zip(before_group["files"], after_group["files"], strict=False):
for field in sorted(after_row.keys() - before_row.keys()):
result["addedFields"][field] = result["addedFields"].get(field, 0) + 1
result["addedRows"] = sum(
len(group["files"]) for group in expanded["changeWalkthrough"]["groups"]
) - sum(len(group["files"]) for group in complete["changeWalkthrough"]["groups"])
else:
groups = sparse["changeWalkthrough"]["groups"]
# Generated rows append to the final available step. Reverse traversal
# removes suffix rows first so ordered reconstruction remains possible.
# Repeat because freeing a slot can enable a previously rejected omission.
changed = True
while changed:
changed = False
for group in reversed(groups):
for index in range(len(group["files"]) - 1, -1, -1):
row = group["files"][index]
if row.get("summary") != "" or metadata is None:
continue
before = len(encoded(sparse))
del group["files"][index]
if identical(sparse):
saved["wholeRows"] += before - len(encoded(sparse))
counts["wholeRows"] += 1
changed = True
else:
group["files"].insert(index, row)
for group in groups:
for row in group["files"]:
for field in FIELDS:
if field not in row:
continue
before = len(encoded(sparse))
value = row.pop(field)
if identical(sparse):
saved[field] += before - len(encoded(sparse))
counts[field] += 1
else:
row[field] = value
if not identical(sparse):
raise RuntimeError(f"identity proof failed: {path}")
result.update(sparseBytes=len(encoded(sparse)), savedBytes=saved, omitted=counts)
assert sum(saved.values()) == len(target) - result["sparseBytes"]
return result


def summarize(rows):
complete = sum(row["completeBytes"] for row in rows)
sparse = sum(row["sparseBytes"] for row in rows)
report = {
"artifacts": len(rows), "diskBytes": sum(row["diskBytes"] for row in rows),
"completeBytes": complete, "sparseBytes": sparse,
"savedBytes": complete - sparse,
"reductionPercent": round(100 * (complete - sparse) / complete, 4) if complete else 0,
"diffUnavailable": sum(row["diffUnavailable"] is not None for row in rows),
"identityUnavailable": sum("limit" in row for row in rows),
"unchangedArtifacts": sum(row["completeBytes"] == row["sparseBytes"] for row in rows),
"fields": {field: {
"omitted": sum(row["omitted"][field] for row in rows),
"savedBytes": sum(row["savedBytes"][field] for row in rows),
} for field in (*FIELDS, "wholeRows")},
"records": rows,
}
return report


def main():
# Historical objects must already be local, even in a partial clone.
os.environ["GIT_NO_LAZY_FETCH"] = "1"
paths = subprocess.run(
["git", "ls-files", "-z", ".flow/artifacts"], cwd=ROOT,
check=True, capture_output=True, encoding="utf-8",
).stdout.split("\0")
paths = sorted(path for path in paths
if "/pr-cognitive-aid/" in path and path.endswith(".json"))
cache = {}
report = {
mode: summarize([measure(path, cache, mode=mode) for path in paths])
for mode in ("strict", "assumptionsAB")
}
print(json.dumps(report, indent=2, ensure_ascii=False))


if __name__ == "__main__":
main()
70 changes: 70 additions & 0 deletions .flow/artifacts/fn-249-make-pr-measurement/measure.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/usr/bin/env bash
# make-pr measurement harness (fn-249 R8 / fn-252 R11).
# usage: measure.sh <point-label> <plugin-dir> <run-number>
# Runs one headless `make-pr --dry-run` on the fixed fixture and appends one
# JSON line (output tokens, tool calls, wall clock) to <point-label>/runs.jsonl.
set -euo pipefail

POINT="$1"; PLUGIN_DIR="$2"; RUN="$3"
ROOT="${MEASURE_ROOT:-$HOME/.cache/flow-next-measure}"
FIX="$ROOT/pr449"
SPEC="fn-248-capture-the-resolved-spec-template"
BASE="07905c2c36b6f96eea815e120e117fff6d2bc8dc"
HEAD_SHA="81990b699b4f352fdb31af5a7da98095311973a2"
MODEL="${MEASURE_MODEL:-claude-fable-5-1}"
OUT="$ROOT/$POINT"; mkdir -p "$OUT"

# Fresh fixture every run: same head, no prior aid artifact, no leftovers.
git -C "$FIX" reset -q --hard "$HEAD_SHA"
git -C "$FIX" clean -qfdx -- .flow/artifacts .flow/tmp 2>/dev/null || true
[ "$(git -C "$FIX" rev-parse HEAD)" = "$HEAD_SHA" ]
# A run picks its own scratch directory, under the user cache or beside the fixture.
# Park every leftover so the next run starts cold: anything matching make-pr-* in the
# user cache, and anything in the harness root that is not the fixture, the harness,
# a point directory or the parking area itself.
PARK="$ROOT/parked/$POINT-before-run$RUN"
for d in "$HOME"/.cache/make-pr-* "$ROOT"/*; do
[ -e "$d" ] || continue
case "$(basename "$d")" in pr449|measure.sh|parked|p[0-9]-*) [ "$(dirname "$d")" = "$ROOT" ] && continue ;; esac
mkdir -p "$PARK"; mv "$d" "$PARK/"
done

STREAM="$OUT/run$RUN.stream.jsonl"
START=$(date +%s.%N)
( cd "$FIX" && FLOWCTL="$PLUGIN_DIR/scripts/flowctl" claude -p \
"/flow-next:make-pr $SPEC --dry-run --base $BASE" \
--model "$MODEL" --setting-sources project,local \
--plugin-dir "$PLUGIN_DIR" --permission-mode bypassPermissions \
--no-session-persistence --output-format stream-json --verbose \
</dev/null >"$STREAM" 2>"$OUT/run$RUN.stderr" ) || echo "claude exit $?" >>"$OUT/run$RUN.stderr"
END=$(date +%s.%N)

python3 - "$STREAM" "$POINT" "$RUN" "$MODEL" "$START" "$END" >>"$OUT/runs.jsonl" <<'PY'
import json, sys
stream, point, run, model, start, end = sys.argv[1:7]
tools = 0; result = {}; out_by_msg = {}
for line in open(stream):
try: ev = json.loads(line)
except ValueError: continue
if ev.get("type") == "assistant":
msg = ev.get("message", {})
tools += sum(1 for b in msg.get("content", []) if b.get("type") == "tool_use")
if msg.get("id"):
out_by_msg[msg["id"]] = msg.get("usage", {}).get("output_tokens", 0)
elif ev.get("type") == "result":
result = ev
usage = result.get("usage", {})
print(json.dumps({
"point": point, "run": int(run), "model": model,
"output_tokens": usage.get("output_tokens"),
"output_tokens_stream_sum": sum(out_by_msg.values()),
"tool_calls": tools,
"wall_clock_s": round(float(end) - float(start), 1),
"duration_ms": result.get("duration_ms"),
"num_turns": result.get("num_turns"),
"is_error": result.get("is_error"),
"subtype": result.get("subtype"),
"result_chars": len(result.get("result") or ""),
}))
PY
tail -1 "$OUT/runs.jsonl"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"point": "p0-baseline", "run": 1, "model": "claude-fable-5-1", "output_tokens": 31698, "output_tokens_stream_sum": 1100, "tool_calls": 22, "wall_clock_s": 325.3, "duration_ms": 322721, "num_turns": 24, "is_error": false, "subtype": "success", "result_chars": 28134}
{"point": "p0-baseline", "run": 2, "model": "claude-fable-5-1", "output_tokens": 21389, "output_tokens_stream_sum": 1038, "tool_calls": 22, "wall_clock_s": 251.1, "duration_ms": 249180, "num_turns": 24, "is_error": false, "subtype": "success", "result_chars": 3048}
{"point": "p0-baseline", "run": 3, "model": "claude-fable-5-1", "output_tokens": 21627, "output_tokens_stream_sum": 1006, "tool_calls": 18, "wall_clock_s": 249.6, "duration_ms": 247394, "num_turns": 20, "is_error": false, "subtype": "success", "result_chars": 6889}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"point": "p1-after-input", "run": 1, "model": "claude-fable-5-1", "output_tokens": 25653, "output_tokens_stream_sum": 147, "tool_calls": 18, "wall_clock_s": 273.3, "duration_ms": 271095, "num_turns": 20, "is_error": false, "subtype": "success", "result_chars": 24765}
{"point": "p1-after-input", "run": 2, "model": "claude-fable-5-1", "output_tokens": 18031, "output_tokens_stream_sum": 1023, "tool_calls": 22, "wall_clock_s": 212.3, "duration_ms": 210358, "num_turns": 24, "is_error": false, "subtype": "success", "result_chars": 2848}
{"point": "p1-after-input", "run": 3, "model": "claude-fable-5-1", "output_tokens": 29223, "output_tokens_stream_sum": 212, "tool_calls": 20, "wall_clock_s": 309.8, "duration_ms": 307710, "num_turns": 22, "is_error": false, "subtype": "success", "result_chars": 27598}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"point": "p2-after-briefing", "run": 1, "model": "claude-fable-5-1", "output_tokens": 11069, "output_tokens_stream_sum": 186, "tool_calls": 15, "wall_clock_s": 133.3, "duration_ms": 132780, "num_turns": 17, "is_error": false, "subtype": "success", "result_chars": 4070}
{"point": "p2-after-briefing", "run": 2, "model": "claude-fable-5-1", "output_tokens": 10717, "output_tokens_stream_sum": 168, "tool_calls": 15, "wall_clock_s": 129.3, "duration_ms": 128716, "num_turns": 17, "is_error": false, "subtype": "success", "result_chars": 3022}
{"point": "p2-after-briefing", "run": 3, "model": "claude-fable-5-1", "output_tokens": 10918, "output_tokens_stream_sum": 214, "tool_calls": 15, "wall_clock_s": 125.4, "duration_ms": 124903, "num_turns": 17, "is_error": false, "subtype": "success", "result_chars": 4648}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"point": "p3-after-readable", "run": 1, "model": "claude-fable-5-1", "output_tokens": 10206, "output_tokens_stream_sum": 228, "tool_calls": 16, "wall_clock_s": 117.0, "duration_ms": 116423, "num_turns": 18, "is_error": false, "subtype": "success", "result_chars": 6401}
{"point": "p3-after-readable", "run": 2, "model": "claude-fable-5-1", "output_tokens": 11908, "output_tokens_stream_sum": 1006, "tool_calls": 19, "wall_clock_s": 141.4, "duration_ms": 140872, "num_turns": 21, "is_error": false, "subtype": "success", "result_chars": 7147}
{"point": "p3-after-readable", "run": 3, "model": "claude-fable-5-1", "output_tokens": 11787, "output_tokens_stream_sum": 246, "tool_calls": 19, "wall_clock_s": 137.5, "duration_ms": 136905, "num_turns": 21, "is_error": false, "subtype": "success", "result_chars": 6376}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"point": "p4-after-polish", "run": 1, "model": "claude-fable-5-1", "output_tokens": 7946, "output_tokens_stream_sum": 889, "tool_calls": 13, "wall_clock_s": 100.1, "duration_ms": 99578, "num_turns": 15, "is_error": false, "subtype": "success", "result_chars": 2026}
{"point": "p4-after-polish", "run": 2, "model": "claude-fable-5-1", "output_tokens": 9075, "output_tokens_stream_sum": 921, "tool_calls": 14, "wall_clock_s": 218.6, "duration_ms": 218074, "num_turns": 16, "is_error": false, "subtype": "success", "result_chars": 2044}
{"point": "p4-after-polish", "run": 3, "model": "claude-fable-5-1", "output_tokens": 8174, "output_tokens_stream_sum": 879, "tool_calls": 11, "wall_clock_s": 96.9, "duration_ms": 96380, "num_turns": 13, "is_error": false, "subtype": "success", "result_chars": 2781}
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,16 @@ Squash also orphans the stack's history: the stacked branch still contains the b
2. Force-push with lease; open a **successor PR** against main (the old PR number is lost — link it with "Supersedes #N" in the body).
3. Squash-orphaned bookkeeping follows: task evidence commits and rebaseline-evidence baseline SHAs recorded on the stack point at commits the squash removed — repoint receipts at the squash SHA and regenerate evidence against a reachable baseline (codex flagged all three on #374).

## Superseded by the chain rules (2026-09-13)
## Superseded by the chain rules (updated 2026-09-20)

The manual successor-PR playbook above is history. Dependent specs now build as **chains** (fn-152) and land drains them (fn-149):
The manual successor-PR playbook above records the earlier incident. Dependent specs build as **chains** and use native stacks where supported:

- `flowctl spec chain <id>` decides when a dependent spec may start (parent open, all tasks done, branch on origin; linear only). Work branches from the parent's remote tip; make-pr targets the parent's branch and links a GitHub stack. Rules: `plugins/flow-next/skills/flow-next-make-pr/workflow.md` §0.3, `flow-next-work/phases.md` Phase 2.
- Land never deletes a branch while an open PR targets it (`pending_branch_deletes`), merges only the frontier, and retargets the layers above a merged parent itself with a leased force-push. Rules: `plugins/flow-next/skills/flow-next-land/references/chains-and-stacks.md`.
- `flowctl spec chain <id>` decides when a dependent spec may start (parent not landed at the base, all tasks done, branch on origin; linear only). Work branches from the parent's remote tip; make-pr targets the parent's branch and links a GitHub stack. Rules: `plugins/flow-next/skills/flow-next-make-pr/workflow.md` §0.3, `flow-next-work/phases.md` Phase 2.
- Land links open children into a native stack before merging and merges only the lowest open layer, one layer per run. GitHub retargets and rebases stack children. Land never deletes a branch while an open PR targets it and never rebases, force-pushes, or retargets a child.
- Without stacks, a conflicted child needs a separately authorized manual single-layer rebase after the parent merges. Follow `plugins/flow-next/docs/troubleshooting.md` §"Land on a chain"; inspect the parent's pre-merge tip, rebase only the child, and re-read its checks and reviews before landing.
- The merged-parent window before a child has a PR is make-pr's rebase-onto from the detected boundary (create run only), so no successor PR is needed.

## Avoiding it next time

- Do not hand-build a dependent PR on a feature branch outside the chain rules; let work and make-pr build it so land can drain it.
- Do not hand-build a dependent PR on a feature branch outside the chain rules; let work and make-pr build it, then land each lowest open layer.
- Related GitHub sharp edge from the same run: a comma list after one closing keyword ("Fixes #A, #B, #C") auto-closes only #A — each issue needs its own keyword ("Fixes #A, fixes #B, fixes #C").
Loading
Loading