Skip to content

Commit 55fa9ef

Browse files
hyperpolymathclaude
andcommitted
feat(echidnabot): add standalone batch_driver for proof_attempts bootstrap
Walks local proof trees (ephapax, echidna, verisimdb), calls echidna /api/verify per file, POSTs outcomes to verisim-api /api/v1/proof_attempts, emits notify-send per result. Bypasses echidnabot's clone-first pipeline to bootstrap training data for the proof_strategy_selection MV directly from local working trees. First real run produced 29 new proof_attempts rows (16 success, 17 failure, 1 timeout) across 6 obligation classes and 5 provers (z3, coq, lean, agda, idris2), making the strategy endpoint data-driven for the first time. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent c321188 commit 55fa9ef

1 file changed

Lines changed: 204 additions & 0 deletions

File tree

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: PMPL-1.0-or-later
3+
# batch_driver.sh — walk local proof trees, verify via echidna, record in verisim-api.
4+
#
5+
# For each configured (repo, glob, prover, obligation_class) target:
6+
# 1. Read proof file content
7+
# 2. POST to echidna /api/verify with (prover, content) → valid/invalid + duration
8+
# 3. POST to verisim-api /api/v1/proof_attempts with outcome + duration
9+
# 4. Emit desktop notification via notify-send
10+
#
11+
# Bypasses echidnabot's clone-based pipeline to bootstrap training data for
12+
# the proof_strategy_selection MV directly from local working trees.
13+
#
14+
# Environment:
15+
# ECHIDNA_URL (default http://127.0.0.1:8090)
16+
# VERISIM_URL (default http://127.0.0.1:8080)
17+
# NOTIFY (1=notify-send on each result, 0=silent; default 1)
18+
# DRY_RUN (1=print only, no HTTP; default 0)
19+
# MAX_FILES (cap per target, default 10 to avoid flooding)
20+
# STRATEGY_TAG (column stored in proof_attempts, default "batch-manual")
21+
22+
set -euo pipefail
23+
24+
ECHIDNA_URL="${ECHIDNA_URL:-http://127.0.0.1:8090}"
25+
VERISIM_URL="${VERISIM_URL:-http://127.0.0.1:8080}"
26+
NOTIFY="${NOTIFY:-1}"
27+
DRY_RUN="${DRY_RUN:-0}"
28+
MAX_FILES="${MAX_FILES:-10}"
29+
STRATEGY_TAG="${STRATEGY_TAG:-batch-manual}"
30+
31+
REPOS_ROOT="/var/mnt/eclipse/repos"
32+
33+
# Target definitions: repo | obligation_class | prover | glob-root | extension
34+
# obligation_class values match ClickHouse Enum8 in proof_attempts.sql:
35+
# safety, linearity, termination, equiv, correctness, confluence, totality,
36+
# invariant, refinement, model-check, other
37+
TARGETS=(
38+
"echidna|safety|Z3|examples|smt2"
39+
"echidna|safety|Z3|proofs/z3|smt2"
40+
"echidna|equiv|Coq|proofs/coq|v"
41+
"echidna|equiv|Lean|proofs/lean|lean"
42+
"echidna|equiv|Agda|proofs/agda|agda"
43+
"echidna|totality|Idris2|verification/proofs/idris2|idr"
44+
"ephapax|equiv|Coq|formal|v"
45+
"ephapax|linearity|Idris2|src/formal/Ephapax/Formal|idr"
46+
"verisimdb|totality|Idris2|verification/proofs/idris2|idr"
47+
)
48+
49+
# --- helpers ----------------------------------------------------------------
50+
51+
prover_lower() {
52+
# echidna REST names (Z3/Coq/Lean/Idris2/Agda) → verisim-api enum (lowercase)
53+
case "$1" in
54+
Z3) echo "z3" ;;
55+
Coq) echo "coq" ;;
56+
Lean) echo "lean" ;;
57+
Idris2) echo "idris2" ;;
58+
Agda) echo "agda" ;;
59+
CVC5|Cvc5) echo "cvc5" ;;
60+
*) echo "other" ;;
61+
esac
62+
}
63+
64+
obligation_id_for() {
65+
# Stable SHA-256 prefix of repo|file|claim → hex string. ClickHouse column
66+
# is String, not UUID, so any stable identifier works.
67+
printf "%s|%s|%s" "$1" "$2" "$3" | sha256sum | cut -c1-40
68+
}
69+
70+
iso_now() { date -u +'%Y-%m-%dT%H:%M:%S.000'; } # ClickHouse DateTime64(3), no Z
71+
uuidv4() { cat /proc/sys/kernel/random/uuid; }
72+
73+
notify() {
74+
# $1 = urgency (normal|critical), $2 = title, $3 = body
75+
[ "$NOTIFY" = "1" ] || return 0
76+
notify-send --urgency="$1" --app-name="echidna-batch" "$2" "$3" 2>/dev/null || true
77+
}
78+
79+
# --- per-file verification --------------------------------------------------
80+
81+
verify_one() {
82+
local repo="$1" obligation_class="$2" prover="$3" file="$4"
83+
local rel_file="${file#"$REPOS_ROOT/$repo/"}"
84+
local claim
85+
claim=$(head -c 200 "$file" | tr '\n' ' ' | tr -s ' ')
86+
local obl_id
87+
obl_id=$(obligation_id_for "$repo" "$rel_file" "$claim")
88+
local attempt_id
89+
attempt_id=$(uuidv4)
90+
local started
91+
started=$(iso_now)
92+
93+
# POST /api/verify to echidna
94+
local verify_body
95+
verify_body=$(jq -Rs --arg p "$prover" '{prover:$p, content:.}' <"$file")
96+
97+
local t0_ms t1_ms duration_ms
98+
t0_ms=$(date +%s%3N)
99+
local verify_resp
100+
if [ "$DRY_RUN" = "1" ]; then
101+
verify_resp='{"valid":true,"goals_remaining":0}'
102+
else
103+
verify_resp=$(curl -sS -X POST "$ECHIDNA_URL/api/verify" \
104+
-H 'Content-Type: application/json' -d "$verify_body" \
105+
--max-time 60 || echo '{"valid":false,"error":"http_failure"}')
106+
fi
107+
t1_ms=$(date +%s%3N)
108+
duration_ms=$((t1_ms - t0_ms))
109+
110+
local valid outcome
111+
valid=$(jq -r '.valid // false' <<<"$verify_resp")
112+
if [ "$valid" = "true" ]; then
113+
outcome="success"
114+
elif jq -e 'has("error")' <<<"$verify_resp" >/dev/null; then
115+
outcome="unknown"
116+
else
117+
outcome="failure"
118+
fi
119+
120+
local completed
121+
completed=$(iso_now)
122+
local prover_lc
123+
prover_lc=$(prover_lower "$prover")
124+
125+
# POST /api/v1/proof_attempts to verisim-api
126+
local insert_body
127+
insert_body=$(jq -n \
128+
--arg attempt_id "$attempt_id" \
129+
--arg obligation_id "$obl_id" \
130+
--arg repo "hyperpolymath/$repo" \
131+
--arg file "$rel_file" \
132+
--arg claim "$claim" \
133+
--arg obligation_class "$obligation_class" \
134+
--arg prover_used "$prover_lc" \
135+
--arg outcome "$outcome" \
136+
--argjson duration_ms "$duration_ms" \
137+
--argjson confidence 0.5 \
138+
--arg strategy_tag "$STRATEGY_TAG" \
139+
--arg started_at "$started" \
140+
--arg completed_at "$completed" \
141+
--arg prover_output "$(printf '%s' "$verify_resp" | head -c 512)" \
142+
'{attempt_id:$attempt_id, obligation_id:$obligation_id, repo:$repo, file:$file,
143+
claim:$claim, obligation_class:$obligation_class, prover_used:$prover_used,
144+
outcome:$outcome, duration_ms:$duration_ms, confidence:$confidence,
145+
parent_attempt_id:null, strategy_tag:$strategy_tag,
146+
started_at:$started_at, completed_at:$completed_at,
147+
prover_output:$prover_output, error_message:null}')
148+
149+
if [ "$DRY_RUN" != "1" ]; then
150+
curl -sS -X POST "$VERISIM_URL/api/v1/proof_attempts" \
151+
-H 'Content-Type: application/json' -d "$insert_body" \
152+
--max-time 10 >/dev/null || echo " insert failed" >&2
153+
fi
154+
155+
# Emit notification + log line
156+
local icon urgency
157+
case "$outcome" in
158+
success) icon=""; urgency="normal" ;;
159+
failure) icon=""; urgency="critical" ;;
160+
*) icon="?"; urgency="low" ;;
161+
esac
162+
printf " %s %-8s %-7s %5dms %s\n" \
163+
"$icon" "$outcome" "$prover_lc" "$duration_ms" "$rel_file"
164+
notify "$urgency" \
165+
"$icon $repo: $outcome" \
166+
"$prover_lc on $rel_file (${duration_ms}ms, class=$obligation_class)"
167+
}
168+
169+
# --- main loop --------------------------------------------------------------
170+
171+
echo "batch_driver: echidna=$ECHIDNA_URL verisim=$VERISIM_URL dry_run=$DRY_RUN"
172+
echo
173+
174+
for target in "${TARGETS[@]}"; do
175+
IFS='|' read -r repo obligation_class prover glob_root extension <<<"$target"
176+
dir="$REPOS_ROOT/$repo/$glob_root"
177+
if [ ! -d "$dir" ]; then
178+
echo "[$repo] SKIP: $dir not present"
179+
continue
180+
fi
181+
182+
# shellcheck disable=SC2207
183+
files=($(find "$dir" -type f -name "*.${extension}" 2>/dev/null | head -n "$MAX_FILES"))
184+
if [ "${#files[@]}" -eq 0 ]; then
185+
echo "[$repo] SKIP: no *.${extension} under $glob_root"
186+
continue
187+
fi
188+
189+
echo "[$repo] target class=$obligation_class prover=$prover glob=$glob_root/*.$extension (${#files[@]} files)"
190+
for f in "${files[@]}"; do
191+
verify_one "$repo" "$obligation_class" "$prover" "$f"
192+
done
193+
echo
194+
done
195+
196+
# Summary
197+
if [ "$DRY_RUN" != "1" ]; then
198+
echo "=== current strategy snapshot ==="
199+
for cls in safety linearity termination equiv correctness totality; do
200+
row=$(curl -sS "$VERISIM_URL/api/v1/proof_attempts/strategy?class=$cls&limit=3" 2>/dev/null)
201+
top=$(jq -r '.recommendations[0] // empty | "top=\(.prover) rate=\(.success_rate) n=\(.total_attempts)"' <<<"$row" 2>/dev/null)
202+
[ -n "$top" ] && printf " %-12s %s\n" "$cls" "$top"
203+
done
204+
fi

0 commit comments

Comments
 (0)