diff --git a/5dive-agent-stop-notify b/5dive-agent-stop-notify new file mode 100755 index 000000000..c141b1248 --- /dev/null +++ b/5dive-agent-stop-notify @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# 5dive-agent-stop-notify — ExecStopPost notifier for 5dive-agent@.service. +# +# DIVE-3965 (P06). The channel plugins can tell you a turn was interrupted, but +# only from INSIDE a process that is still alive. The three states that matter +# most — SIGKILL, OOM, and a permanent config failure that trips +# RestartPreventExitStatus — kill the notifier along with everything else, so +# the chat's last word is whatever the agent happened to be saying. systemd is +# the only observer that outlives the unit, and ExecStopPost is where it speaks. +# +# WHY IT IS A SEPARATE SCRIPT AND NOT `5dive ...`: +# ExecStopPost runs as User=agent-%i with no sudo, and it must survive a +# half-installed bundle (a `5dive` that is mid-self-update is exactly when an +# agent is most likely to be dying). This file therefore depends on bash, curl +# and coreutils only, reads no state it cannot read as the agent, and ALWAYS +# exits 0 — a notifier must never be the reason a stop is recorded as failed. +# The unit also prefixes it with `-` for the same reason, belt and braces. +# +# WHAT systemd HANDS US (see systemd.service(5), "Environment variables in +# spawned processes"): +# SERVICE_RESULT success | protocol | timeout | exit-code | signal | +# core-dump | watchdog | start-limit-hit | resources | oom-kill +# EXIT_CODE exited | killed | dumped +# EXIT_STATUS numeric exit status, or the signal name for killed/dumped +set -uo pipefail + +NAME="${1:-}" +[[ -n "$NAME" ]] || exit 0 + +HOME_DIR="${STOP_NOTIFY_HOME:-/home/agent-${NAME}}" +REGISTRY="${STOP_NOTIFY_REGISTRY:-/var/lib/5dive/agents.json}" +STATE_DIR="${STOP_NOTIFY_STATE_DIR:-${HOME_DIR}/.5dive/stop-notify}" +DEDUP_SECS="${STOP_NOTIFY_DEDUP_SECS:-900}" +NOW="${STOP_NOTIFY_NOW:-$(date +%s)}" +SEND_CMD="${STOP_NOTIFY_SEND_CMD:-}" + +RESULT="${SERVICE_RESULT:-unknown}" +EXIT_KIND="${EXIT_CODE:-}" +EXIT_VAL="${EXIT_STATUS:-}" + +# ── the decision, kept as one function so the harness can grade it directly ── +# +# Prints "|" or nothing at all when the stop is not worth a +# message. The cause key is what dedup counts, so two different causes inside +# one window BOTH get through: an OOM followed by a config failure is two +# findings, not a repeat. +classify_stop() { # + local result="$1" kind="$2" val="$3" desired="$4" + + # A deliberate stop is not an incident. Two independent readings say so, and + # either is enough: systemd's own verdict, and the operator intent DIVE-857 + # records in the registry at `5dive agent stop`. The second is load-bearing + # because a launcher that dies on SIGTERM reports `signal/SIGTERM`, which is + # byte-identical to a kill — without the intent flag every clean `agent stop` + # would page the chat. + [[ "$desired" == "stopped" ]] && return 0 + [[ "$result" == "success" ]] && return 0 + if [[ "$kind" == "killed" && ( "$val" == "TERM" || "$val" == "SIGTERM" || "$val" == "15" ) ]]; then + return 0 + fi + + case "$result" in + oom-kill) + printf 'oom|ran out of memory and was killed by the kernel. It will not come back on its own if the box is still under pressure.\n' ;; + watchdog) + printf 'watchdog|stopped responding to its own watchdog and was killed.\n' ;; + start-limit-hit) + printf 'crash-loop|failed to start too many times in a row, so systemd gave up. It stays down until someone clears it.\n' ;; + timeout) + printf 'timeout|did not shut down in time and was killed.\n' ;; + resources) + printf 'resources|could not be started — the box could not give it the resources it asked for.\n' ;; + core-dump) + printf 'core-dump|crashed hard (signal %s) and dumped core.\n' "${val:-unknown}" ;; + signal) + printf 'signal-%s|was killed by signal %s.\n' "${val:-unknown}" "${val:-unknown}" ;; + exit-code) + # 2 and 3 are 5dive-agent-start's permanent conditions and the unit's + # RestartPreventExitStatus: no retry is coming, which is the part a + # person needs to be told. + case "$val" in + 2) printf 'permanent-2|could not start: its agent type is not one this box knows how to run. It will NOT be retried.\n' ;; + 3) printf 'permanent-3|could not start: its CLI or plugin is not installed. It will NOT be retried.\n' ;; + *) printf 'exit-%s|exited with error status %s.\n' "${val:-unknown}" "${val:-unknown}" ;; + esac ;; + protocol|*) + printf 'other-%s|stopped unexpectedly (systemd: %s).\n' "$result" "$result" ;; + esac +} + +desired_state() { + [[ -r "$REGISTRY" ]] || return 0 + command -v jq >/dev/null 2>&1 || return 0 + jq -r --arg n "$NAME" '.agents[$n].desiredState // ""' "$REGISTRY" 2>/dev/null +} + +verdict="$(classify_stop "$RESULT" "$EXIT_KIND" "$EXIT_VAL" "$(desired_state)")" +[[ -n "$verdict" ]] || exit 0 +cause="${verdict%%|*}" +sentence="${verdict#*|}" + +# ── dedup: a crash-loop is one finding, not forty ─────────────────────────── +# +# Keyed on the CAUSE, so a different failure inside the window still gets +# through. The suppressed repeats are counted rather than dropped: the next +# message that does go out says how many there were, which is the difference +# between "it crashed" and "it has crashed 38 times". +mkdir -p "$STATE_DIR" 2>/dev/null || true +mark="${STATE_DIR}/${NAME}.last" +prev_cause="" prev_at=0 prev_count=0 +if [[ -r "$mark" ]]; then read -r prev_cause prev_at prev_count < "$mark" 2>/dev/null || true; fi +[[ "$prev_at" =~ ^[0-9]+$ ]] || prev_at=0 +[[ "$prev_count" =~ ^[0-9]+$ ]] || prev_count=0 + +repeats=0 +if [[ "$prev_cause" == "$cause" ]] && (( NOW - prev_at < DEDUP_SECS )); then + # Inside the window: count it and say nothing. + printf '%s %s %s\n' "$cause" "$prev_at" "$(( prev_count + 1 ))" > "$mark" 2>/dev/null || true + exit 0 +fi +[[ "$prev_cause" == "$cause" ]] && repeats="$prev_count" +printf '%s %s 0\n' "$cause" "$NOW" > "$mark" 2>/dev/null || true + +text="⚠️ Agent ${NAME} ${sentence}" +if (( repeats > 0 )); then + text+=$'\n'"(${repeats} further occurrence(s) were suppressed in the last $(( DEDUP_SECS / 60 )) minutes.)" +fi + +# ── delivery ──────────────────────────────────────────────────────────────── +# +# The unit's own EnvironmentFile already put TELEGRAM_BOT_TOKEN in our +# environment, and ExecStopPost inherits it. The paired chats are the +# allowFrom list of whichever runtime's channel state this seat has. +token="${TELEGRAM_BOT_TOKEN:-}" +[[ -n "$token" ]] || exit 0 + +access="" +for d in .codex .claude .grok .gemini; do + cand="${HOME_DIR}/${d}/channels/telegram/access.json" + [[ -r "$cand" ]] && { access="$cand"; break; } +done +[[ -n "$access" ]] || exit 0 +command -v jq >/dev/null 2>&1 || exit 0 + +mapfile -t chats < <(jq -r '.allowFrom // [] | .[]' "$access" 2>/dev/null | head -5) +for chat in "${chats[@]:-}"; do + [[ -n "$chat" ]] || continue + if [[ -n "$SEND_CMD" ]]; then + "$SEND_CMD" "$chat" "$text" || true + else + curl -fsS -m 10 -X POST "https://api.telegram.org/bot${token}/sendMessage" \ + --data-urlencode "chat_id=${chat}" \ + --data-urlencode "text=${text}" >/dev/null 2>&1 || true + fi +done +exit 0 diff --git a/changelog.d/DIVE-3965.md b/changelog.d/DIVE-3965.md new file mode 100644 index 000000000..c83b61048 --- /dev/null +++ b/changelog.d/DIVE-3965.md @@ -0,0 +1,19 @@ +## Unreleased — feat(agents): a crashed agent now says why it died, once, in its own chat (DIVE-3965) + +- A Codex (or any) channel agent that was SIGKILLed, OOM-killed, or stopped by a permanent + start failure could not report it: every process that could have sent the message died + with it, so the chat's last word was whatever the agent happened to be saying. systemd is + the only observer that outlives the unit, so the notification moves to `ExecStopPost`. +- `5dive-agent-stop-notify` classifies the stop from systemd's own verdict — out of memory, + killed by signal, start-limit crash-loop, stop timeout, and the two permanent conditions + (`RestartPreventExitStatus=2 3`) that will never be retried — and says which one happened + in plain words. +- A deliberate stop is silent: both systemd's `success` result and the operator intent + recorded in the registry at `5dive agent stop` are honoured, so a restart does not page + anyone. +- Repeats of the SAME cause inside 15 minutes are suppressed and counted, and the next + message that does go out says how many were held back. A DIFFERENT cause in that window + is always sent, so a crash-loop can never hide an OOM. +- The notifier depends on bash, curl and jq only, and always exits 0 — an agent is most + likely to be dying exactly when the `5dive` bundle is mid-update, and an `ExecStopPost` + must never turn a clean stop into a failed unit. diff --git a/docker/Dockerfile b/docker/Dockerfile index 2337ef0da..a9fc0c63f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -53,7 +53,7 @@ RUN systemctl mask \ # `curl: (37) Couldn't open file` — this list and install.sh's fetches are two # copies of one set, and only the docker-install job notices when they disagree. # (DIVE-3269 added 5dive-stage-fork-plugins.sh and this line went red first.) -COPY 5dive 5dive-agent-list-snapshot 5dive-agent-start 5dive-refresh-plugins.sh 5dive-stage-fork-plugins.sh 5dive-refresh-skills.sh install.sh projects-CLAUDE.md telegram-agent-CLAUDE.md model-tiering-CLAUDE.md operational-comms-CLAUDE.md /opt/5dive-bundle/ +COPY 5dive 5dive-agent-list-snapshot 5dive-agent-start 5dive-agent-stop-notify 5dive-refresh-plugins.sh 5dive-stage-fork-plugins.sh 5dive-refresh-skills.sh install.sh projects-CLAUDE.md telegram-agent-CLAUDE.md model-tiering-CLAUDE.md operational-comms-CLAUDE.md /opt/5dive-bundle/ COPY hooks /opt/5dive-bundle/hooks/ COPY skills /opt/5dive-bundle/skills/ COPY systemd /opt/5dive-bundle/systemd/ diff --git a/install.sh b/install.sh index d295a5083..115ec53dc 100755 --- a/install.sh +++ b/install.sh @@ -962,6 +962,34 @@ JOURNALD chmod 755 "$BIN_DIR/5dive-agent-start" ok "5dive-agent-start → $BIN_DIR/5dive-agent-start" + # DIVE-3965: the unit's ExecStopPost notifier. Installed next to the launcher + # because it shares the launcher's constraint — it has to work when the + # bundle itself is the thing that is broken. + # + # OPTIONAL AT THE PIN, and that is the whole reason this is not the bare + # `curl -fsSL` the launcher above uses. This file is ADDITIVE and the fleet + # pin predates the tag that ships it, so a fail-closed fetch here 404s on + # every fresh install and on every box's 04:00Z self-update and aborts the + # WHOLE install under `set -e` (DIVE-4349; caught by + # scripts/install-pin-compat.sh before it reached a box). A box that does not + # get the notifier simply has no ExecStopPost target — which the unit's `-` + # prefix already tolerates: the notification is missing, the unit still stops. + # + # MOVE IT UP into the fail-closed set only once the fleet pin is at or past + # the tag that ships it; install-pin-compat grades that on every PR touching + # this file, so "the pin has caught up" stays a check result, not a memory. + if fetch_optional_at_pin "5dive-agent-stop-notify" "$BIN_DIR/5dive-agent-stop-notify"; then + chmod 755 "$BIN_DIR/5dive-agent-stop-notify" + ok "5dive-agent-stop-notify → $BIN_DIR/5dive-agent-stop-notify" + else + # Symmetric with the optional-hook loop below: the bundle at this pin does + # not wire it, so a copy left from a newer install would be a file nothing + # invokes (the unit template comes from the same pin and carries no + # ExecStopPost line there). + rm -f "$BIN_DIR/5dive-agent-stop-notify" + say "5dive-agent-stop-notify is not shipped at this pin — agent crash notifications stay off on this box until the fleet pin passes the tag that carries it" + fi + # >>> DIVE-4194 box-side scripts # The helper scripts below used to be three hand-written curl/chmod/ok blocks # right here, and this file was their ONLY writer. The control plane's nightly @@ -1574,7 +1602,7 @@ if [[ "${1:-}" == "--uninstall" ]]; then systemctl daemon-reload || true # 3. Binaries + shared libs - rm -f "$BIN_DIR/5dive" "$BIN_DIR/5dive-agent-start" + rm -f "$BIN_DIR/5dive" "$BIN_DIR/5dive-agent-start" "$BIN_DIR/5dive-agent-stop-notify" ok "removed CLI binaries" if [[ -d "$LIB_DIR" ]]; then rm -rf "$LIB_DIR" diff --git a/systemd/5dive-agent@.service b/systemd/5dive-agent@.service index f69a82bea..7f18e5e00 100644 --- a/systemd/5dive-agent@.service +++ b/systemd/5dive-agent@.service @@ -43,6 +43,11 @@ ExecStart=/usr/local/bin/5dive-agent-start %i # when agy exits and then also gets the graceful cgroup SIGTERM, so its poll is # still aborted cleanly. Stop drops from 15s to ~0.05s for agy. ExecStop=/bin/bash -c '/usr/bin/tmux kill-session -t agent-%i 2>/dev/null; /usr/bin/pkill -HUP -u agent-%i -x agy 2>/dev/null; true' +# DIVE-3965: the only observer that outlives a SIGKILL, an OOM, or a permanent +# start failure. `-` so a notifier fault can never turn a clean stop into a +# failed one; the script exits 0 unconditionally for the same reason, and stays +# silent for a deliberate `5dive agent stop` and for a clean exit. +ExecStopPost=-/usr/local/bin/5dive-agent-stop-notify %i Restart=on-failure RestartSec=3 # A missing binary or an unknown agent type is a PERMANENT condition — no amount diff --git a/tests/agent_stop_notify_unit.sh b/tests/agent_stop_notify_unit.sh new file mode 100755 index 000000000..4c06f222f --- /dev/null +++ b/tests/agent_stop_notify_unit.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# DIVE-3965 — `5dive-agent-stop-notify` is the ExecStopPost half of Codex channel +# crash parity: the notification that has to survive the death of every process +# that could otherwise have sent it. +# +# Graded by RUNNING the shipped script with the environment systemd actually +# hands an ExecStopPost (SERVICE_RESULT/EXIT_CODE/EXIT_STATUS), against a stub +# sender that records its argv. Not by sourcing the classifier: the thing that +# breaks in this file is the wiring between the classifier, the dedup mark and +# the allowFrom read, and a harness that calls one function reaches none of it. +# +# Every negative arm pins the ABSENCE of a send AND a zero exit — an +# ExecStopPost that exits non-zero turns a clean stop into a failed unit, so +# "nothing was sent" is only half of the claim being made here. +set -uo pipefail + +TMP="" +trap 'rc=$?; [[ -n "$TMP" ]] && rm -rf "$TMP"; echo "HARNESS-RC=$rc"' EXIT + +# DIVE-2211: name the tree this harness grades. No `2>/dev/null` — the helper's +# stderr line IS the payload. +. "$(dirname "${BASH_SOURCE[0]}")/lib/grading_tree.sh" \ + || printf 'grading tree: UNRESOLVED (tests/lib/grading_tree.sh not reachable; no tree named)\n' >&2 +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SUT="${ROOT}/5dive-agent-stop-notify" +[[ -x "$SUT" ]] || { echo "FAIL: $SUT is missing or not executable"; exit 1; } + +TMP="$(mktemp -d)" +HOME_DIR="${TMP}/home" +mkdir -p "${HOME_DIR}/.codex/channels/telegram" +printf '{"allowFrom":["1234567890"],"groups":{},"pending":{}}\n' \ + > "${HOME_DIR}/.codex/channels/telegram/access.json" +printf '{"agents":{"tester":{"desiredState":"running"}}}\n' > "${TMP}/agents.json" + +SENT="${TMP}/sent.log" +cat > "${TMP}/send-stub" <<'STUB' +#!/usr/bin/env bash +printf '%s\t%s\n' "$1" "${2//$'\n'/ }" >> "$SENT" +STUB +chmod 755 "${TMP}/send-stub" + +fails=0 +pass() { echo "ok — $1"; } +fail() { echo "FAIL — $1"; fails=$((fails+1)); } + +# run