feat(agents): report a crash, an OOM and a permanent start failure from ExecStopPost (DIVE-3965)

5dive-agent-stop-notify

@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+# 5dive-agent-stop-notify <name> — ExecStopPost notifier for 5dive-agent@.service.
+#
+# DIVE-3965 (P06). The channel plugins can tell you a turn was interrupted, but
+# only from INSIDE a process that is still alive. The three states that matter
+# most — SIGKILL, OOM, and a permanent config failure that trips
+# RestartPreventExitStatus — kill the notifier along with everything else, so
+# the chat's last word is whatever the agent happened to be saying. systemd is
+# the only observer that outlives the unit, and ExecStopPost is where it speaks.
+#
+# WHY IT IS A SEPARATE SCRIPT AND NOT `5dive ...`:
+# ExecStopPost runs as User=agent-%i with no sudo, and it must survive a
+# half-installed bundle (a `5dive` that is mid-self-update is exactly when an
+# agent is most likely to be dying). This file therefore depends on bash, curl
+# and coreutils only, reads no state it cannot read as the agent, and ALWAYS
+# exits 0 — a notifier must never be the reason a stop is recorded as failed.
+# The unit also prefixes it with `-` for the same reason, belt and braces.
+#
+# WHAT systemd HANDS US (see systemd.service(5), "Environment variables in
+# spawned processes"):
+#   SERVICE_RESULT  success | protocol | timeout | exit-code | signal |
+#                   core-dump | watchdog | start-limit-hit | resources | oom-kill
+#   EXIT_CODE       exited | killed | dumped
+#   EXIT_STATUS     numeric exit status, or the signal name for killed/dumped
+set -uo pipefail
+
+NAME="${1:-}"
+[[ -n "$NAME" ]] || exit 0
+
+HOME_DIR="${STOP_NOTIFY_HOME:-/home/agent-${NAME}}"
+REGISTRY="${STOP_NOTIFY_REGISTRY:-/var/lib/5dive/agents.json}"
+STATE_DIR="${STOP_NOTIFY_STATE_DIR:-${HOME_DIR}/.5dive/stop-notify}"
+DEDUP_SECS="${STOP_NOTIFY_DEDUP_SECS:-900}"
+NOW="${STOP_NOTIFY_NOW:-$(date +%s)}"
+SEND_CMD="${STOP_NOTIFY_SEND_CMD:-}"
+
+RESULT="${SERVICE_RESULT:-unknown}"
+EXIT_KIND="${EXIT_CODE:-}"
+EXIT_VAL="${EXIT_STATUS:-}"
+
+# ── the decision, kept as one function so the harness can grade it directly ──
+#
+# Prints "<cause key>|<sentence>" or nothing at all when the stop is not worth a
+# message. The cause key is what dedup counts, so two different causes inside
+# one window BOTH get through: an OOM followed by a config failure is two
+# findings, not a repeat.
+classify_stop() {   # <result> <exit-kind> <exit-val> <desired-state>
+  local result="$1" kind="$2" val="$3" desired="$4"
+
+  # A deliberate stop is not an incident. Two independent readings say so, and
+  # either is enough: systemd's own verdict, and the operator intent DIVE-857
+  # records in the registry at `5dive agent stop`. The second is load-bearing
+  # because a launcher that dies on SIGTERM reports `signal/SIGTERM`, which is
+  # byte-identical to a kill — without the intent flag every clean `agent stop`
+  # would page the chat.
+  [[ "$desired" == "stopped" ]] && return 0
+  [[ "$result" == "success" ]] && return 0
+  if [[ "$kind" == "killed" && ( "$val" == "TERM" || "$val" == "SIGTERM" || "$val" == "15" ) ]]; then
+    return 0
+  fi
+
+  case "$result" in
+    oom-kill)
+      printf 'oom|ran out of memory and was killed by the kernel. It will not come back on its own if the box is still under pressure.\n' ;;
+    watchdog)
+      printf 'watchdog|stopped responding to its own watchdog and was killed.\n' ;;
+    start-limit-hit)
+      printf 'crash-loop|failed to start too many times in a row, so systemd gave up. It stays down until someone clears it.\n' ;;
+    timeout)
+      printf 'timeout|did not shut down in time and was killed.\n' ;;
+    resources)
+      printf 'resources|could not be started — the box could not give it the resources it asked for.\n' ;;
+    core-dump)
+      printf 'core-dump|crashed hard (signal %s) and dumped core.\n' "${val:-unknown}" ;;
+    signal)
+      printf 'signal-%s|was killed by signal %s.\n' "${val:-unknown}" "${val:-unknown}" ;;
+    exit-code)
+      # 2 and 3 are 5dive-agent-start's permanent conditions and the unit's
+      # RestartPreventExitStatus: no retry is coming, which is the part a
+      # person needs to be told.
+      case "$val" in
+        2) printf 'permanent-2|could not start: its agent type is not one this box knows how to run. It will NOT be retried.\n' ;;
+        3) printf 'permanent-3|could not start: its CLI or plugin is not installed. It will NOT be retried.\n' ;;
+        *) printf 'exit-%s|exited with error status %s.\n' "${val:-unknown}" "${val:-unknown}" ;;
+      esac ;;
+    protocol|*)
+      printf 'other-%s|stopped unexpectedly (systemd: %s).\n' "$result" "$result" ;;
+  esac
+}
+
+desired_state() {
+  [[ -r "$REGISTRY" ]] || return 0
+  command -v jq >/dev/null 2>&1 || return 0
+  jq -r --arg n "$NAME" '.agents[$n].desiredState // ""' "$REGISTRY" 2>/dev/null
+}
+
+verdict="$(classify_stop "$RESULT" "$EXIT_KIND" "$EXIT_VAL" "$(desired_state)")"
+[[ -n "$verdict" ]] || exit 0
+cause="${verdict%%|*}"
+sentence="${verdict#*|}"
+
+# ── dedup: a crash-loop is one finding, not forty ───────────────────────────
+#
+# Keyed on the CAUSE, so a different failure inside the window still gets
+# through. The suppressed repeats are counted rather than dropped: the next
+# message that does go out says how many there were, which is the difference
+# between "it crashed" and "it has crashed 38 times".
+mkdir -p "$STATE_DIR" 2>/dev/null || true
+mark="${STATE_DIR}/${NAME}.last"
+prev_cause="" prev_at=0 prev_count=0
+if [[ -r "$mark" ]]; then read -r prev_cause prev_at prev_count < "$mark" 2>/dev/null || true; fi
+[[ "$prev_at" =~ ^[0-9]+$ ]] || prev_at=0
+[[ "$prev_count" =~ ^[0-9]+$ ]] || prev_count=0
+
+repeats=0
+if [[ "$prev_cause" == "$cause" ]] && (( NOW - prev_at < DEDUP_SECS )); then
+  # Inside the window: count it and say nothing.
+  printf '%s %s %s\n' "$cause" "$prev_at" "$(( prev_count + 1 ))" > "$mark" 2>/dev/null || true
+  exit 0
+fi
+[[ "$prev_cause" == "$cause" ]] && repeats="$prev_count"
+printf '%s %s 0\n' "$cause" "$NOW" > "$mark" 2>/dev/null || true
+
+text="⚠️ Agent ${NAME} ${sentence}"
+if (( repeats > 0 )); then
+  text+=$'\n'"(${repeats} further occurrence(s) were suppressed in the last $(( DEDUP_SECS / 60 )) minutes.)"
+fi
+
+# ── delivery ────────────────────────────────────────────────────────────────
+#
+# The unit's own EnvironmentFile already put TELEGRAM_BOT_TOKEN in our
+# environment, and ExecStopPost inherits it. The paired chats are the
+# allowFrom list of whichever runtime's channel state this seat has.
+token="${TELEGRAM_BOT_TOKEN:-}"
+[[ -n "$token" ]] || exit 0
+
+access=""
+for d in .codex .claude .grok .gemini; do
+  cand="${HOME_DIR}/${d}/channels/telegram/access.json"
+  [[ -r "$cand" ]] && { access="$cand"; break; }
+done
+[[ -n "$access" ]] || exit 0
+command -v jq >/dev/null 2>&1 || exit 0
+
+mapfile -t chats < <(jq -r '.allowFrom // [] | .[]' "$access" 2>/dev/null | head -5)
+for chat in "${chats[@]:-}"; do
+  [[ -n "$chat" ]] || continue
+  if [[ -n "$SEND_CMD" ]]; then
+    "$SEND_CMD" "$chat" "$text" || true
+  else
+    curl -fsS -m 10 -X POST "https://api.telegram.org/bot${token}/sendMessage" \
+      --data-urlencode "chat_id=${chat}" \
+      --data-urlencode "text=${text}" >/dev/null 2>&1 || true
+  fi
+done
+exit 0

changelog.d/DIVE-3965.md

@@ -0,0 +1,19 @@
+## Unreleased — feat(agents): a crashed agent now says why it died, once, in its own chat (DIVE-3965)
+
+- A Codex (or any) channel agent that was SIGKILLed, OOM-killed, or stopped by a permanent
+  start failure could not report it: every process that could have sent the message died
+  with it, so the chat's last word was whatever the agent happened to be saying. systemd is
+  the only observer that outlives the unit, so the notification moves to `ExecStopPost`.
+- `5dive-agent-stop-notify` classifies the stop from systemd's own verdict — out of memory,
+  killed by signal, start-limit crash-loop, stop timeout, and the two permanent conditions
+  (`RestartPreventExitStatus=2 3`) that will never be retried — and says which one happened
+  in plain words.
+- A deliberate stop is silent: both systemd's `success` result and the operator intent
+  recorded in the registry at `5dive agent stop` are honoured, so a restart does not page
+  anyone.
+- Repeats of the SAME cause inside 15 minutes are suppressed and counted, and the next
+  message that does go out says how many were held back. A DIFFERENT cause in that window
+  is always sent, so a crash-loop can never hide an OOM.
+- The notifier depends on bash, curl and jq only, and always exits 0 — an agent is most
+  likely to be dying exactly when the `5dive` bundle is mid-update, and an `ExecStopPost`
+  must never turn a clean stop into a failed unit.

docker/Dockerfile

@@ -53,7 +53,7 @@ RUN systemctl mask \
 # `curl: (37) Couldn't open file` — this list and install.sh's fetches are two
 # copies of one set, and only the docker-install job notices when they disagree.
 # (DIVE-3269 added 5dive-stage-fork-plugins.sh and this line went red first.)
-COPY 5dive 5dive-agent-list-snapshot 5dive-agent-start 5dive-refresh-plugins.sh 5dive-stage-fork-plugins.sh 5dive-refresh-skills.sh install.sh projects-CLAUDE.md telegram-agent-CLAUDE.md model-tiering-CLAUDE.md operational-comms-CLAUDE.md /opt/5dive-bundle/
+COPY 5dive 5dive-agent-list-snapshot 5dive-agent-start 5dive-agent-stop-notify 5dive-refresh-plugins.sh 5dive-stage-fork-plugins.sh 5dive-refresh-skills.sh install.sh projects-CLAUDE.md telegram-agent-CLAUDE.md model-tiering-CLAUDE.md operational-comms-CLAUDE.md /opt/5dive-bundle/
 COPY hooks /opt/5dive-bundle/hooks/
 COPY skills /opt/5dive-bundle/skills/
 COPY systemd /opt/5dive-bundle/systemd/

install.sh

@@ -962,6 +962,34 @@ JOURNALD
   chmod 755 "$BIN_DIR/5dive-agent-start"
   ok "5dive-agent-start → $BIN_DIR/5dive-agent-start"
 
+  # DIVE-3965: the unit's ExecStopPost notifier. Installed next to the launcher
+  # because it shares the launcher's constraint — it has to work when the
+  # bundle itself is the thing that is broken.
+  #
+  # OPTIONAL AT THE PIN, and that is the whole reason this is not the bare
+  # `curl -fsSL` the launcher above uses. This file is ADDITIVE and the fleet
+  # pin predates the tag that ships it, so a fail-closed fetch here 404s on
+  # every fresh install and on every box's 04:00Z self-update and aborts the
+  # WHOLE install under `set -e` (DIVE-4349; caught by
+  # scripts/install-pin-compat.sh before it reached a box). A box that does not
+  # get the notifier simply has no ExecStopPost target — which the unit's `-`
+  # prefix already tolerates: the notification is missing, the unit still stops.
+  #
+  # MOVE IT UP into the fail-closed set only once the fleet pin is at or past
+  # the tag that ships it; install-pin-compat grades that on every PR touching
+  # this file, so "the pin has caught up" stays a check result, not a memory.
+  if fetch_optional_at_pin "5dive-agent-stop-notify" "$BIN_DIR/5dive-agent-stop-notify"; then
+    chmod 755 "$BIN_DIR/5dive-agent-stop-notify"
+    ok "5dive-agent-stop-notify → $BIN_DIR/5dive-agent-stop-notify"
+  else
+    # Symmetric with the optional-hook loop below: the bundle at this pin does
+    # not wire it, so a copy left from a newer install would be a file nothing
+    # invokes (the unit template comes from the same pin and carries no
+    # ExecStopPost line there).
+    rm -f "$BIN_DIR/5dive-agent-stop-notify"
+    say "5dive-agent-stop-notify is not shipped at this pin — agent crash notifications stay off on this box until the fleet pin passes the tag that carries it"
+  fi
+
   # >>> DIVE-4194 box-side scripts
   # The helper scripts below used to be three hand-written curl/chmod/ok blocks
   # right here, and this file was their ONLY writer. The control plane's nightly
@@ -1574,7 +1602,7 @@ if [[ "${1:-}" == "--uninstall" ]]; then
   systemctl daemon-reload || true
 
   # 3. Binaries + shared libs
-  rm -f "$BIN_DIR/5dive" "$BIN_DIR/5dive-agent-start"
+  rm -f "$BIN_DIR/5dive" "$BIN_DIR/5dive-agent-start" "$BIN_DIR/5dive-agent-stop-notify"
   ok "removed CLI binaries"
   if [[ -d "$LIB_DIR" ]]; then
     rm -rf "$LIB_DIR"

systemd/5dive-agent@.service

@@ -43,6 +43,11 @@ ExecStart=/usr/local/bin/5dive-agent-start %i
 # when agy exits and then also gets the graceful cgroup SIGTERM, so its poll is
 # still aborted cleanly. Stop drops from 15s to ~0.05s for agy.
 ExecStop=/bin/bash -c '/usr/bin/tmux kill-session -t agent-%i 2>/dev/null; /usr/bin/pkill -HUP -u agent-%i -x agy 2>/dev/null; true'
+# DIVE-3965: the only observer that outlives a SIGKILL, an OOM, or a permanent
+# start failure. `-` so a notifier fault can never turn a clean stop into a
+# failed one; the script exits 0 unconditionally for the same reason, and stays
+# silent for a deliberate `5dive agent stop` and for a clean exit.
+ExecStopPost=-/usr/local/bin/5dive-agent-stop-notify %i
 Restart=on-failure
 RestartSec=3
 # A missing binary or an unknown agent type is a PERMANENT condition — no amount

tests/agent_stop_notify_unit.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+# DIVE-3965 — `5dive-agent-stop-notify` is the ExecStopPost half of Codex channel
+# crash parity: the notification that has to survive the death of every process
+# that could otherwise have sent it.
+#
+# Graded by RUNNING the shipped script with the environment systemd actually
+# hands an ExecStopPost (SERVICE_RESULT/EXIT_CODE/EXIT_STATUS), against a stub
+# sender that records its argv. Not by sourcing the classifier: the thing that
+# breaks in this file is the wiring between the classifier, the dedup mark and
+# the allowFrom read, and a harness that calls one function reaches none of it.
+#
+# Every negative arm pins the ABSENCE of a send AND a zero exit — an
+# ExecStopPost that exits non-zero turns a clean stop into a failed unit, so
+# "nothing was sent" is only half of the claim being made here.
+set -uo pipefail
+
+TMP=""
+trap 'rc=$?; [[ -n "$TMP" ]] && rm -rf "$TMP"; echo "HARNESS-RC=$rc"' EXIT
+
+# DIVE-2211: name the tree this harness grades. No `2>/dev/null` — the helper's
+# stderr line IS the payload.
+. "$(dirname "${BASH_SOURCE[0]}")/lib/grading_tree.sh" \
+  || printf 'grading tree: UNRESOLVED (tests/lib/grading_tree.sh not reachable; no tree named)\n' >&2
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+SUT="${ROOT}/5dive-agent-stop-notify"
+[[ -x "$SUT" ]] || { echo "FAIL: $SUT is missing or not executable"; exit 1; }
+
+TMP="$(mktemp -d)"
+HOME_DIR="${TMP}/home"
+mkdir -p "${HOME_DIR}/.codex/channels/telegram"
+printf '{"allowFrom":["1234567890"],"groups":{},"pending":{}}\n' \
+  > "${HOME_DIR}/.codex/channels/telegram/access.json"
+printf '{"agents":{"tester":{"desiredState":"running"}}}\n' > "${TMP}/agents.json"
+
+SENT="${TMP}/sent.log"
+cat > "${TMP}/send-stub" <<'STUB'
+#!/usr/bin/env bash
+printf '%s\t%s\n' "$1" "${2//$'\n'/ }" >> "$SENT"
+STUB
+chmod 755 "${TMP}/send-stub"
+
+fails=0
+pass() { echo "ok   — $1"; }
+fail() { echo "FAIL — $1"; fails=$((fails+1)); }
+
+# run <label> <result> <exit-kind> <exit-val> [extra env assignments...]
+run() {
+  local label="$1" result="$2" kind="$3" val="$4"; shift 4
+  : > "$SENT"
+  env -u TELEGRAM_STATE_DIR SERVICE_RESULT="$result" EXIT_CODE="$kind" EXIT_STATUS="$val" \
+      TELEGRAM_BOT_TOKEN="stub-token" \
+      STOP_NOTIFY_HOME="$HOME_DIR" \
+      STOP_NOTIFY_REGISTRY="${TMP}/agents.json" \
+      STOP_NOTIFY_STATE_DIR="${TMP}/state" \
+      STOP_NOTIFY_SEND_CMD="${TMP}/send-stub" \
+      SENT="$SENT" \
+      "$@" \
+      "$SUT" tester
+  RC=$?
+  OUT="$(cat "$SENT" 2>/dev/null)"
+  [[ $RC -eq 0 ]] || fail "$label: exited $RC — an ExecStopPost must always exit 0"
+}
+
+silent() { [[ -z "$OUT" ]] && pass "$1" || fail "$1: sent '$OUT'"; }
+sent_matching() {
+  if grep -qE "$2" <<<"$OUT"; then pass "$1"; else fail "$1: got '$OUT', wanted /$2/"; fi
+  grep -q '^1234567890	' <<<"$OUT" || fail "$1: did not address the allowFrom chat"
+}
+
+# ── 1. the quiet cases ──────────────────────────────────────────────────────
+run "clean exit" success exited 0
+silent "a clean exit says nothing"
+
+run "operator stop" signal killed TERM
+silent "a SIGTERM (the ordinary restart/stop path) says nothing"
+
+rm -rf "${TMP}/state"
+printf '{"agents":{"tester":{"desiredState":"stopped"}}}\n' > "${TMP}/agents.json"
+run "deliberate stop" exit-code exited 1
+silent "a recorded operator stop says nothing even when the exit is non-zero"
+printf '{"agents":{"tester":{"desiredState":"running"}}}\n' > "${TMP}/agents.json"
+
+# ── 2. cause awareness ──────────────────────────────────────────────────────
+rm -rf "${TMP}/state"
+run "oom" oom-kill killed KILL
+sent_matching "an OOM kill is reported as running out of memory" 'ran out of memory'
+
+rm -rf "${TMP}/state"
+run "permanent 3" exit-code exited 3
+sent_matching "a not-installed CLI says it will NOT be retried" 'NOT be retried'
+grep -q 'plugin is not installed' <<<"$OUT" || fail "exit 3 must name the missing install, not just the number"
+
+rm -rf "${TMP}/state"
+run "crash loop" start-limit-hit exited 1
+sent_matching "a start-limit trip says systemd gave up" 'gave up'
+
+rm -rf "${TMP}/state"
+run "sigkill" signal killed KILL
+sent_matching "a SIGKILL is reported as a kill by signal" 'killed by signal KILL'
+
+# The causes above must not be the same sentence — the whole point of the row.
+rm -rf "${TMP}/state"
+run "oom text" oom-kill killed KILL; oom_text="$OUT"
+rm -rf "${TMP}/state"
+run "exit text" exit-code exited 9; exit_text="$OUT"
+[[ "$oom_text" != "$exit_text" ]] \
+  && pass "two different causes produce two different sentences" \
+  || fail "OOM and a bare non-zero exit produced identical text"
+
+# ── 3. dedup, and what it must NOT swallow ──────────────────────────────────
+rm -rf "${TMP}/state"
+run "first crash" exit-code exited 9
+[[ -n "$OUT" ]] && pass "the first crash of a window is sent" || fail "the first crash was suppressed"
+
+run "repeat crash" exit-code exited 9
+silent "an identical cause inside the window is suppressed"
+
+run "different cause" oom-kill killed KILL
+[[ -n "$OUT" ]] \
+  && pass "a DIFFERENT cause inside the same window is still sent" \
+  || fail "dedup swallowed a different failure — a crash-loop would hide an OOM"
+
+# A window that has elapsed re-opens, and the message carries the count of what
+# was suppressed: "it crashed" and "it has crashed 3 times" are different facts.
+rm -rf "${TMP}/state"
+run "t0" exit-code exited 9
+run "t1" exit-code exited 9
+run "t2" exit-code exited 9
+run "after window" exit-code exited 9 STOP_NOTIFY_DEDUP_SECS=0
+if grep -q 'further occurrence' <<<"$OUT"; then
+  pass "the next message after a suppressed run reports how many were suppressed"
+else
+  fail "suppressed repeats were dropped without a count: '$OUT'"
+fi
+
+# ── 4. it must not send where it has no addressee or no token ───────────────
+rm -rf "${TMP}/state"
+: > "$SENT"
+env -u TELEGRAM_BOT_TOKEN SERVICE_RESULT=oom-kill EXIT_CODE=killed EXIT_STATUS=KILL \
+    STOP_NOTIFY_HOME="$HOME_DIR" STOP_NOTIFY_REGISTRY="${TMP}/agents.json" \
+    STOP_NOTIFY_STATE_DIR="${TMP}/state2" STOP_NOTIFY_SEND_CMD="${TMP}/send-stub" \
+    SENT="$SENT" "$SUT" tester
+rc=$?
+[[ $rc -eq 0 && ! -s "$SENT" ]] \
+  && pass "no bot token: silent, and still exit 0" \
+  || fail "no-token path sent '$(cat "$SENT")' rc=$rc"
+
+rm -rf "${TMP}/state"
+run "no access file" oom-kill killed KILL STOP_NOTIFY_HOME="${TMP}/empty-home"
+silent "an agent with no paired channel state is silent"
+
+env SERVICE_RESULT=oom-kill "$SUT" >/dev/null 2>&1
+[[ $? -eq 0 ]] && pass "a missing agent name exits 0 instead of erroring" \
+               || fail "a missing agent name did not exit 0"
+
+echo "--- $fails failure(s)"
+[[ $fails -eq 0 ]]