Persist Codex usage stats

SPEC.md

@@ -1309,6 +1309,14 @@ SHOULD return:
   - `output_tokens`
   - `total_tokens`
   - `seconds_running` (aggregate runtime seconds as of snapshot time, including active sessions)
+- `lifetime_totals` (optional durable totals across restarts)
+  - `input_tokens`
+  - `output_tokens`
+  - `total_tokens`
+  - `runtime_seconds`
+  - `sessions`
+  - `runs`
+- `recent_sessions` (optional durable list of recently completed Codex sessions)
 - `rate_limits` (latest coding-agent rate limit payload, if available)
 
 RECOMMENDED snapshot error modes:
@@ -1339,6 +1347,9 @@ Token accounting rules:
 - Do not treat generic `usage` maps as cumulative totals unless the event type defines them that
   way.
 - Accumulate aggregate totals in orchestrator state.
+- Implementations MAY persist run and session statistics to embedded local storage for durable
+  observability across process restarts.
+- Durable stores SHOULD tolerate missing/deleted databases by recreating empty history.
 
 Runtime accounting:
 
@@ -1452,6 +1463,29 @@ Minimum endpoints:
         "total_tokens": 7400,
         "seconds_running": 1834.2
       },
+      "lifetime_totals": {
+        "input_tokens": 12000,
+        "output_tokens": 4200,
+        "total_tokens": 16200,
+        "runtime_seconds": 4810,
+        "sessions": 8,
+        "runs": 3
+      },
+      "recent_sessions": [
+        {
+          "issue_id": "abc123",
+          "identifier": "MT-649",
+          "started_at": "2026-02-24T20:10:12Z",
+          "completed_at": "2026-02-24T20:14:59Z",
+          "turns": 2,
+          "input_tokens": 1200,
+          "output_tokens": 800,
+          "total_tokens": 2000,
+          "runtime_seconds": 287,
+          "final_state": "Done",
+          "model": "gpt-5.5"
+        }
+      ],
       "rate_limits": null
     }
     ```

elixir/README.md

@@ -115,6 +115,11 @@ Optional flags:
 - `--logs-root` tells Symphony to write logs under a different directory (default: `./log`)
 - `--port` also starts the Phoenix observability service (default: disabled)
 
+Symphony writes durable run and Codex session statistics to
+`~/.local/share/symphony/stats.db`. The SQLite database stores per-run token/runtime aggregates and
+the last completed Codex sessions. Deleting the database resets the dashboard history without
+affecting orchestration.
+
 The `WORKFLOW.md` file uses YAML front matter for configuration, plus a Markdown body used as the
 Codex session prompt.
 
@@ -221,6 +226,13 @@ The observability UI now runs on a minimal Phoenix stack:
 - Bandit as the HTTP server
 - Phoenix dependency static assets for the LiveView client bootstrap
 
+The dashboard and `/api/v1/state` include durable statistics from `stats.db`:
+
+- `lifetime_totals` reports input/output/total tokens, runtime seconds, sessions, and run count
+  across Symphony restarts.
+- `recent_sessions` lists the last 20 completed Codex sessions with issue, token, runtime, turn,
+  final state, and model fields.
+
 ## Project Layout
 
 - `lib/`: application code and Mix tasks

elixir/lib/symphony_elixir/orchestrator.ex

@@ -7,7 +7,7 @@ defmodule SymphonyElixir.Orchestrator do
   require Logger
   import Bitwise, only: [<<<: 2]
 
-  alias SymphonyElixir.{AgentRunner, Config, StatusDashboard, Tracker, Workspace}
+  alias SymphonyElixir.{AgentRunner, Config, Stats, StatusDashboard, Tracker, Workspace}
   alias SymphonyElixir.GitHub.Adapter, as: GitHubAdapter
   alias SymphonyElixir.Linear.Issue
 
@@ -34,13 +34,17 @@ defmodule SymphonyElixir.Orchestrator do
       :poll_check_in_progress,
       :tick_timer_ref,
       :tick_token,
+      :stats_run_id,
+      :stats_started_at,
       running: %{},
       completed: MapSet.new(),
       claimed: MapSet.new(),
       blocked: %{},
       retry_attempts: %{},
       codex_totals: nil,
-      codex_rate_limits: nil
+      codex_rate_limits: nil,
+      peak_concurrent_agents: 0,
+      sessions_launched: 0
     ]
   end
 
@@ -52,18 +56,24 @@ defmodule SymphonyElixir.Orchestrator do
 
   @impl true
   def init(_opts) do
+    Process.flag(:trap_exit, true)
     now_ms = System.monotonic_time(:millisecond)
+    now = DateTime.utc_now()
     config = Config.settings!()
 
     case validate_tracker_startup(config.tracker) do
       :ok ->
+        stats_run_id = Stats.start_run(%{started_at: now})
+
         state = %State{
           poll_interval_ms: config.polling.interval_ms,
           max_concurrent_agents: config.agent.max_concurrent_agents,
           next_poll_due_at_ms: now_ms,
           poll_check_in_progress: false,
           tick_timer_ref: nil,
           tick_token: nil,
+          stats_run_id: stats_run_id,
+          stats_started_at: now,
           codex_totals: @empty_codex_totals,
           codex_rate_limits: nil
         }
@@ -101,6 +111,8 @@ defmodule SymphonyElixir.Orchestrator do
 
   def handle_info({:tick, _tick_token}, state), do: {:noreply, state}
 
+  def handle_info({:EXIT, _pid, reason}, state), do: {:stop, reason, state}
+
   def handle_info(:tick, state) do
     state = refresh_runtime_config(state)
 
@@ -181,6 +193,7 @@ defmodule SymphonyElixir.Orchestrator do
           state
           |> apply_codex_token_delta(token_delta)
           |> apply_codex_rate_limits(update)
+          |> persist_stats_run()
 
         notify_dashboard()
         {:noreply, %{state | running: Map.put(running, issue_id, updated_running_entry)}}
@@ -253,6 +266,21 @@ defmodule SymphonyElixir.Orchestrator do
     })
   end
 
+  @impl true
+  def terminate(reason, %State{} = state) do
+    now = DateTime.utc_now()
+
+    state =
+      Enum.reduce(state.running, state, fn {_issue_id, running_entry}, state_acc ->
+        record_session_completion_totals(state_acc, running_entry, now, "stopped")
+      end)
+
+    finish_stats_run(state, now, reason)
+    :ok
+  end
+
+  def terminate(_reason, _state), do: :ok
+
   defp maybe_dispatch(%State{} = state) do
     state =
       state
@@ -979,16 +1007,21 @@ defmodule SymphonyElixir.Orchestrator do
             codex_last_reported_total_tokens: 0,
             turn_count: 0,
             retry_attempt: normalize_retry_attempt(attempt),
+            model: Stats.model_from_command(Config.settings!().codex.command),
             started_at: DateTime.utc_now()
           })
 
-        %{
+        updated_state = %{
           state
           | running: running,
             claimed: MapSet.put(state.claimed, issue.id),
-            retry_attempts: Map.delete(state.retry_attempts, issue.id)
+            retry_attempts: Map.delete(state.retry_attempts, issue.id),
+            peak_concurrent_agents: max(state.peak_concurrent_agents, map_size(running)),
+            sessions_launched: state.sessions_launched + 1
         }
 
+        persist_stats_run(updated_state)
+
       {:error, reason} ->
         Logger.error("Unable to spawn agent for #{issue_context(issue)}: #{inspect(reason)}")
         next_attempt = if is_integer(attempt), do: attempt + 1, else: nil
@@ -1627,7 +1660,22 @@ defmodule SymphonyElixir.Orchestrator do
   end
 
   defp record_session_completion_totals(state, running_entry) when is_map(running_entry) do
-    runtime_seconds = running_seconds(running_entry.started_at, DateTime.utc_now())
+    record_session_completion_totals(state, running_entry, DateTime.utc_now(), nil)
+  end
+
+  defp record_session_completion_totals(state, _running_entry), do: state
+
+  defp record_session_completion_totals(state, running_entry, completed_at, final_state_override)
+       when is_map(running_entry) do
+    runtime_seconds = running_seconds(running_entry.started_at, completed_at)
+
+    record_stats_session(
+      state,
+      running_entry,
+      completed_at,
+      runtime_seconds,
+      final_state_override
+    )
 
     codex_totals =
       apply_token_delta(
@@ -1641,9 +1689,83 @@ defmodule SymphonyElixir.Orchestrator do
       )
 
     %{state | codex_totals: codex_totals}
+    |> persist_stats_run()
+  end
+
+  defp record_stats_session(state, running_entry, completed_at, runtime_seconds, final_state_override) do
+    if stats_session_recordable?(running_entry) do
+      Stats.record_session(%{
+        run_id: state.stats_run_id,
+        issue_id: running_entry_issue_id(running_entry),
+        identifier: Map.get(running_entry, :identifier),
+        started_at: Map.get(running_entry, :started_at),
+        completed_at: completed_at,
+        turns: Map.get(running_entry, :turn_count, 0),
+        input_tokens: Map.get(running_entry, :codex_input_tokens, 0),
+        output_tokens: Map.get(running_entry, :codex_output_tokens, 0),
+        total_tokens: Map.get(running_entry, :codex_total_tokens, 0),
+        runtime_seconds: runtime_seconds,
+        final_state: final_state_override || running_entry_final_state(running_entry),
+        model: Map.get(running_entry, :model)
+      })
+    end
   end
 
-  defp record_session_completion_totals(state, _running_entry), do: state
+  defp stats_session_recordable?(running_entry) do
+    match?(%DateTime{}, Map.get(running_entry, :started_at)) and
+      (is_binary(Map.get(running_entry, :session_id)) or Map.get(running_entry, :turn_count, 0) > 0 or
+         Enum.any?(
+           [
+             Map.get(running_entry, :codex_input_tokens, 0),
+             Map.get(running_entry, :codex_output_tokens, 0),
+             Map.get(running_entry, :codex_total_tokens, 0)
+           ],
+           &(&1 > 0)
+         ))
+  end
+
+  defp running_entry_issue_id(%{issue: %Issue{id: issue_id}}) when is_binary(issue_id), do: issue_id
+  defp running_entry_issue_id(%{issue_id: issue_id}) when is_binary(issue_id), do: issue_id
+  defp running_entry_issue_id(_running_entry), do: nil
+
+  defp running_entry_final_state(%{issue: %Issue{state: state}}) when is_binary(state), do: state
+  defp running_entry_final_state(%{state: state}) when is_binary(state), do: state
+  defp running_entry_final_state(_running_entry), do: nil
+
+  defp persist_stats_run(%State{} = state) do
+    Stats.update_run(state.stats_run_id, stats_run_attrs(state))
+    state
+  end
+
+  defp finish_stats_run(%State{} = state, %DateTime{} = stopped_at, reason) do
+    Stats.finish_run(
+      state.stats_run_id,
+      stats_run_attrs(state, %{
+        stopped_at: stopped_at,
+        restart_reason: restart_reason(reason)
+      })
+    )
+  end
+
+  defp stats_run_attrs(%State{} = state, extra \\ %{}) do
+    codex_totals = state.codex_totals || @empty_codex_totals
+
+    %{
+      started_at: state.stats_started_at,
+      peak_concurrent_agents: state.peak_concurrent_agents,
+      sessions_launched: state.sessions_launched,
+      input_tokens: Map.get(codex_totals, :input_tokens, 0),
+      output_tokens: Map.get(codex_totals, :output_tokens, 0),
+      total_tokens: Map.get(codex_totals, :total_tokens, 0),
+      runtime_seconds: Map.get(codex_totals, :seconds_running, 0)
+    }
+    |> Map.merge(extra)
+  end
+
+  defp restart_reason(:normal), do: "normal"
+  defp restart_reason(:shutdown), do: "shutdown"
+  defp restart_reason({:shutdown, reason}), do: "shutdown: #{inspect(reason)}"
+  defp restart_reason(reason), do: inspect(reason)
 
   defp refresh_runtime_config(%State{} = state) do
     config = Config.settings!()