state.py

"""Shared state: fabric I/O, retriever, training helpers, model registry."""

import json
import logging
import os
import re
import secrets
import shutil
import subprocess
import tempfile
import urllib.request
import urllib.error
from datetime import datetime, timezone
from pathlib import Path

logger = logging.getLogger(__name__)

FABRIC_DIR = Path(os.environ.get("FABRIC_DIR", Path.home() / "fabric"))
HERMES_HOME = Path(os.environ.get("HERMES_HOME", "")) if os.environ.get("HERMES_HOME") else None
AGENT_NAME = os.environ.get("HERMES_AGENT_NAME", "")
PLUGIN_DIR = Path(__file__).parent

if not AGENT_NAME and HERMES_HOME and ".hermes-" in str(HERMES_HOME):
    AGENT_NAME = str(HERMES_HOME).split(".hermes-")[-1].rstrip("/")

# ── Shared regexes (used by hooks.py and scoring) ────────
DECISION_RE = re.compile(
    r"(?i)\b(decided|resolved|completed|fixed|deployed|shipped|reviewed|approved|rejected)\b"
)
OUTCOME_RE = re.compile(
    r"(?i)(result:|outcome:|conclusion:|because|root cause|instead of|\d+%|\d+x)"
)
COMPLETION_RE = re.compile(
    r"(?i)\b(completed|finished|done|shipped|deployed|resolved|closed|merged|fixed)\b"
)

# ── Session state ────────────────────────────────────────
session_id = ""
exchanges: list = []

# ── Training job tracking ────────────────────────────────
_JOB_FILE = (HERMES_HOME or Path.home()) / ".icarus-training-job.txt"


def _last_job_id():
    if _JOB_FILE.exists():
        return _JOB_FILE.read_text("utf-8").strip()
    return ""


def _save_job_id(jid):
    _JOB_FILE.write_text(jid, "utf-8")


# ── Model registry ───────────────────────────────────────
_REGISTRY_FILE = (HERMES_HOME or Path.home()) / ".icarus-models.json"


def _load_registry():
    if _REGISTRY_FILE.exists():
        try:
            return json.loads(_REGISTRY_FILE.read_text("utf-8"))
        except Exception:
            pass
    return {"models": [], "active_model": None}


def _save_registry(registry):
    try:
        _REGISTRY_FILE.write_text(json.dumps(registry, indent=2), "utf-8")
    except Exception as exc:
        logger.warning("icarus: failed to save model registry: %s", exc)


def list_models():
    return _load_registry()


# ── Retrieval telemetry ──────────────────────────────────
_TELEMETRY_FILE = (HERMES_HOME or Path.home()) / ".icarus-telemetry.jsonl"

# in-memory buffer for current session
_recall_log: list = []


def _summarize_telemetry_events(events):
    """Compute telemetry summary for a filtered event list."""
    recalls = [e for e in events if e.get("event") == "recall"]
    usages = [e for e in events if e.get("event") == "usage"]
    recalled_ids = set()
    for r in recalls:
        recalled_ids.update(r.get("result_ids", []))
    used_ids = set(u.get("entry_id", "") for u in usages)
    used_ids.discard("")
    recalled_ids.discard("")
    used_from_recall = used_ids & recalled_ids
    return {
        "total_recalls": len(recalls),
        "total_usages": sum(1 for u in usages if u.get("entry_id", "") in recalled_ids),
        "unique_entries_recalled": len(recalled_ids),
        "unique_entries_used": len(used_from_recall),
        "usage_rate": round(len(used_from_recall) / max(len(recalled_ids), 1), 2),
    }


def log_recall(query, results, source="pre_llm_call"):
    """Log what was recalled and injected."""
    entry = {
        "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "event": "recall",
        "source": source,
        "query": query[:100],
        "result_count": len(results),
        "result_ids": [r.get("id", "") for r in results[:5] if isinstance(r, dict)],
        "result_summaries": [r.get("summary", "")[:60] for r in results[:5] if isinstance(r, dict)],
        "session_id": session_id,
        "agent": AGENT_NAME,
    }
    _recall_log.append(entry)
    try:
        with open(_TELEMETRY_FILE, "a", encoding="utf-8") as f:
            f.write(json.dumps(entry) + "\n")
    except Exception:
        pass


def was_recalled(entry_id):
    """Return True if this entry_id was recalled in the current session."""
    if not entry_id:
        return False
    current_session = session_id
    for event in reversed(_recall_log):
        if event.get("event") != "recall":
            continue
        if current_session and event.get("session_id") != current_session:
            continue
        if entry_id in event.get("result_ids", []):
            return True
    return False


def log_usage(entry_id, action="referenced"):
    """Log when a recalled entry is actually used (referenced in a write)."""
    entry = {
        "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "event": "usage",
        "action": action,
        "entry_id": entry_id,
        "session_id": session_id,
        "agent": AGENT_NAME,
    }
    _recall_log.append(entry)
    try:
        with open(_TELEMETRY_FILE, "a", encoding="utf-8") as f:
            f.write(json.dumps(entry) + "\n")
    except Exception:
        pass


def get_telemetry(last_n=50, session_id_filter=None, agent_filter=None):
    """Read recent telemetry entries."""
    empty_summary = {
        "total_recalls": 0,
        "total_usages": 0,
        "unique_entries_recalled": 0,
        "unique_entries_used": 0,
        "usage_rate": 0.0,
    }
    if not _TELEMETRY_FILE.exists():
        return {"events": [], "summary": empty_summary}
    lines = _TELEMETRY_FILE.read_text("utf-8").strip().split("\n")
    events = []
    for line in lines[-last_n:]:
        try:
            events.append(json.loads(line))
        except Exception:
            pass

    if session_id_filter:
        events = [e for e in events if e.get("session_id") == session_id_filter]
    if agent_filter:
        events = [e for e in events if e.get("agent") == agent_filter]

    return {
        "events": events,
        "summary": _summarize_telemetry_events(events),
    }


def build_brief():
    """Build the daily brief: pending work, recent own work, changes, suggested action."""
    agent = AGENT_NAME or "agent"
    brief = {}

    # pending
    open_tasks, reviews, open_tickets = read_pending()
    brief["pending"] = {
        "open_tasks": len(open_tasks),
        "reviews_of_my_work": len(reviews),
        "open_tickets": len(open_tickets),
        "items": [],
    }
    for t in open_tasks[:3]:
        brief["pending"]["items"].append({
            "from": t.get("agent", "?"),
            "summary": t.get("summary", "?"),
            "type": t.get("type", "?"),
            "id": t.get("id", "?"),
        })
    for r in reviews[:3]:
        brief["pending"]["items"].append({
            "from": r.get("agent", "?"),
            "summary": r.get("summary", "?"),
            "type": "review",
            "id": r.get("id", "?"),
            "review_of": r.get("review_of", ""),
        })
    for t in open_tickets[:3]:
        brief["pending"]["items"].append({
            "from": t.get("agent", "?"),
            "summary": t.get("summary", "?"),
            "type": t.get("type", "ticket"),
            "id": t.get("id", "?"),
            "customer_id": t.get("customer_id", "?"),
        })

    # recent own work (last 5 entries by this agent)
    own = read_recent(agent=agent, limit=5)
    brief["recent_work"] = [
        {"summary": e.get("summary", "?"), "timestamp": str(e.get("timestamp", ""))[:16]}
        for e in own
    ]

    # recent activity from others (changes since last session)
    others = read_cross_agent(limit=5)
    brief["from_others"] = others

    # suggested action
    if open_tasks:
        t = open_tasks[0]
        brief["suggested_action"] = f"Pick up: {t.get('summary', '?')} from {t.get('agent', '?')} (id {t.get('id', '?')})"
    elif reviews:
        r = reviews[0]
        brief["suggested_action"] = f"Address review: {r.get('summary', '?')} from {r.get('agent', '?')} ({r.get('review_of', '')})"
    elif open_tickets:
        t = open_tickets[0]
        brief["suggested_action"] = f"Resolve ticket: {t.get('summary', '?')} [{t.get('customer_id', '?')}]"
    else:
        brief["suggested_action"] = "No pending work. Continue current task or check fabric_recall for context."

    # telemetry summary (if available)
    tel = get_telemetry(last_n=20)
    if tel.get("summary", {}).get("total_recalls", 0) > 0:
        brief["recall_stats"] = tel["summary"]

    return brief


# ── Creative state ───────────────────────────────────────
_STATE_FILE = (HERMES_HOME or Path.home()) / ".icarus-state.json"


def load_creative():
    if _STATE_FILE.exists():
        try:
            return json.loads(_STATE_FILE.read_text("utf-8"))
        except Exception:
            pass
    return {"cycle": 0, "themes": [], "questions": [], "learnings": []}


def save_creative(s):
    try:
        _STATE_FILE.write_text(json.dumps(s, indent=2), "utf-8")
    except Exception as exc:
        logger.warning("icarus: save state failed: %s", exc)


# ── Fabric I/O ───────────────────────────────────────────

def _yaml_scalar(value):
    """Encode a scalar as a YAML-safe quoted string."""
    return json.dumps(str(value))


def _parse_frontmatter_scalar(text, key):
    """Read a scalar frontmatter value and normalize quoted YAML strings."""
    m = re.search(rf"^{key}: (.+)$", text, re.MULTILINE)
    if not m:
        return ""
    raw = m.group(1).strip()
    try:
        import yaml as _yaml
        value = _yaml.safe_load(raw)
    except Exception:
        value = raw
    if value is None:
        return ""
    if isinstance(value, list):
        return ", ".join(str(v) for v in value)
    return str(value)

def write_entry(entry_type, content, summary, tier="hot", tags="", platform="cli",
                status="", outcome="", review_of="", revises="", customer_id="",
                assigned_to="", training_value="", verified="", evidence="",
                source_tool="", artifact_paths=""):
    """Write a fabric entry with full schema v1 fields. Returns the filepath."""
    FABRIC_DIR.mkdir(parents=True, exist_ok=True)
    now = datetime.now(timezone.utc)
    ts = now.strftime("%Y-%m-%dT%H%MZ")
    ts_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ")
    agent = AGENT_NAME or "agent"
    suffix = secrets.token_hex(2)
    # derive a short slug from the summary for human-readable filenames
    slug = re.sub(r"[^a-z0-9]+", "-", summary.lower().strip())[:40].strip("-")
    if slug:
        filename = f"{agent}-{entry_type}-{slug}-{suffix}.md"
    else:
        filename = f"{agent}-{entry_type}-{ts}-{suffix}.md"

    sid = session_id or os.environ.get(
        "FABRIC_SESSION_ID", f"sess-{now.strftime('%Y%m%d-%H%M%S')}-{os.getpid()}")
    project_id = os.environ.get(
        "FABRIC_PROJECT_ID",
        Path.cwd().name if Path.cwd() != Path.home() else "unknown")

    lines = [
        "---",
        f"id: {_yaml_scalar(secrets.token_hex(4))}",
        f"agent: {_yaml_scalar(agent)}",
        f"platform: {_yaml_scalar(platform)}",
        f"timestamp: {_yaml_scalar(ts_iso)}",
        f"type: {_yaml_scalar(entry_type)}",
        f"tier: {_yaml_scalar(tier)}",
        f"summary: {_yaml_scalar(summary)}",
        f"project_id: {_yaml_scalar(project_id)}",
        f"session_id: {_yaml_scalar(sid)}",
    ]
    if tags:
        lines.append(f"tags: [{tags}]")
    if status:
        lines.append(f"status: {_yaml_scalar(status)}")
    if outcome:
        lines.append(f"outcome: {_yaml_scalar(outcome)}")
    if review_of:
        lines.append(f"review_of: {_yaml_scalar(review_of)}")
    if revises:
        lines.append(f"revises: {_yaml_scalar(revises)}")
    if customer_id:
        lines.append(f"customer_id: {_yaml_scalar(customer_id)}")
    if assigned_to:
        lines.append(f"assigned_to: {_yaml_scalar(assigned_to)}")
    if training_value:
        lines.append(f"training_value: {_yaml_scalar(training_value)}")
    if verified:
        lines.append(f"verified: {_yaml_scalar(verified)}")
    if evidence:
        lines.append(f"evidence: {_yaml_scalar(evidence)}")
    if source_tool:
        lines.append(f"source_tool: {_yaml_scalar(source_tool)}")
    if artifact_paths:
        lines.append(f"artifact_paths: [{artifact_paths}]")
    lines.extend(["---", "", content])

    path = FABRIC_DIR / filename
    path.write_text("\n".join(lines), "utf-8")
    logger.info("icarus: wrote %s", filename)

    # opt-in obsidian formatting
    if os.environ.get("ICARUS_OBSIDIAN"):
        try:
            from . import obsidian
            obsidian.format_entry(path, FABRIC_DIR, review_of=review_of, revises=revises)
            obsidian.ensure_daily_note(FABRIC_DIR, filename, summary)
        except Exception as exc:
            logger.debug("icarus: obsidian formatting failed: %s", exc)

    return str(path)


def read_recent(agent="", limit=5):
    """Read recent hot entries."""
    if not FABRIC_DIR.exists():
        return []
    out = []
    for f in sorted(FABRIC_DIR.glob("*.md"), key=lambda p: p.stat().st_mtime, reverse=True):
        head = _parse_head(f)
        if head.get("tier") != "hot":
            continue
        if agent and head.get("agent") != agent:
            continue
        out.append({
            "agent": head.get("agent", ""),
            "timestamp": head.get("timestamp", ""),
            "summary": head.get("summary", ""),
        })
        if len(out) >= limit:
            break
    return out


def read_cross_agent(limit=3):
    """Read recent entries from OTHER agents."""
    if not FABRIC_DIR.exists():
        return []
    out = []
    for f in sorted(FABRIC_DIR.glob("*.md"), key=lambda p: p.stat().st_mtime, reverse=True):
        head = _parse_head(f)
        if AGENT_NAME and head.get("agent") == AGENT_NAME:
            continue
        if head.get("type") not in ("review", "dialogue", "decision"):
            continue
        agent = head.get("agent", "")
        summary = head.get("summary", "")
        if summary:
            out.append(f"{agent}: {summary}")
        if len(out) >= limit:
            break
    return out


def _parse_head(filepath, max_bytes=800):
    """Parse frontmatter fields from a fabric entry header."""
    text = filepath.read_text("utf-8")[:max_bytes]
    fields = {}
    for key in ("agent", "type", "tier", "status", "summary", "timestamp",
                "review_of", "revises", "customer_id", "assigned_to", "id",
                "project_id", "session_id",
                "outcome", "training_value", "verified", "evidence",
                "source_tool", "artifact_paths"):
        value = _parse_frontmatter_scalar(text, key)
        if value != "":
            fields[key] = value
    fields["file"] = filepath.name
    return fields


def has_entry_ref(ref):
    """Return True when agent:id resolves to a real fabric entry."""
    if not ref or ":" not in ref or not FABRIC_DIR.exists():
        return False
    agent, entry_id = ref.split(":", 1)
    agent = agent.strip()
    entry_id = entry_id.strip()
    if not agent or not entry_id:
        return False

    for d in (FABRIC_DIR, FABRIC_DIR / "cold"):
        if not d.exists():
            continue
        for f in d.glob("*.md"):
            h = _parse_head(f)
            if h.get("agent", "").strip() == agent and h.get("id", "").strip() == entry_id:
                return True
    return False


def curate_entry(entry_id, training_value):
    """Update the training_value field on an existing fabric entry."""
    if training_value not in ("high", "normal", "low"):
        return {"error": f"training_value must be high/normal/low, got '{training_value}'"}

    for d in (FABRIC_DIR, FABRIC_DIR / "cold"):
        if not d.exists():
            continue
        for f in d.glob("*.md"):
            head = f.read_text("utf-8")[:400]
            m = re.search(r"^id: (.+)$", head, re.MULTILINE)
            if not m or m.group(1).strip() != entry_id:
                continue

            text = f.read_text("utf-8")
            if re.search(r"^training_value: .+$", text, re.MULTILINE):
                text = re.sub(r"^training_value: .+$", f"training_value: {training_value}", text, count=1, flags=re.MULTILINE)
            else:
                text = text.replace("\n---\n", f"\ntraining_value: {training_value}\n---\n", 1)
            f.write_text(text, "utf-8")
            return {"status": "updated", "file": f.name, "training_value": training_value}

    return {"error": f"entry {entry_id} not found"}


def read_pending(customer_id=None):
    """Find entries needing this agent's attention."""
    if not FABRIC_DIR.exists():
        return [], [], []

    agent = AGENT_NAME
    open_tasks = []
    reviews = []
    open_tickets = []

    for f in sorted(FABRIC_DIR.glob("*.md"), key=lambda p: p.stat().st_mtime, reverse=True):
        h = _parse_head(f)
        entry_agent = h.get("agent", "")
        assigned_to = h.get("assigned_to", "").strip()

        if h.get("status") == "open" and entry_agent != agent:
            if not agent or assigned_to != agent:
                continue
            if customer_id and h.get("customer_id") != customer_id:
                continue
            open_tasks.append(h)

        if h.get("type") == "review" and entry_agent != agent:
            ref = h.get("review_of", "")
            if agent and ref.startswith(f"{agent}:"):
                reviews.append(h)

        if h.get("status") == "open" and h.get("customer_id"):
            if not agent or assigned_to != agent:
                continue
            if customer_id and h.get("customer_id") != customer_id:
                continue
            if h not in open_tasks:
                open_tickets.append(h)

        if len(open_tasks) + len(reviews) + len(open_tickets) >= 30:
            break

    return open_tasks, reviews, open_tickets


def search_entries(query, limit=10):
    """Keyword search across fabric."""
    if not FABRIC_DIR.exists():
        return []
    results = []
    q = query.lower()
    for d in [FABRIC_DIR, FABRIC_DIR / "cold"]:
        if not d.exists():
            continue
        for f in sorted(d.glob("*.md"), key=lambda p: p.stat().st_mtime, reverse=True):
            text = f.read_text("utf-8")
            if q not in text.lower():
                continue
            head = _parse_head(f)
            summary = head.get("summary", "")
            agent = head.get("agent", "")
            matches = [line.strip() for line in text.split("\n") if q in line.lower()][:3]
            results.append({"file": f.name, "agent": agent, "summary": summary, "matches": matches})
            if len(results) >= limit:
                return results
    return results


# ── Retriever ────────────────────────────────────────────

_retriever = None


def _load_retriever():
    paths = [
        PLUGIN_DIR / "fabric-retrieve.py",
        Path(os.environ.get("FABRIC_RETRIEVE_PATH", "")),
    ]
    if HERMES_HOME:
        paths.append(HERMES_HOME / "plugins" / "icarus" / "fabric-retrieve.py")
    for p in paths:
        if p and p.exists():
            try:
                import importlib.util
                spec = importlib.util.spec_from_file_location("fabric_retrieve", str(p))
                mod = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(mod)
                mod.FABRIC_DIR = FABRIC_DIR
                return mod
            except Exception as exc:
                logger.debug("icarus: retriever load failed from %s: %s", p, exc)
    return None


def recall(query, max_results=5, agent=None, project=None):
    """Smart ranked retrieval. Falls back to read_recent."""
    global _retriever
    if _retriever is None:
        _retriever = _load_retriever()
    if _retriever is None:
        return read_recent(agent, max_results)

    _retriever.FABRIC_DIR = FABRIC_DIR
    try:
        results = _retriever.retrieve(query, max_results=max_results, agent=agent, project=project)
        return [{"score": score, **entry} for score, entry in results]
    except Exception as exc:
        logger.debug("icarus: retrieval error: %s", exc)
        return read_recent(agent, max_results)


# ── Training ─────────────────────────────────────────────

def _together_key():
    key = os.environ.get("TOGETHER_API_KEY", "")
    if key:
        return key
    if HERMES_HOME and (HERMES_HOME / ".env").exists():
        for line in (HERMES_HOME / ".env").read_text().split("\n"):
            if line.startswith("TOGETHER_API_KEY="):
                return line.split("=", 1)[1].strip()
    return ""


def _together_request(method, url, data=None):
    """Make an authenticated request to Together AI."""
    key = _together_key()
    if not key:
        raise RuntimeError("TOGETHER_API_KEY not set")
    headers = {"Authorization": f"Bearer {key}"}
    body = None
    if data is not None:
        body = json.dumps(data).encode()
        headers["Content-Type"] = "application/json"
    req = urllib.request.Request(url, data=body, headers=headers, method=method)
    resp = urllib.request.urlopen(req, timeout=30)
    return json.loads(resp.read())


def export_training(mode="normal"):
    """Export fabric entries as training pairs. Returns stats dict."""
    export_script = PLUGIN_DIR / "export-training.py"
    if not export_script.exists():
        return {"error": "export-training.py not found"}

    with tempfile.TemporaryDirectory() as tmpdir:
        cmd = ["python3", str(export_script), "--output", tmpdir]
        if mode != "normal":
            cmd.extend(["--mode", mode])
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
        if result.returncode != 0:
            return {"error": result.stderr or "export failed"}

        output = result.stdout
        pairs = 0
        m = re.search(r"total pairs:\s+(\d+)", output)
        if m:
            pairs = int(m.group(1))

        tokens = 0
        m = re.search(r"estimated tokens:\s+([\d,]+)", output)
        if m:
            tokens = int(m.group(1).replace(",", ""))

        together_path = Path(tmpdir) / "together.jsonl"
        training_data = together_path.read_text("utf-8") if together_path.exists() else ""

        pair_types = {}
        raw_pairs_path = Path(tmpdir) / "raw-pairs.json"
        if raw_pairs_path.exists():
            try:
                raw_pairs = json.loads(raw_pairs_path.read_text("utf-8"))
                for pair in raw_pairs:
                    ptype = pair.get("metadata", {}).get("type", "unknown")
                    pair_types[ptype] = pair_types.get(ptype, 0) + 1
            except Exception:
                pair_types = {}

        return {
            "pairs": pairs,
            "estimated_tokens": tokens,
            "mode": mode,
            "pair_types": pair_types,
            "output": output.strip(),
            "training_data_path": str(together_path) if together_path.exists() else None,
            "_training_data": training_data,
        }

def _select_training_export_mode(min_pairs):
    """Choose the highest-quality export mode that still has enough pairs."""
    tried = {}
    for mode in ("high-precision", "normal", "high-volume"):
        result = export_training(mode=mode)
        tried[mode] = result
        if "error" in result:
            continue
        if result.get("pairs", 0) >= min_pairs:
            return mode, result, tried
    # fallback to normal result if all are below threshold but export succeeded
    for mode in ("normal", "high-volume", "high-precision"):
        result = tried.get(mode)
        if result and "error" not in result:
            return mode, result, tried
    return "normal", {"error": "export failed"}, tried


def start_training(model=None, suffix=None, epochs=3, batch_size=None, learning_rate=None,
                   checkpoints=None, mode=None, min_pairs=10):
    """Export, upload, and start a Together AI fine-tune."""
    key = _together_key()
    if not key:
        return {"error": "TOGETHER_API_KEY not set in .env"}

    if mode:
        export = export_training(mode=mode)
        export_mode = mode
        tried_modes = {mode: export}
    else:
        export_mode, export, tried_modes = _select_training_export_mode(min_pairs)
    if "error" in export:
        return export
    if export["pairs"] < min_pairs:
        return {
            "error": f"only {export['pairs']} pairs in {export_mode}, need at least {min_pairs}",
            "mode": export_mode,
            "tried_modes": {k: v.get("pairs", 0) for k, v in tried_modes.items() if "error" not in v},
        }

    training_data = export.get("_training_data", "")
    if not training_data:
        return {"error": "no training data produced"}

    boundary = secrets.token_hex(16)
    body = (
        f"--{boundary}\r\n"
        f'Content-Disposition: form-data; name="purpose"\r\n\r\nfine-tune\r\n'
        f"--{boundary}\r\n"
        f'Content-Disposition: form-data; name="file"; filename="training.jsonl"\r\n'
        f"Content-Type: application/octet-stream\r\n\r\n"
        f"{training_data}\r\n"
        f"--{boundary}--\r\n"
    ).encode()

    req = urllib.request.Request(
        "https://api.together.xyz/v1/files/upload",
        data=body,
        headers={
            "Authorization": f"Bearer {key}",
            "Content-Type": f"multipart/form-data; boundary={boundary}",
        },
        method="POST",
    )
    try:
        resp = urllib.request.urlopen(req, timeout=60)
        upload_data = json.loads(resp.read())
    except Exception as exc:
        return {"error": f"upload failed: {exc}"}

    file_id = upload_data.get("id", "")
    if not file_id:
        return {"error": "upload succeeded but no file ID returned"}

    agent = AGENT_NAME or "agent"
    ft_model = model or os.environ.get("TOGETHER_MODEL", "Qwen/Qwen2-7B-Instruct")
    ft_suffix = suffix or os.environ.get("TOGETHER_SUFFIX", f"{agent}-v1")
    ft_batch = int(batch_size if batch_size is not None else os.environ.get("TOGETHER_BATCH_SIZE", "8"))
    ft_lr = float(learning_rate if learning_rate is not None else os.environ.get("TOGETHER_LR", "1e-5"))
    ft_checkpoints = int(checkpoints if checkpoints is not None else os.environ.get("TOGETHER_CHECKPOINTS", "1"))

    if ft_batch < 8:
        return {"error": f"batch_size must be >= 8 (got {ft_batch})"}
    if ft_lr <= 0:
        return {"error": f"learning_rate must be > 0 (got {ft_lr})"}
    if ft_checkpoints < 1:
        return {"error": f"n_checkpoints must be >= 1 (got {ft_checkpoints})"}

    try:
        ft_data = _together_request("POST", "https://api.together.xyz/v1/fine-tunes", {
            "training_file": file_id,
            "model": ft_model,
            "n_epochs": epochs,
            "suffix": ft_suffix,
            "batch_size": ft_batch,
            "learning_rate": ft_lr,
            "n_checkpoints": ft_checkpoints,
        })
    except Exception as exc:
        return {"error": f"fine-tune start failed: {exc}"}

    job_id = ft_data.get("id", "")
    if not job_id:
        return {"error": "fine-tune accepted but no job ID"}

    _save_job_id(job_id)

    # register as pending in model registry
    registry = _load_registry()
    registry["models"].append({
        "job_id": job_id,
        "base_model": ft_model,
        "output_model": None,
        "suffix": ft_suffix,
        "created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "pair_count": export["pairs"],
        "estimated_tokens": export.get("estimated_tokens", 0),
        "pair_types": export.get("pair_types", {}),
        "export_mode": export_mode,
        "status": "pending",
        "eval_scores": None,
        "active": False,
    })
    _save_registry(registry)

    return {
        "job_id": job_id,
        "model": ft_model,
        "suffix": ft_suffix,
        "epochs": epochs,
        "batch_size": ft_batch,
        "learning_rate": ft_lr,
        "n_checkpoints": ft_checkpoints,
        "pairs": export["pairs"],
        "estimated_tokens": export.get("estimated_tokens", 0),
        "pair_types": export.get("pair_types", {}),
        "mode": export_mode,
        "file_id": file_id,
    }


def check_training(job_id=None):
    """Check a Together AI fine-tune job status."""
    jid = job_id or _last_job_id()
    if not jid:
        return {"error": "no job ID — run fabric_train first"}
    try:
        data = _together_request("GET", f"https://api.together.xyz/v1/fine-tunes/{jid}")
    except Exception as exc:
        return {"error": f"status check failed: {exc}"}

    result = {"job_id": jid, "status": data.get("status", "unknown")}

    # update model registry
    registry = _load_registry()
    for m in registry["models"]:
        if m["job_id"] != jid:
            continue
        if data.get("status") == "completed":
            m["status"] = "completed"
            m["output_model"] = data.get("model_output_name", "")
            result["model_id"] = m["output_model"]
            result["instruction"] = f"Run fabric_eval to test, then fabric_switch_model to activate."
        elif data.get("status") in ("failed", "cancelled", "error"):
            m["status"] = data["status"]
            result["error"] = data.get("error", "unknown")
        break
    _save_registry(registry)

    return result


def run_eval(candidate_model, base_model=None, sample_count=10):
    """Run replacement-model eval. Returns comparison results."""
    eval_script = PLUGIN_DIR / "scripts" / "eval-replacement.py"
    if not eval_script.exists():
        return {"error": "eval-replacement.py not found"}

    key = _together_key()
    if not key:
        return {"error": "TOGETHER_API_KEY not set"}

    base = base_model or os.environ.get("LLM_MODEL", "Qwen/Qwen2-7B-Instruct")

    cmd = [
        "python3", str(eval_script),
        "--candidate-model", candidate_model,
        "--base-model", base,
        "--sample-count", str(sample_count),
        "--together-api-key", key,
        "--fabric-dir", str(FABRIC_DIR),
    ]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
    except subprocess.TimeoutExpired:
        return {"error": "eval timed out (5 min limit)"}

    if result.returncode != 0:
        return {"error": result.stderr or "eval failed"}

    try:
        scores = json.loads(result.stdout)
    except json.JSONDecodeError:
        return {"error": "eval output not valid JSON", "raw": result.stdout[:500]}

    # update registry with eval scores
    registry = _load_registry()
    for m in registry["models"]:
        if m.get("output_model") == candidate_model:
            m["eval_scores"] = scores.get("candidate_scores")
            break
    _save_registry(registry)

    return scores


def switch_model(model_id, min_eval_score=0.7):
    """Switch the agent to use a replacement model."""
    if not HERMES_HOME:
        return {"error": "HERMES_HOME not set"}

    env_file = HERMES_HOME / ".env"
    if not env_file.exists():
        return {"error": f".env not found at {env_file}"}

    registry = _load_registry()
    target = None
    for m in registry["models"]:
        if m.get("output_model") == model_id:
            target = m
            break
    if not target:
        return {"error": f"model {model_id} not in registry"}

    if target.get("eval_scores") is None:
        return {"error": "no eval scores — run fabric_eval first"}

    scores = target["eval_scores"]
    if isinstance(scores, dict):
        avg = sum(scores.values()) / max(len(scores), 1)
    elif isinstance(scores, (int, float)):
        avg = scores
    else:
        return {"error": f"unexpected eval_scores format: {type(scores)}"}

    if avg < min_eval_score:
        return {
            "error": f"eval score {avg:.2f} below threshold {min_eval_score}",
            "scores": scores,
        }

    # find current model for rollback
    lines = env_file.read_text("utf-8").split("\n")
    old_model = None
    for l in lines:
        if l.startswith("LLM_MODEL="):
            old_model = l.split("=", 1)[1].strip()

    # backup .env
    backup = HERMES_HOME / ".env.backup"
    shutil.copy2(env_file, backup)

    key = _together_key()
    if not key:
        return {"error": "TOGETHER_API_KEY not set in .env"}

    # Replacement models are served from Together. Repoint the OpenAI-compatible
    # provider config deliberately and rely on the backup / rollback path to
    # preserve the previous provider state.
    filtered = [
        l for l in lines
        if not l.startswith(("LLM_MODEL=", "OPENAI_BASE_URL=", "OPENAI_API_KEY="))
    ]
    filtered.append(f"LLM_MODEL={model_id}")
    filtered.append("OPENAI_BASE_URL=https://api.together.xyz/v1")
    filtered.append(f"OPENAI_API_KEY={key}")

    # atomic write
    tmp = env_file.with_suffix(".tmp")
    tmp.write_text("\n".join(filtered), "utf-8")
    tmp.rename(env_file)

    # update registry
    for m in registry["models"]:
        if m.get("active"):
            m["active"] = False
    target["active"] = True
    registry["active_model"] = model_id
    _save_registry(registry)

    rollback = f"fabric_switch_model(model_id='{old_model}')" if old_model else "restore from .env.backup"
    return {
        "status": "switched",
        "old_model": old_model,
        "new_model": model_id,
        "eval_score": avg,
        "backup": str(backup),
        "rollback": rollback,
    }


def rollback_model():
    """Restore .env from backup and deactivate current model in registry."""
    if not HERMES_HOME:
        return {"error": "HERMES_HOME not set"}

    backup = HERMES_HOME / ".env.backup"
    env_file = HERMES_HOME / ".env"

    if not backup.exists():
        return {"error": "no .env.backup found — nothing to roll back to"}

    # read what we're rolling back from
    current_model = None
    if env_file.exists():
        for l in env_file.read_text("utf-8").split("\n"):
            if l.startswith("LLM_MODEL="):
                current_model = l.split("=", 1)[1].strip()

    shutil.copy2(backup, env_file)

    # read what we rolled back to
    restored_model = None
    for l in env_file.read_text("utf-8").split("\n"):
        if l.startswith("LLM_MODEL="):
            restored_model = l.split("=", 1)[1].strip()

    # update registry
    registry = _load_registry()
    for m in registry["models"]:
        if m.get("active"):
            m["active"] = False
    restored_match = None
    for m in registry["models"]:
        if m.get("output_model") == restored_model:
            m["active"] = True
            restored_match = restored_model
    registry["active_model"] = restored_match
    _save_registry(registry)

    return {
        "status": "rolled_back",
        "from_model": current_model,
        "to_model": restored_model,
    }


# ── Session scoring ───────────────────────────────────────

def _count_session_entries():
    """Count entries written during the current session."""
    if not session_id or not FABRIC_DIR.exists():
        return 0
    count = 0
    for f in FABRIC_DIR.glob("*.md"):
        head = f.read_text("utf-8")[:400]
        if f"session_id: {session_id}" in head:
            count += 1
    return count


def _count_session_linked_entries():
    """Count linked workflow entries written during the current session."""
    if not session_id or not FABRIC_DIR.exists():
        return 0
    count = 0
    for f in FABRIC_DIR.glob("*.md"):
        head = _parse_head(f)
        if head.get("session_id") != session_id:
            continue
        if head.get("review_of") or head.get("revises"):
            count += 1
    return count


def list_session_entries():
    """List entries written during the current session."""
    if not session_id or not FABRIC_DIR.exists():
        return []
    results = []
    for f in sorted(FABRIC_DIR.glob("*.md"), key=lambda p: p.stat().st_mtime):
        head = f.read_text("utf-8")[:400]
        if f"session_id: {session_id}" not in head:
            continue
        h = _parse_head(f)
        results.append(h)
    return results


def score_session():
    """Score the current session quality. Returns component scores and total."""
    scores = {}

    substantive = [ex for ex in exchanges if len(ex.get("assistant", "").strip()) > 100]
    scores["depth"] = min(len(substantive) / 5, 1.0)

    all_text = " ".join(ex.get("assistant", "") for ex in exchanges)
    has_decision = bool(DECISION_RE.search(all_text))
    has_outcome = bool(OUTCOME_RE.search(all_text))
    scores["decision"] = 1.0 if (has_decision and has_outcome) else (0.5 if has_decision else 0.0)

    tel = get_telemetry(last_n=500, session_id_filter=session_id, agent_filter=AGENT_NAME)
    scores["recall_usage"] = tel.get("summary", {}).get("usage_rate", 0.0)

    scores["linked_entries"] = min(_count_session_linked_entries() / 2, 1.0)

    substantial_user = sum(1 for ex in exchanges if len(ex.get("user", "").strip()) > 50)
    scores["user_engagement"] = min(substantial_user / 3, 1.0)

    weights = {"depth": 2, "decision": 3, "recall_usage": 2, "linked_entries": 2, "user_engagement": 1}
    total = sum(scores[k] * weights[k] for k in scores) / sum(weights.values())
    scores["total"] = round(total, 2)

    return scores


# ── Corpus reporting ─────────────────────────────────────

def get_entry_usage_stats():
    """Per-entry-type recall and usage rates from telemetry."""
    tel = get_telemetry(last_n=500)
    recalled_ids = set()
    used_ids = set()
    for event in tel.get("events", []):
        if event.get("event") == "recall":
            recalled_ids.update(event.get("result_ids", []))
        elif event.get("event") == "usage":
            eid = event.get("entry_id", "")
            if eid:
                used_ids.add(eid)
    used_ids &= recalled_ids

    type_recalled: dict = {}
    type_used: dict = {}
    if FABRIC_DIR.exists():
        for f in FABRIC_DIR.glob("*.md"):
            h = _parse_head(f)
            eid = h.get("id", "")
            etype = h.get("type", "unknown")
            if eid in recalled_ids:
                type_recalled[etype] = type_recalled.get(etype, 0) + 1
            if eid in used_ids:
                type_used[etype] = type_used.get(etype, 0) + 1

    all_types = sorted(set(list(type_recalled.keys()) + list(type_used.keys())))
    return {
        "by_type": {
            t: {
                "recalled": type_recalled.get(t, 0),
                "used": type_used.get(t, 0),
                "usage_rate": round(type_used.get(t, 0) / max(type_recalled.get(t, 0), 1), 2),
            }
            for t in all_types
        },
    }


def build_weekly_report():
    """Corpus health report: entry types, training values, recall stats."""
    entries = []
    if FABRIC_DIR.exists():
        for f in FABRIC_DIR.glob("*.md"):
            entries.append(_parse_head(f))

    by_type: dict = {}
    by_tv = {"high": 0, "normal": 0, "low": 0, "unset": 0}
    verified_count = 0
    trainable_ids = set()

    for e in entries:
        t = e.get("type", "unknown")
        by_type[t] = by_type.get(t, 0) + 1
        tv = e.get("training_value", "")
        by_tv[tv if tv in by_tv else "unset"] += 1
        if str(e.get("verified", "")).lower() == "true":
            verified_count += 1
        if tv == "high" or str(e.get("verified", "")).lower() == "true":
            entry_id = e.get("id")
            if entry_id:
                trainable_ids.add(str(entry_id))

    usage_stats = get_entry_usage_stats()
    tel = get_telemetry(last_n=200)

    return {
        "total_entries": len(entries),
        "by_type": by_type,
        "by_training_value": by_tv,
        "verified_entries": verified_count,
        "recall_usage": tel.get("summary", {}),
        "usage_by_type": usage_stats.get("by_type", {}),
        "trainable_estimate": len(trainable_ids),
    }


# ── SOUL ─────────────────────────────────────────────────

def load_soul():
    if HERMES_HOME:
        soul = HERMES_HOME / "SOUL.md"
        if soul.exists():
            return soul.read_text("utf-8")
    return ""


# ── Memory file ──────────────────────────────────────────

def write_memory_file(s):
    """Write Icarus creative state to CREATIVE.md — NOT to MEMORY.md.

    MEMORY.md is owned exclusively by the memory tool (§-delimited format).
    Writing to it from here caused drift that blocked memory persistence
    (issue #26045).  CREATIVE.md is safe because nothing else writes to it.
    """
    if not HERMES_HOME:
        return
    mem_dir = HERMES_HOME / "memories"
    mem_dir.mkdir(parents=True, exist_ok=True)
    agent = AGENT_NAME or "agent"
    mem_path = mem_dir / "CREATIVE.md"

    lines = [f"# {agent} creative state\n"]
    if s.get("questions"):
        lines.append("## open questions")
        for q in s["questions"][-5:]:
            lines.append(f"- {q}")
        lines.append("")
    if s.get("learnings"):
        lines.append("## learnings")
        for ln in s["learnings"][-5:]:
            lines.append(f"- {ln}")
        lines.append("")
    lines.append(f"cycles: {s.get('cycle', 0)}")

    mem_path.write_text("\n".join(lines), "utf-8")