Add Slack read-only test cases for MVP coverage

agent_eval/openshell/crabline_score.py

@@ -168,3 +168,51 @@ def score_threaded_answer(
         )
     need = f"answer={answer!r}" + (f" and marker={marker!r}" if marker else "")
     return False, f"no accepted threaded chat.postMessage containing {need}"
+
+
+def score_read_code_in_response(
+    outputs: Optional[dict] = None,
+    **_: Any,
+) -> Union[bool, Tuple[bool, str]]:
+    """True when the agent's response contains expected_code from read-only data.
+
+    The code is seeded into Slack history; the agent must read it via
+    conversations.history and report it back.
+    """
+    outputs = outputs or {}
+    ann = outputs.get("annotations") or {}
+    code = (ann.get("expected_code") or "").strip()
+    if not code:
+        return False, "annotations.expected_code is empty"
+    response = (outputs.get("output_content") or "").strip()
+    if not response:
+        return False, "agent response is empty"
+    if code in response:
+        return True, f"response contains code={code!r}"
+    return False, f"response does not contain code={code!r}"
+
+
+def score_channels_listed(
+    outputs: Optional[dict] = None,
+    **_: Any,
+) -> Union[bool, Tuple[bool, str]]:
+    """True when the agent's response contains all expected channel/group/DM IDs.
+
+    Used by :read cases: multiple channels are seeded, and the agent must
+    call conversations.list and report all of them.
+    """
+    outputs = outputs or {}
+    ann = outputs.get("annotations") or {}
+    expected = ann.get("expected_channels")
+    if not expected or not isinstance(expected, list):
+        return False, "annotations.expected_channels is missing or not a list"
+    response = (outputs.get("output_content") or "").strip()
+    if not response:
+        return False, "agent response is empty"
+    missing = [ch for ch in expected if ch not in response]
+    if not missing:
+        return True, f"response contains all {len(expected)} expected channels"
+    return (
+        False,
+        f"response missing {len(missing)}/{len(expected)} channels: {missing}",
+    )

agent_eval/openshell/crabline_seed.py

@@ -74,33 +74,29 @@ def load_case_annotations(config, case_id: str) -> dict:
     return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
 
 
-def seed_crabline_for_case(
-    annotations: dict,
-    *,
-    api_root: Optional[str] = None,
-    token: Optional[str] = None,
-) -> Optional[dict[str, Any]]:
-    """Post annotations.crabline_seed to Crabline. Return seed metadata or None."""
-    seed = (annotations or {}).get("crabline_seed")
-    if not seed:
-        return None
+def _seed_one(
+    seed: dict,
+    api_root: str,
+    token: str,
+) -> dict[str, Any]:
+    """Post a single seed item to Crabline. Return metadata."""
     text = (seed.get("text") or "").strip()
     if not text:
-        raise ValueError("annotations.crabline_seed.text is required")
-    users = (seed.get("users") or "UCANARY01").strip()
-    api_root = api_root or _api_root()
-    token = token or _bot_token()
-    if not token:
-        raise RuntimeError(
-            "SLACK_BOT_TOKEN (or Crabline ready file) required to seed Crabline"
-        )
-
-    opened = _slack_form(api_root, "conversations.open", token, {"users": users})
-    if not opened.get("ok"):
-        raise RuntimeError(f"conversations.open failed: {opened}")
-    channel = (opened.get("channel") or {}).get("id")
-    if not channel:
-        raise RuntimeError(f"conversations.open missing channel id: {opened}")
+        raise ValueError("crabline seed text is required")
+    direct_channel = (seed.get("channel") or "").strip()
+    users = (seed.get("users") or "").strip()
+    if not direct_channel and not users:
+        users = "UCANARY01"
+
+    if direct_channel:
+        channel = direct_channel
+    else:
+        opened = _slack_form(api_root, "conversations.open", token, {"users": users})
+        if not opened.get("ok"):
+            raise RuntimeError(f"conversations.open failed: {opened}")
+        channel = (opened.get("channel") or {}).get("id")
+        if not channel:
+            raise RuntimeError(f"conversations.open missing channel id: {opened}")
 
     posted = _slack_form(
         api_root,
@@ -112,18 +108,59 @@ def seed_crabline_for_case(
         raise RuntimeError(f"chat.postMessage seed failed: {posted}")
 
     ts = posted.get("ts") or (posted.get("message") or {}).get("ts")
-    result = {
+    result: dict[str, Any] = {
         "ok": True,
-        "users": users,
         "channel": channel,
         "ts": ts,
         "text": text,
-        "api_root": api_root,
     }
+    if users:
+        result["users"] = users
     logger.info(
         "Crabline seed: channel=%s ts=%s text=%r",
         channel,
         ts,
         text[:80],
     )
     return result
+
+
+def seed_crabline_for_case(
+    annotations: dict,
+    *,
+    api_root: Optional[str] = None,
+    token: Optional[str] = None,
+) -> Optional[dict[str, Any]]:
+    """Post annotations.crabline_seed (or crabline_seeds) to Crabline.
+
+    Supports:
+    - ``crabline_seed`` (singular): one seed with ``users`` or ``channel`` + ``text``
+    - ``crabline_seeds`` (plural): list of seeds for multi-item discovery tests
+
+    Returns seed metadata dict (single) or dict with ``seeds`` list (multi), or None.
+    """
+    api_root = api_root or _api_root()
+    token = token or _bot_token()
+    if not token:
+        raise RuntimeError(
+            "SLACK_BOT_TOKEN (or Crabline ready file) required to seed Crabline"
+        )
+
+    seeds_list = (annotations or {}).get("crabline_seeds")
+    if seeds_list and isinstance(seeds_list, list):
+        results = []
+        for item in seeds_list:
+            results.append(_seed_one(item, api_root, token))
+        return {
+            "ok": True,
+            "seeds": results,
+            "channels": [r["channel"] for r in results],
+            "api_root": api_root,
+        }
+
+    seed = (annotations or {}).get("crabline_seed")
+    if not seed:
+        return None
+    result = _seed_one(seed, api_root, token)
+    result["api_root"] = api_root
+    return result