Skip to content

Commit abd2923

Browse files
committed
test(skills): direct unit tests for skill_runner/skills/critique-slash + doc fix
Three follow-up items from the architectural review (scores 60-70): 1. **openkb/skill/generator.py docstring** — claimed "future targets plug in by declaring an output dir and a validator" but no plug-in registry exists; current dispatch is a literal ``if target_type == "skill"`` else-branch. Reworded to be honest about scope and to point at the deferred-followups entry for the registry refactor. 2. **tests/test_skill_runner.py** (NEW, 8 tests) — direct coverage of the function every CLI/chat path now funnels through: - SkillNotFoundError lists what IS available - body lands in agent.instructions with "## User intent" section - write_file + read_output_or_skill_file tools wired - output_path_template substitutes {slug} and enforces post-run existence - od.mode=="deck" triggers validate_deck with the skill's grammar - od.mode missing or non-deck skips validation - MaxTurnsExceeded → RuntimeError with skill-name + step-cap info - default max_turns matches MAX_TURNS constant 3. **tests/test_skills.py** (NEW, 16 tests) — direct coverage of the scanner & frontmatter parser: - empty when no roots exist - SDK shape (name/description/path, path absolute) - skips dirs without SKILL.md - skips skills missing description - falls back to dir name when frontmatter omits name (pinned as intentional sane-default, not a bug) - truncates description to 1024 chars - first-hit-wins precedence across roots - extra_roots appended after defaults (doesn't override) - DEFAULT_SKILL_ROOTS constant pinned - frontmatter: happy path, no-delim, unclosed, malformed YAML, non-dict YAML, body-containing-dashes preservation - autouse $HOME isolation so the user's real ~/.claude/skills doesn't pollute test results 4. **tests/test_critique_slash.py** (NEW, 6 tests) — direct coverage of /critique <path> chat slash: - no arg → usage print, no run_skill call - missing file → [ERROR], no run_skill call - happy path → run_skill called with openkb-html-critic + relative path in intent - absolute path inside KB → converted to relative form - SkillNotFoundError → [ERROR] (does not crash chat turn) - RuntimeError → [ERROR] (does not propagate) Regression: 569 tests pass (was 539; net +30 new tests in this commit). The fifth deferred follow-up — bitmap image inline handling — is out of scope (no concrete user need yet). The Generator if/else → registry refactor and the chat-freeform iteration-backup gap stay deferred per the original PR description.
1 parent 916b938 commit abd2923

3 files changed

Lines changed: 667 additions & 0 deletions

File tree

tests/test_critique_slash.py

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
"""Direct unit tests for ``openkb.agent.chat._handle_slash_critique``.
2+
3+
The ``/critique <path>`` slash command is the user-facing entry point
4+
for the html-critic skill. It does path resolution, file-not-found
5+
gating, and translates SkillNotFoundError / RuntimeError into
6+
user-visible error messages — none of that was exercised before.
7+
"""
8+
from __future__ import annotations
9+
10+
from pathlib import Path
11+
from unittest.mock import AsyncMock, patch
12+
13+
import pytest
14+
from prompt_toolkit.styles import Style
15+
16+
from openkb.agent.chat import _handle_slash_critique
17+
from openkb.agent.skill_runner import SkillNotFoundError
18+
19+
20+
def _make_kb_with_config(tmp_path: Path) -> Path:
21+
"""Critique needs a config.yaml to read the model from."""
22+
(tmp_path / ".openkb").mkdir()
23+
(tmp_path / ".openkb" / "config.yaml").write_text(
24+
"model: openai/gpt-4o\nlanguage: en\n", encoding="utf-8"
25+
)
26+
return tmp_path
27+
28+
29+
_STYLE = Style.from_dict({})
30+
31+
32+
@pytest.mark.asyncio
33+
async def test_critique_no_arg_prints_usage(tmp_path: Path, capsys):
34+
"""``/critique`` with no arg must print Usage and NOT call run_skill."""
35+
kb_dir = _make_kb_with_config(tmp_path)
36+
with patch("openkb.agent.skill_runner.run_skill", new=AsyncMock()) as run_skill:
37+
await _handle_slash_critique("", kb_dir, _STYLE)
38+
# whitespace only — same as empty
39+
await _handle_slash_critique(" ", kb_dir, _STYLE)
40+
41+
out = capsys.readouterr().out
42+
assert "Usage" in out or "usage" in out
43+
run_skill.assert_not_called()
44+
45+
46+
@pytest.mark.asyncio
47+
async def test_critique_missing_file_prints_error(tmp_path: Path, capsys):
48+
"""When the target file doesn't exist, print an ERROR and skip
49+
run_skill — no point asking the critic to read a nonexistent file."""
50+
kb_dir = _make_kb_with_config(tmp_path)
51+
with patch("openkb.agent.skill_runner.run_skill", new=AsyncMock()) as run_skill:
52+
await _handle_slash_critique(
53+
"output/decks/ghost/index.html", kb_dir, _STYLE
54+
)
55+
56+
out = capsys.readouterr().out
57+
assert "[ERROR]" in out
58+
assert "not found" in out.lower() or "ghost" in out
59+
run_skill.assert_not_called()
60+
61+
62+
@pytest.mark.asyncio
63+
async def test_critique_invokes_html_critic_skill(tmp_path: Path, capsys):
64+
"""Happy path: file exists → run_skill called with the html-critic
65+
skill name and a path that includes the relative target."""
66+
kb_dir = _make_kb_with_config(tmp_path)
67+
target = kb_dir / "output" / "decks" / "real" / "index.html"
68+
target.parent.mkdir(parents=True)
69+
target.write_text("<html>existing deck</html>", encoding="utf-8")
70+
71+
with patch("openkb.agent.skill_runner.run_skill", new=AsyncMock()) as run_skill:
72+
await _handle_slash_critique(
73+
"output/decks/real/index.html", kb_dir, _STYLE
74+
)
75+
76+
run_skill.assert_called_once()
77+
kwargs = run_skill.call_args.kwargs
78+
assert kwargs["skill_name"] == "openkb-html-critic"
79+
assert "output/decks/real/index.html" in kwargs["intent"]
80+
assert kwargs["kb_dir"] == kb_dir
81+
out = capsys.readouterr().out
82+
assert "Critique pass complete" in out
83+
84+
85+
@pytest.mark.asyncio
86+
async def test_critique_accepts_absolute_path_inside_kb(tmp_path: Path):
87+
"""Absolute paths under the KB are accepted and converted to the
88+
relative form for the skill's intent."""
89+
kb_dir = _make_kb_with_config(tmp_path)
90+
target = kb_dir / "output" / "decks" / "abs" / "index.html"
91+
target.parent.mkdir(parents=True)
92+
target.write_text("<html></html>", encoding="utf-8")
93+
94+
with patch("openkb.agent.skill_runner.run_skill", new=AsyncMock()) as run_skill:
95+
await _handle_slash_critique(str(target), kb_dir, _STYLE)
96+
97+
run_skill.assert_called_once()
98+
intent = run_skill.call_args.kwargs["intent"]
99+
# Either the relative-to-kb form or the absolute path is in the
100+
# intent — implementation may choose either, both reach the skill.
101+
assert "abs/index.html" in intent or str(target) in intent
102+
103+
104+
@pytest.mark.asyncio
105+
async def test_critique_catches_skill_not_found(tmp_path: Path, capsys):
106+
"""If the openkb-html-critic skill is missing, surface a friendly
107+
[ERROR] line instead of crashing the chat turn."""
108+
kb_dir = _make_kb_with_config(tmp_path)
109+
target = kb_dir / "output" / "test.html"
110+
target.parent.mkdir(parents=True, exist_ok=True)
111+
target.write_text("<html></html>", encoding="utf-8")
112+
113+
async def missing(**_):
114+
raise SkillNotFoundError(
115+
"Skill 'openkb-html-critic' not found. Available: foo."
116+
)
117+
118+
with patch("openkb.agent.skill_runner.run_skill", new=AsyncMock(side_effect=missing)):
119+
# Should NOT raise — chat turn must survive
120+
await _handle_slash_critique(str(target), kb_dir, _STYLE)
121+
122+
out = capsys.readouterr().out
123+
assert "[ERROR]" in out
124+
assert "openkb-html-critic" in out or "not found" in out.lower()
125+
126+
127+
@pytest.mark.asyncio
128+
async def test_critique_catches_runtime_error_from_run_skill(tmp_path: Path, capsys):
129+
"""RuntimeError from run_skill (e.g. MaxTurnsExceeded translation)
130+
is surfaced as [ERROR] not propagated."""
131+
kb_dir = _make_kb_with_config(tmp_path)
132+
target = kb_dir / "output" / "test.html"
133+
target.parent.mkdir(parents=True, exist_ok=True)
134+
target.write_text("<html></html>", encoding="utf-8")
135+
136+
async def hits_cap(**_):
137+
raise RuntimeError("Skill 'openkb-html-critic' hit the 40-step cap")
138+
139+
with patch("openkb.agent.skill_runner.run_skill", new=AsyncMock(side_effect=hits_cap)):
140+
await _handle_slash_critique(str(target), kb_dir, _STYLE)
141+
142+
out = capsys.readouterr().out
143+
assert "[ERROR]" in out
144+
assert "step cap" in out or "step" in out.lower()

0 commit comments

Comments
 (0)