From 9ad42796c7686ba46bcb1a459ccf07685691c9b3 Mon Sep 17 00:00:00 2001 From: Zane Date: Wed, 29 Jul 2026 19:20:12 +1200 Subject: [PATCH 1/3] Add rank-aware retrieval evaluation and run artifacts, measure source-level precision, MRR, and nDCG while exporting reproducible JSON and CSV reports --- eval/evaluate.py | 259 ++++++++++++++++++++++++++++------- eval/metrics.py | 58 ++++++++ eval/reporting.py | 56 ++++++++ tests/test_eval_metrics.py | 51 +++++++ tests/test_eval_reporting.py | 40 ++++++ tests/test_eval_runner.py | 65 +++++++++ 6 files changed, 479 insertions(+), 50 deletions(-) create mode 100644 eval/metrics.py create mode 100644 eval/reporting.py create mode 100644 tests/test_eval_metrics.py create mode 100644 tests/test_eval_reporting.py create mode 100644 tests/test_eval_runner.py diff --git a/eval/evaluate.py b/eval/evaluate.py index acdc0c3..ec74530 100644 --- a/eval/evaluate.py +++ b/eval/evaluate.py @@ -1,16 +1,25 @@ -"""Run the pipeline over test_questions.json and score it.""" +"""Run retrieval/generation evals and save reproducible result artifacts.""" from __future__ import annotations +import argparse +import hashlib import json +import subprocess import sys +import time +from datetime import UTC, datetime from pathlib import Path +from typing import Any -from rag.generation import AnswerWithSources, judge_answer -from rag.pipeline import answer_question +from eval.metrics import evaluate_source_ranking, source_matches, unique_sources +from eval.reporting import write_run_artifacts +from rag.config import settings +from rag.generation import AnswerWithSources, generate_answer, judge_answer from rag.retrieval import retrieve QUESTIONS_PATH = Path(__file__).resolve().parent / "test_questions.json" +RESULTS_PATH = Path(__file__).resolve().parent / "results" def load_questions(path: Path = QUESTIONS_PATH) -> list[dict]: @@ -18,16 +27,10 @@ def load_questions(path: Path = QUESTIONS_PATH) -> list[dict]: return json.load(f) -def _source_match(expected: str, source: str) -> bool: - return expected == source or expected in source or Path(source).name == expected - - def retrieval_hit(question: str, relevant_sources: list[str], k: int | None = None) -> bool: chunks = retrieve(question, k=k) - retrieved = {c.source for c in chunks} - return any( - _source_match(expected, src) for expected in relevant_sources for src in retrieved - ) + metrics = evaluate_source_ranking([chunk.source for chunk in chunks], relevant_sources) + return metrics.hit_at_k def citation_precision(result: AnswerWithSources) -> float: @@ -44,73 +47,229 @@ def citation_source_hit(result: AnswerWithSources, relevant_sources: list[str]) if not result.citations or not result.chunks: return False n = len(result.chunks) - cited_sources = { - result.chunks[i - 1].source for i in result.citations if 1 <= i <= n - } + cited_sources = {result.chunks[i - 1].source for i in result.citations if 1 <= i <= n} return any( - _source_match(expected, src) - for expected in relevant_sources - for src in cited_sources + source_matches(expected, src) for expected in relevant_sources for src in cited_sources ) -def run_eval(path: Path = QUESTIONS_PATH) -> int: +def _average(values: list[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def _run_config(k: int) -> dict[str, Any]: + return { + "retrieval_mode": settings.retrieval_mode, + "top_k": k, + "embedding_provider": settings.embedding_provider, + "embedding_model": ( + settings.ollama_embed_model + if settings.embedding_provider == "ollama" + else settings.hf_embed_model + ), + "chat_model": settings.ollama_chat_model, + "chunk_size": settings.chunk_size, + "chunk_overlap": settings.chunk_overlap, + "llm_temperature": settings.llm_temperature, + } + + +def _file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _git_revision() -> str | None: + """Best-effort code revision for reproducible local/CI reports.""" + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=Path(__file__).resolve().parents[1], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + return result.stdout.strip() or None + + +def run_eval( + path: Path = QUESTIONS_PATH, + *, + k: int | None = None, + retrieval_only: bool = False, + output_dir: Path = RESULTS_PATH, + save_artifacts: bool = True, +) -> int: questions = load_questions(path) if not questions: print("No questions in", path) return 1 - hits = 0 - cite_hits = 0 + top_k = k if k is not None else settings.top_k + if top_k < 1: + raise ValueError("k must be at least 1") + + started = time.perf_counter() + run_time = datetime.now(UTC) + milliseconds = run_time.microsecond // 1000 + run_id = f"{run_time:%Y%m%dT%H%M%S}{milliseconds:03d}Z" + results: list[dict[str, Any]] = [] + + hit_scores: list[float] = [] + precision_at_1_scores: list[float] = [] + reciprocal_ranks: list[float] = [] + ndcg_scores: list[float] = [] quality_scores: list[float] = [] + cite_hit_scores: list[float] = [] cite_precision_scores: list[float] = [] - print(f"{'#':>2} {'question':<44} {'hit':>4} {'cite':>4} {'qual':>5}") - print("-" * 70) + print( + f"{'#':>2} {'question':<38} {'rank':>4} {'p@1':>4} " + f"{'rr':>4} {'ndcg':>5} {'cite':>4} {'qual':>5}" + ) + print("-" * 84) for i, item in enumerate(questions, start=1): q = item["question"] expected = item.get("expected_answer", "") relevant = item.get("relevant_sources") or [] - hit = retrieval_hit(q, relevant) - hits += int(hit) + chunks = retrieve(q, k=top_k) + retrieved_sources = unique_sources([chunk.source for chunk in chunks]) + retrieval_metrics = evaluate_source_ranking(retrieved_sources, relevant) + + hit_scores.append(float(retrieval_metrics.hit_at_k)) + precision_at_1_scores.append(retrieval_metrics.precision_at_1) + reciprocal_ranks.append(retrieval_metrics.reciprocal_rank) + ndcg_scores.append(retrieval_metrics.ndcg_at_k) - result = answer_question(q) - score = judge_answer(q, expected, result.answer) if expected else 0.0 - quality_scores.append(score) + answer: str | None = None + citations: list[int] = [] + c_prec: float | None = None + c_hit: bool | None = None + score: float | None = None - c_prec = citation_precision(result) - c_hit = citation_source_hit(result, relevant) - cite_precision_scores.append(c_prec) - cite_hits += int(c_hit) + if not retrieval_only: + answer_result = generate_answer(q, chunks) + answer = answer_result.answer + citations = answer_result.citations + c_prec = citation_precision(answer_result) + c_hit = citation_source_hit(answer_result, relevant) + score = judge_answer(q, expected, answer) if expected else 0.0 + cite_precision_scores.append(c_prec) + cite_hit_scores.append(float(c_hit)) + quality_scores.append(score) - q_short = (q[:41] + "...") if len(q) > 44 else q - print(f"{i:>2} {q_short:<44} {hit!s:>4} {c_hit!s:>4} {score:>5.2f}") - print(f" answer: {result.answer[:200]}") - print(f" sources: {', '.join(result.sources) or '(none)'}") - if result.citations: - print(f" citations: {result.citations} (precision={c_prec:.2f})") + result_row = { + "question": q, + "expected_answer": expected, + "relevant_sources": relevant, + "retrieved_sources": retrieved_sources, + "retrieval": retrieval_metrics.as_dict(), + "answer": answer, + "citations": citations, + "citation_source_hit": c_hit, + "citation_precision": c_prec, + "judge_score": score, + } + results.append(result_row) + + q_short = (q[:35] + "...") if len(q) > 38 else q + rank = retrieval_metrics.first_relevant_rank or "-" + cite_display = "-" if c_hit is None else str(c_hit) + quality_display = "-" if score is None else f"{score:.2f}" + print( + f"{i:>2} {q_short:<38} {rank!s:>4} " + f"{retrieval_metrics.precision_at_1:>4.2f} " + f"{retrieval_metrics.reciprocal_rank:>4.2f} " + f"{retrieval_metrics.ndcg_at_k:>5.2f} " + f"{cite_display:>4} {quality_display:>5}" + ) + if answer is not None: + print(f" answer: {answer[:200]}") + print(f" sources: {', '.join(retrieved_sources) or '(none)'}") + if citations: + print(f" citations: {citations} (precision={c_prec:.2f})") n = len(questions) - print("-" * 70) - print(f"Retrieval hit rate: {hits / n:.2%} ({hits}/{n})") - print(f"Citation source hit: {cite_hits / n:.2%} ({cite_hits}/{n})") - print(f"Citation precision avg: {sum(cite_precision_scores) / n:.2f}") - print(f"Answer quality (judge): {sum(quality_scores) / n:.2f}") - print() - print( - "Note: judge score is noisy (same local model grading itself). " - "Citation metrics are deterministic." - ) + summary: dict[str, int | float | None] = { + "question_count": n, + "hit_rate_at_k": _average(hit_scores), + "precision_at_1": _average(precision_at_1_scores), + "mrr": _average(reciprocal_ranks), + "ndcg_at_k": _average(ndcg_scores), + "citation_source_hit_rate": (_average(cite_hit_scores) if cite_hit_scores else None), + "citation_precision": (_average(cite_precision_scores) if cite_precision_scores else None), + "answer_quality": _average(quality_scores) if quality_scores else None, + "duration_seconds": round(time.perf_counter() - started, 3), + } + report = { + "schema_version": 1, + "run_id": run_id, + "created_at": run_time.isoformat(), + "mode": "retrieval_only" if retrieval_only else "full", + "questions_path": str(path), + "questions_sha256": _file_sha256(path), + "code_revision": _git_revision(), + "config": _run_config(top_k), + "summary": summary, + "results": results, + } + + print("-" * 84) + print(f"Hit rate@{top_k}: {summary['hit_rate_at_k']:.2%}") + print(f"Precision@1: {summary['precision_at_1']:.2%}") + print(f"MRR: {summary['mrr']:.3f}") + print(f"nDCG@{top_k}: {summary['ndcg_at_k']:.3f}") + if not retrieval_only: + print(f"Citation source hit: {summary['citation_source_hit_rate']:.2%}") + print(f"Citation precision: {summary['citation_precision']:.3f}") + print(f"Answer quality: {summary['answer_quality']:.3f}") + print("Judge quality is noisy; retrieval/citation metrics are deterministic.") + + if save_artifacts: + json_path, csv_path = write_run_artifacts(report, output_dir) + print(f"Saved JSON: {json_path}") + print(f"Saved CSV: {csv_path}") return 0 def main(argv: list[str] | None = None) -> int: - args = argv if argv is not None else sys.argv[1:] - path = Path(args[0]) if args else QUESTIONS_PATH + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "questions", + nargs="?", + type=Path, + default=QUESTIONS_PATH, + help=f"Question set (default: {QUESTIONS_PATH})", + ) + parser.add_argument("--k", type=int, help="Override retrieval top-k") + parser.add_argument( + "--retrieval-only", + action="store_true", + help="Skip generation and judge calls for a fast retrieval eval", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=RESULTS_PATH, + help=f"Artifact directory (default: {RESULTS_PATH})", + ) + parser.add_argument( + "--no-artifacts", + action="store_true", + help="Print results without writing JSON/CSV artifacts", + ) + args = parser.parse_args(argv) try: - return run_eval(path) + return run_eval( + args.questions, + k=args.k, + retrieval_only=args.retrieval_only, + output_dir=args.output_dir, + save_artifacts=not args.no_artifacts, + ) except Exception as exc: # noqa: BLE001 print(f"eval failed: {exc}", file=sys.stderr) return 1 diff --git a/eval/metrics.py b/eval/metrics.py new file mode 100644 index 0000000..88da91c --- /dev/null +++ b/eval/metrics.py @@ -0,0 +1,58 @@ +"""Deterministic, source-level retrieval metrics.""" + +from __future__ import annotations + +import math +from dataclasses import asdict, dataclass +from pathlib import Path + + +def source_matches(expected: str, actual: str) -> bool: + """Match a source label against either a full path or basename.""" + return expected == actual or expected in actual or Path(actual).name == expected + + +def unique_sources(sources: list[str]) -> list[str]: + """Preserve retrieval order while collapsing repeated chunks from one source.""" + return list(dict.fromkeys(sources)) + + +@dataclass(frozen=True) +class RetrievalMetrics: + hit_at_k: bool + precision_at_1: float + reciprocal_rank: float + ndcg_at_k: float + first_relevant_rank: int | None + + def as_dict(self) -> dict[str, bool | float | int | None]: + return asdict(self) + + +def evaluate_source_ranking( + retrieved_sources: list[str], + relevant_sources: list[str], +) -> RetrievalMetrics: + """Score an ordered source ranking against source-level relevance labels.""" + ranked = unique_sources(retrieved_sources) + relevance = [ + int(any(source_matches(expected, source) for expected in relevant_sources)) + for source in ranked + ] + + first_rank = next((rank for rank, rel in enumerate(relevance, start=1) if rel), None) + reciprocal_rank = 1.0 / first_rank if first_rank else 0.0 + precision_at_1 = float(bool(relevance and relevance[0])) + + dcg = sum(rel / math.log2(rank + 1) for rank, rel in enumerate(relevance, start=1)) + ideal_relevant = min(len(relevant_sources), len(ranked)) + ideal_dcg = sum(1.0 / math.log2(rank + 1) for rank in range(1, ideal_relevant + 1)) + ndcg = dcg / ideal_dcg if ideal_dcg else 0.0 + + return RetrievalMetrics( + hit_at_k=first_rank is not None, + precision_at_1=precision_at_1, + reciprocal_rank=reciprocal_rank, + ndcg_at_k=ndcg, + first_relevant_rank=first_rank, + ) diff --git a/eval/reporting.py b/eval/reporting.py new file mode 100644 index 0000000..f12a1dd --- /dev/null +++ b/eval/reporting.py @@ -0,0 +1,56 @@ +"""Write reproducible eval run artifacts.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +CSV_FIELDS = [ + "question", + "relevant_sources", + "retrieved_sources", + "hit_at_k", + "precision_at_1", + "reciprocal_rank", + "ndcg_at_k", + "first_relevant_rank", + "citation_source_hit", + "citation_precision", + "judge_score", + "answer", +] + + +def _csv_row(result: dict[str, Any]) -> dict[str, Any]: + retrieval = result["retrieval"] + return { + "question": result["question"], + "relevant_sources": "|".join(result["relevant_sources"]), + "retrieved_sources": "|".join(result["retrieved_sources"]), + **retrieval, + "citation_source_hit": result.get("citation_source_hit"), + "citation_precision": result.get("citation_precision"), + "judge_score": result.get("judge_score"), + "answer": result.get("answer"), + } + + +def write_run_artifacts( + report: dict[str, Any], + output_dir: Path, +) -> tuple[Path, Path]: + """Write a full JSON report and a flat per-question CSV.""" + output_dir.mkdir(parents=True, exist_ok=True) + stem = f"eval-{report['run_id']}" + json_path = output_dir / f"{stem}.json" + csv_path = output_dir / f"{stem}.csv" + + json_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + with csv_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS) + writer.writeheader() + writer.writerows(_csv_row(result) for result in report["results"]) + + return json_path, csv_path diff --git a/tests/test_eval_metrics.py b/tests/test_eval_metrics.py new file mode 100644 index 0000000..9f5ff7e --- /dev/null +++ b/tests/test_eval_metrics.py @@ -0,0 +1,51 @@ +"""Tests for deterministic source-level retrieval metrics.""" + +import pytest + +from eval.metrics import evaluate_source_ranking, source_matches, unique_sources + + +def test_source_matches_basename_or_full_path(): + assert source_matches("guide.md", "data/corpus/guide.md") + assert source_matches("data/corpus/guide.md", "data/corpus/guide.md") + assert not source_matches("other.md", "data/corpus/guide.md") + + +def test_unique_sources_keeps_first_occurrence(): + assert unique_sources(["a.md", "a.md", "b.md", "a.md"]) == ["a.md", "b.md"] + + +def test_perfect_source_ranking(): + metrics = evaluate_source_ranking(["relevant.md", "other.md"], ["relevant.md"]) + assert metrics.hit_at_k is True + assert metrics.precision_at_1 == 1.0 + assert metrics.reciprocal_rank == 1.0 + assert metrics.ndcg_at_k == 1.0 + assert metrics.first_relevant_rank == 1 + + +def test_relevant_source_lower_in_ranking(): + metrics = evaluate_source_ranking(["other.md", "relevant.md"], ["relevant.md"]) + assert metrics.hit_at_k is True + assert metrics.precision_at_1 == 0.0 + assert metrics.reciprocal_rank == 0.5 + assert metrics.ndcg_at_k == pytest.approx(1 / 1.5849625007) + assert metrics.first_relevant_rank == 2 + + +def test_duplicate_chunks_do_not_penalize_source_rank(): + metrics = evaluate_source_ranking( + ["other.md", "other.md", "relevant.md"], + ["relevant.md"], + ) + assert metrics.first_relevant_rank == 2 + assert metrics.reciprocal_rank == 0.5 + + +def test_missed_source_scores_zero(): + metrics = evaluate_source_ranking(["other.md"], ["relevant.md"]) + assert metrics.hit_at_k is False + assert metrics.precision_at_1 == 0.0 + assert metrics.reciprocal_rank == 0.0 + assert metrics.ndcg_at_k == 0.0 + assert metrics.first_relevant_rank is None diff --git a/tests/test_eval_reporting.py b/tests/test_eval_reporting.py new file mode 100644 index 0000000..85cea13 --- /dev/null +++ b/tests/test_eval_reporting.py @@ -0,0 +1,40 @@ +"""Tests for eval JSON and CSV artifacts.""" + +import csv +import json + +from eval.reporting import write_run_artifacts + + +def test_write_run_artifacts(tmp_path): + report = { + "run_id": "20260729T010203Z", + "results": [ + { + "question": "What is it?", + "relevant_sources": ["expected.md"], + "retrieved_sources": ["expected.md", "other.md"], + "retrieval": { + "hit_at_k": True, + "precision_at_1": 1.0, + "reciprocal_rank": 1.0, + "ndcg_at_k": 1.0, + "first_relevant_rank": 1, + }, + "citation_source_hit": True, + "citation_precision": 1.0, + "judge_score": 0.8, + "answer": "A fact [1].", + } + ], + } + + json_path, csv_path = write_run_artifacts(report, tmp_path) + + assert json.loads(json_path.read_text()) == report + with csv_path.open(newline="") as handle: + rows = list(csv.DictReader(handle)) + assert len(rows) == 1 + assert rows[0]["question"] == "What is it?" + assert rows[0]["retrieved_sources"] == "expected.md|other.md" + assert rows[0]["reciprocal_rank"] == "1.0" diff --git a/tests/test_eval_runner.py b/tests/test_eval_runner.py new file mode 100644 index 0000000..51a4dbf --- /dev/null +++ b/tests/test_eval_runner.py @@ -0,0 +1,65 @@ +"""Integration-style tests for the eval orchestration (all providers mocked).""" + +import json + +from eval import evaluate +from rag.generation import AnswerWithSources +from rag.retrieval import Chunk + + +def test_full_eval_retrieves_once_and_writes_report(tmp_path, monkeypatch): + questions_path = tmp_path / "questions.json" + questions_path.write_text( + json.dumps( + [ + { + "question": "What is the fact?", + "expected_answer": "The fact is one.", + "relevant_sources": ["expected.md"], + } + ] + ) + ) + chunks = [ + Chunk( + content="The fact is one.", + source="data/corpus/expected.md", + metadata={}, + distance=0.1, + ) + ] + retrieve_calls = 0 + + def fake_retrieve(question, k): + nonlocal retrieve_calls + retrieve_calls += 1 + assert question == "What is the fact?" + assert k == 3 + return chunks + + def fake_generate(question, retrieved): + assert retrieved is chunks + return AnswerWithSources( + answer="The fact is one [1].", + sources=[chunks[0].source], + chunks=chunks, + citations=[1], + ) + + monkeypatch.setattr(evaluate, "retrieve", fake_retrieve) + monkeypatch.setattr(evaluate, "generate_answer", fake_generate) + monkeypatch.setattr(evaluate, "judge_answer", lambda *_: 0.75) + + exit_code = evaluate.run_eval(questions_path, k=3, output_dir=tmp_path / "results") + + assert exit_code == 0 + assert retrieve_calls == 1 + reports = list((tmp_path / "results").glob("*.json")) + assert len(reports) == 1 + report = json.loads(reports[0].read_text()) + assert report["config"]["top_k"] == 3 + assert len(report["questions_sha256"]) == 64 + assert "code_revision" in report + assert report["summary"]["mrr"] == 1.0 + assert report["summary"]["citation_source_hit_rate"] == 1.0 + assert report["summary"]["answer_quality"] == 0.75 From b671f87975c168cbdda17b1876435a9422716939 Mon Sep 17 00:00:00 2001 From: Zane Date: Wed, 29 Jul 2026 19:33:34 +1200 Subject: [PATCH 2/3] Document the retrieval evaluation workflow, add a fast retrieval-only target and record the rank-aware baseline --- .gitignore | 3 +++ Makefile | 5 ++++- README.md | 40 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 87b33cb..f53f9c1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ data/* !data/corpus/ !data/corpus/** +# Generated eval reports +eval/results/ + # OS / editor .DS_Store .idea/ diff --git a/Makefile b/Makefile index a16551c..969d593 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: db-up db-down db-reset install fetch-corpus ingest eval test fmt +.PHONY: db-up db-down db-reset install fetch-corpus ingest eval eval-retrieval test fmt db-up: ## start Postgres+pgvector docker compose up -d @@ -27,6 +27,9 @@ ingest: ## build the index from ./data/corpus eval: ## run the evaluation harness PYTHONPATH=src python -m eval.evaluate +eval-retrieval: ## fast retrieval-only eval (no chat/judge calls) + PYTHONPATH=src python -m eval.evaluate --retrieval-only + test: ## run unit tests pytest diff --git a/README.md b/README.md index c1f9ba7..bab54da 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ make install make fetch-corpus # optional refresh; committed corpus works offline make ingest # upsert chunks from ./data/corpus make eval # 20 grounded questions +make eval-retrieval # fast: retrieval metrics only, no chat calls ``` Schema changes require a fresh volume: `make db-reset`. @@ -72,10 +73,18 @@ Postgres + pgvector (HNSW) + generated `tsvector` for hybrid retrieval, Ollama ## Results -| Configuration | Retrieval | Judge quality | -| ------------- | --------- | ------------- | -| hybrid + llama3.2, loose prompt | 100% (20/20) | 0.65 | -| hybrid + qwen2.5, grounded / low-temp prompt | 100% (20/20) | 0.67 | +Retrieval (`hybrid`, `nomic-embed-text`, top-5): + +| Hit rate@5 | Precision@1 | MRR | nDCG@5 | +| ---------- | ----------- | --- | ------ | +| 100% | 80% | 0.900 | 0.926 | + +Generation: + +| Configuration | Judge quality | +| ------------- | ------------- | +| llama3.2, loose prompt | 0.65 | +| qwen2.5:7b, grounded / low-temp prompt | 0.67 | Retrieval was already solid. Tightening the prompt and swapping chat models barely moved the LLM-as-judge score — which is a bit expected when the same @@ -83,6 +92,29 @@ local stack is grading itself. Answers *look* cleaner by eye; the judge just doesn’t capture that well. Eval also tracks citation precision / citation source hits now so we’re not leaning only on the vibe score. +### Eval metrics and artifacts + +The retrieval report is source-level: repeated chunks from one document count as +one ranked source. It prints: + +- **Hit rate@k** — whether any expected source appeared in the top-k. +- **Precision@1** — whether the first source was relevant. +- **MRR** — rewards putting the first relevant source near the top. +- **nDCG@k** — rewards relevant sources appearing higher in the ranking. + +Each run writes a full JSON report and a flat per-question CSV under +`eval/results/` (gitignored). The report includes the model, retrieval mode, +chunking settings, top-k, code revision, question-set hash, summary metrics, +and per-question rankings. + +```bash +make eval-retrieval # quick retrieval iteration +make eval # full generation + judge run +python -m eval.evaluate --k 10 # try another top-k +python -m eval.evaluate --no-artifacts # print only +python -m eval.evaluate --output-dir /tmp/eval-runs +``` + ## What I'd improve next - Metadata filters (category) at query time From 88a6fae78cee3291286c372b54669ce4b161d4f3 Mon Sep 17 00:00:00 2001 From: Zane Date: Wed, 29 Jul 2026 19:40:01 +1200 Subject: [PATCH 3/3] Add automated repository quality checks, enforce formatting, lint, and tests locally and on every PR into main --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++++++ Makefile | 7 ++++++- src/rag/embeddings.py | 4 +++- src/rag/generation.py | 6 +----- 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0c0bbee --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + quality: + name: Tests and lint + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install project + run: python -m pip install -e ".[dev]" + + - name: Run quality checks + run: make check diff --git a/Makefile b/Makefile index 969d593..e094619 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: db-up db-down db-reset install fetch-corpus ingest eval eval-retrieval test fmt +.PHONY: db-up db-down db-reset install fetch-corpus ingest eval eval-retrieval test fmt check db-up: ## start Postgres+pgvector docker compose up -d @@ -35,3 +35,8 @@ test: ## run unit tests fmt: ## format & lint ruff format . && ruff check --fix . + +check: ## run the same quality checks as CI + ruff format --check . + ruff check . + pytest diff --git a/src/rag/embeddings.py b/src/rag/embeddings.py index ea14230..3087cc2 100644 --- a/src/rag/embeddings.py +++ b/src/rag/embeddings.py @@ -42,7 +42,9 @@ def _embed_ollama(texts: list[str]) -> list[list[float]]: legacy.raise_for_status() emb = legacy.json().get("embedding") if not emb: - raise EmbeddingError(f"Unexpected Ollama embeddings response: {legacy.text}") + raise EmbeddingError( + f"Unexpected Ollama embeddings response: {legacy.text}" + ) embeddings.append(emb) return _validate_dim(embeddings) resp.raise_for_status() diff --git a/src/rag/generation.py b/src/rag/generation.py index 5025381..61504b8 100644 --- a/src/rag/generation.py +++ b/src/rag/generation.py @@ -156,11 +156,7 @@ def judge_answer(question: str, expected: str, actual: str) -> float: {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, { "role": "user", - "content": ( - f"QUESTION: {question}\n" - f"EXPECTED: {expected}\n" - f"ACTUAL: {actual}\n" - ), + "content": (f"QUESTION: {question}\nEXPECTED: {expected}\nACTUAL: {actual}\n"), }, ] raw = _ollama_chat(