From 239bdda4bcc23e6573af80152efa212bcffd9f5b Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Mon, 5 Oct 2026 21:30:44 +0200 Subject: [PATCH] Introduce category definitions for category metadata Goal 2 step 2, following the hybrid ownership model: each category gets a package under bcbench.categories, starting with a typed definition.py. - CategoryDefinition holds what was static metadata on EvaluationCategory: dataset file and entry type (requires_repo follows from it), bc-eval evaluators and core score, runner label, container requirement, and whether the agent may see the BC container credentials. - category_definition() maps a category to its definition with an exhaustive match; the CLI and commands use it, and the corresponding EvaluationCategory properties are removed. - Runtime code receives inputs instead of looking them up: the Copilot and Claude agents take pass_bc_credentials, the bc-eval export takes the loaded dataset entries, and the red-team target takes its entry. - CopilotModelName moves to bcbench.types so config.py no longer imports the CLI options module. Pipelines, result types, judge models and prompts still hang off EvaluationCategory and move in the next steps. CATEGORIES.md and the Copilot instructions describe the new layout. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 2 +- CATEGORIES.md | 20 ++- src/bcbench/agent/claude/agent.py | 3 +- src/bcbench/agent/copilot/agent.py | 3 +- src/bcbench/categories/__init__.py | 39 +++++ src/bcbench/categories/bug_fix/__init__.py | 0 src/bcbench/categories/bug_fix/definition.py | 14 ++ .../categories/code_review/__init__.py | 0 .../categories/code_review/definition.py | 14 ++ src/bcbench/categories/data_query/__init__.py | 0 .../categories/data_query/definition.py | 14 ++ src/bcbench/categories/definition.py | 41 +++++ .../ext_request_advisor/__init__.py | 0 .../ext_request_advisor/definition.py | 14 ++ .../ext_request_implement/__init__.py | 0 .../ext_request_implement/definition.py | 14 ++ .../categories/ext_request_triage/__init__.py | 0 .../ext_request_triage/definition.py | 14 ++ src/bcbench/categories/nl2al/__init__.py | 0 src/bcbench/categories/nl2al/definition.py | 14 ++ .../categories/test_generation/__init__.py | 0 .../categories/test_generation/definition.py | 14 ++ src/bcbench/cli_options.py | 16 +- src/bcbench/commands/category.py | 15 +- src/bcbench/commands/contamination.py | 3 +- src/bcbench/commands/dataset.py | 11 +- src/bcbench/commands/evaluate.py | 13 +- src/bcbench/commands/redteam.py | 3 + src/bcbench/commands/result.py | 3 +- src/bcbench/commands/run.py | 11 +- src/bcbench/config.py | 2 +- src/bcbench/redteam.py | 8 +- src/bcbench/results/bceval_export.py | 7 +- src/bcbench/types.py | 156 ++---------------- tests/test_bcquality_article_coverage.py | 4 +- tests/test_categories.py | 25 +++ tests/test_category_command.py | 11 +- tests/test_claude_code_agent.py | 1 + tests/test_cli_commands.py | 36 ++-- tests/test_codereview.py | 3 +- tests/test_codereview_fixtures.py | 4 +- tests/test_copilot_agent.py | 10 +- tests/test_custom_instructions.py | 3 +- tests/test_evaluate_pipeline.py | 3 +- tests/test_pr_review_metrics.py | 7 +- tests/test_redteam.py | 3 + tests/test_result_writer.py | 64 +++---- tests/test_setup_operations.py | 4 +- tests/test_type_exhaustiveness.py | 15 +- 49 files changed, 380 insertions(+), 281 deletions(-) create mode 100644 src/bcbench/categories/__init__.py create mode 100644 src/bcbench/categories/bug_fix/__init__.py create mode 100644 src/bcbench/categories/bug_fix/definition.py create mode 100644 src/bcbench/categories/code_review/__init__.py create mode 100644 src/bcbench/categories/code_review/definition.py create mode 100644 src/bcbench/categories/data_query/__init__.py create mode 100644 src/bcbench/categories/data_query/definition.py create mode 100644 src/bcbench/categories/definition.py create mode 100644 src/bcbench/categories/ext_request_advisor/__init__.py create mode 100644 src/bcbench/categories/ext_request_advisor/definition.py create mode 100644 src/bcbench/categories/ext_request_implement/__init__.py create mode 100644 src/bcbench/categories/ext_request_implement/definition.py create mode 100644 src/bcbench/categories/ext_request_triage/__init__.py create mode 100644 src/bcbench/categories/ext_request_triage/definition.py create mode 100644 src/bcbench/categories/nl2al/__init__.py create mode 100644 src/bcbench/categories/nl2al/definition.py create mode 100644 src/bcbench/categories/test_generation/__init__.py create mode 100644 src/bcbench/categories/test_generation/definition.py create mode 100644 tests/test_categories.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b9f4c3939..756024543 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -34,7 +34,7 @@ BC-Bench is category-based and designed to grow over time. It currently has two Preserve one-way dependency flow from orchestration toward lower-level abstractions. Keep domain and result models independent of runtime code. CLI commands are composition roots: they select concrete agents and inject them into evaluation pipelines through `AgentRunner`; pipelines must not select agent implementations. -`bcbench.types` is the central category registry. Extend `EvaluationCategory` for category-owned mappings such as datasets, pipelines, results, and scoring behavior instead of duplicating those decisions elsewhere. Keep imports following the existing direction and avoid circular dependencies. +Each evaluation category owns a package under `bcbench.categories` (a typed `CategoryDefinition` plus its category-specific code); code shared by several categories stays in the common modules. The CLI selects categories with `category_definition()`; runtime code receives what it needs explicitly instead of looking it up on `EvaluationCategory`. Keep imports following the existing direction and avoid circular dependencies. Logging setup lives in bcbench-core (`bcbench_core.logs.setup_logging`) so applications reuse it rather than copy it; the CLI calls it once, and modules only use `logging.getLogger(__name__)`. diff --git a/CATEGORIES.md b/CATEGORIES.md index 006a3af1d..7a7f111bc 100644 --- a/CATEGORIES.md +++ b/CATEGORIES.md @@ -2,7 +2,7 @@ BC-Bench is **category-based**. A category is a distinct evaluation scenario: `bug-fix` asks an agent to patch buggy code, `test-generation` asks it to write reproduction tests, `code-review` asks it to flag issues in a diff, `nl2al` asks it to turn a natural-language spec into AL code, `extensibility-request-advisor` asks it to classify and assess a fully specified extensibility need and draft a GitHub issue, `extensibility-request-implement` asks it to implement an approved extensibility request (e.g. add an integration event) in an existing repo, and `extensibility-request-triage` asks it to triage an extensibility request (emit managed labels, an advisory comment, and an open/closed decision). -Categories also differ in how they're scored and run. `bug-fix` and `test-generation` are execution-based: they build and run AL code, so they need a BC container. `code-review`, `nl2al`, `extensibility-request-advisor`, `extensibility-request-implement`, and `extensibility-request-triage` all leverage LLM-as-a-judge: `code-review` scores precision/recall/F1 of flagged issues against expected findings (an LLM judge only matches comments), and the other judge-based categories have an LLM grade the agent output against an LMChecklist. Advisor entries use an offline, single-shot proxy for the production skill's interactive flow: the prompt supplies the complete scenario, GitHub submission is disabled, and the agent persists classification, feasibility analysis, alternatives, and the final issue draft to `advisor_result.json`. The `EvaluationCategory` properties (`requires_container`, `runner`, `evaluators`, `core_score`) capture these differences for the workflows. +Categories also differ in how they're scored and run. `bug-fix` and `test-generation` are execution-based: they build and run AL code, so they need a BC container. `code-review`, `nl2al`, `extensibility-request-advisor`, `extensibility-request-implement`, and `extensibility-request-triage` all leverage LLM-as-a-judge: `code-review` scores precision/recall/F1 of flagged issues against expected findings (an LLM judge only matches comments), and the other judge-based categories have an LLM grade the agent output against an LMChecklist. Advisor entries use an offline, single-shot proxy for the production skill's interactive flow: the prompt supplies the complete scenario, GitHub submission is disabled, and the agent persists classification, feasibility analysis, alternatives, and the final issue draft to `advisor_result.json`. Each category's definition (`requires_container`, `runner`, `evaluators`, `core_score`) captures these differences for the workflows. Categories may share a dataset (`bug-fix` and `test-generation` do today), but a new category should generally have its own: dataset schema, entry type, result type, pipeline, etc. @@ -10,15 +10,21 @@ This doc is a map; the source files and their comments are the source of truth. ## Architecture -Start with `EvaluationCategory` in [src/bcbench/types.py](src/bcbench/types.py). It is the category registry. Each enum value maps to the pieces the rest of the CLI and workflows consume: +`EvaluationCategory` in [src/bcbench/types.py](src/bcbench/types.py) names the categories. Each category has a package under [src/bcbench/categories/](src/bcbench/categories/) that owns its behaviour; code shared by several categories (base classes, shared result families, shared datasets) stays in the common modules. The CLI looks up a category with `category_definition()` in [src/bcbench/categories/__init__.py](src/bcbench/categories/__init__.py). + +A category's `definition.py` declares ([src/bcbench/categories/definition.py](src/bcbench/categories/definition.py)): + +- `dataset_file` / `entry_type` — the dataset file for raw tasks and the typed Python model for one dataset row (aka one task). `requires_repo` follows from the entry type. +- `evaluators` / `core_score` — the bc-eval evaluator list and headline score, emitted to workflows by [src/bcbench/commands/category.py](src/bcbench/commands/category.py). +- `requires_container` / `runner` — whether the category needs a BC container, and which runner evaluates it. +- `pass_bc_credentials` — whether the agent may see the BC container credentials. + +Still mapped from `EvaluationCategory` while the redesign continues: -- `dataset_path` — the dataset file for raw tasks. -- `entry_class` — the typed Python model for one dataset row (aka one task). - `result_class` — the recorded outcome for one evaluated task. - `summary_class` / `aggregate_class` — the aggregate views used by result summaries and leaderboards. - `pipeline` — the category-specific setup, agent run, and evaluation behavior. -- `evaluators` / `core_score` — the bc-eval evaluator list and headline score, emitted to workflows by [src/bcbench/commands/category.py](src/bcbench/commands/category.py). -- `requires_container` / `requires_repo` / `runner` — whether the category needs a BC container, whether it needs the dataset repository cloned, and which runner evaluates it. +- `judge_model` — the pinned LLM judge for judge-scored categories. - Prompt template — the category-specific prompt in [src/bcbench/agent/shared/config.yaml](src/bcbench/agent/shared/config.yaml), loaded by [src/bcbench/agent/shared/prompt.py](src/bcbench/agent/shared/prompt.py). Keep dataset entry classes and result classes focused on typed data. Put category-specific behavior in the pipeline. @@ -27,7 +33,7 @@ Keep dataset entry classes and result classes focused on typed data. Put categor Use the existing implementations as examples: `bug-fix` and `test-generation` for execution-based categories, `code-review` and `nl2al` for judge-based ones. -1. Add the enum value and mappings in [src/bcbench/types.py](src/bcbench/types.py). +1. Add the enum value and remaining mappings in [src/bcbench/types.py](src/bcbench/types.py), and a category package with `definition.py` under [src/bcbench/categories/](src/bcbench/categories/), wired into `category_definition()`. 2. Add the category dataset JSONL and entry class in [src/bcbench/dataset/dataset_entry.py](src/bcbench/dataset/dataset_entry.py). 3. Add a result class under [src/bcbench/results/](src/bcbench/results/) and map it from `EvaluationCategory.result_class`. 4. Add a pipeline under [src/bcbench/evaluate/](src/bcbench/evaluate/). diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 7fa9a31de..dd0aec03a 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -34,6 +34,7 @@ def run_claude_code( category: EvaluationCategory, repo_path: Path, output_dir: Path, + pass_bc_credentials: bool, runtime: AgentRuntimeConfig | None = None, timeout: int = 60 * 60, ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: @@ -132,7 +133,7 @@ def run_claude_code( "MCP_TIMEOUT": "180000", "MCP_TOOL_TIMEOUT": "180000", }, - pass_bc_credentials=category.pass_on_bc_container_credentials, + pass_bc_credentials=pass_bc_credentials, ), timeout=timeout, check=True, diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index 8b9001ea9..71172b2b1 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -28,6 +28,7 @@ def run_copilot_agent( category: EvaluationCategory, repo_path: Path, output_dir: Path, + pass_bc_credentials: bool, runtime: AgentRuntimeConfig | None = None, timeout: int = 60 * 60, ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: @@ -93,7 +94,7 @@ def run_copilot_agent( model=model, work_dir=repo_path, timeout=timeout, - env=agent_subprocess_env(pass_bc_credentials=category.pass_on_bc_container_credentials), + env=agent_subprocess_env(pass_bc_credentials=pass_bc_credentials), options=options, ) logger.info(f"Copilot CLI run complete for: {entry.instance_id}") diff --git a/src/bcbench/categories/__init__.py b/src/bcbench/categories/__init__.py new file mode 100644 index 000000000..f62a93a89 --- /dev/null +++ b/src/bcbench/categories/__init__.py @@ -0,0 +1,39 @@ +"""Evaluation categories: each package under bcbench.categories owns one category's definition and behaviour.""" + +from typing import assert_never + +from bcbench.categories.bug_fix.definition import DEFINITION as BUG_FIX +from bcbench.categories.code_review.definition import DEFINITION as CODE_REVIEW +from bcbench.categories.data_query.definition import DEFINITION as DATA_QUERY +from bcbench.categories.definition import CategoryDefinition +from bcbench.categories.ext_request_advisor.definition import DEFINITION as EXT_REQUEST_ADVISOR +from bcbench.categories.ext_request_implement.definition import DEFINITION as EXT_REQUEST_IMPLEMENT +from bcbench.categories.ext_request_triage.definition import DEFINITION as EXT_REQUEST_TRIAGE +from bcbench.categories.nl2al.definition import DEFINITION as NL2AL +from bcbench.categories.test_generation.definition import DEFINITION as TEST_GENERATION +from bcbench.dataset import BaseDatasetEntry +from bcbench.types import EvaluationCategory + +__all__ = ["CategoryDefinition", "category_definition"] + + +def category_definition(category: EvaluationCategory) -> CategoryDefinition[BaseDatasetEntry]: + match category: + case EvaluationCategory.BUG_FIX: + return BUG_FIX + case EvaluationCategory.TEST_GENERATION: + return TEST_GENERATION + case EvaluationCategory.CODE_REVIEW: + return CODE_REVIEW + case EvaluationCategory.NL2AL: + return NL2AL + case EvaluationCategory.DATA_QUERY: + return DATA_QUERY + case EvaluationCategory.EXT_REQUEST_ADVISOR: + return EXT_REQUEST_ADVISOR + case EvaluationCategory.EXT_REQUEST_IMPLEMENT: + return EXT_REQUEST_IMPLEMENT + case EvaluationCategory.EXT_REQUEST_TRIAGE: + return EXT_REQUEST_TRIAGE + case _: + assert_never(category) diff --git a/src/bcbench/categories/bug_fix/__init__.py b/src/bcbench/categories/bug_fix/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/bug_fix/definition.py b/src/bcbench/categories/bug_fix/definition.py new file mode 100644 index 000000000..b05838ba6 --- /dev/null +++ b/src/bcbench/categories/bug_fix/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import BugFixEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.BUG_FIX, + dataset_file="bcbench.jsonl", + entry_type=BugFixEntry, + evaluators=("resolution_rate", "build_rate"), + core_score="ResolutionRate", + runner="GitHub-BCBench", + requires_container=True, + pass_bc_credentials=True, +) diff --git a/src/bcbench/categories/code_review/__init__.py b/src/bcbench/categories/code_review/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/code_review/definition.py b/src/bcbench/categories/code_review/definition.py new file mode 100644 index 000000000..0c399924e --- /dev/null +++ b/src/bcbench/categories/code_review/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import CodeReviewEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.CODE_REVIEW, + dataset_file="codereview.jsonl", + entry_type=CodeReviewEntry, + evaluators=("precision_score", "recall_score", "f1_score", "valid_review_output"), + core_score="F1Score", + runner="ubuntu-latest", + requires_container=False, + pass_bc_credentials=True, +) diff --git a/src/bcbench/categories/data_query/__init__.py b/src/bcbench/categories/data_query/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/data_query/definition.py b/src/bcbench/categories/data_query/definition.py new file mode 100644 index 000000000..79ba0b99b --- /dev/null +++ b/src/bcbench/categories/data_query/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import DataQueryEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.DATA_QUERY, + dataset_file="dataquery.jsonl", + entry_type=DataQueryEntry, + evaluators=("resolution_rate", "build_rate"), + core_score="ResolutionRate", + runner="GitHub-BCBench", + requires_container=True, + pass_bc_credentials=False, +) diff --git a/src/bcbench/categories/definition.py b/src/bcbench/categories/definition.py new file mode 100644 index 000000000..8a33f5031 --- /dev/null +++ b/src/bcbench/categories/definition.py @@ -0,0 +1,41 @@ +"""The contract every evaluation category fulfils.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Generic, Literal, TypeVar + +from bcbench.dataset import BaseDatasetEntry, RepoGroundedEntry +from bcbench.types import EvaluationCategory + +# Covariant so a definition for a specific entry type is usable wherever any definition is expected +E_co = TypeVar("E_co", bound=BaseDatasetEntry, covariant=True) + +# GitHub Actions runner labels; only categories that build BaseApp need the self-hosted runners +type Runner = Literal["GitHub-BCBench", "ubuntu-latest", "windows-latest"] + + +@dataclass(frozen=True) +class CategoryDefinition(Generic[E_co]): # noqa: UP046 - PEP 695 syntax cannot declare covariance explicitly + category: EvaluationCategory + dataset_file: str + entry_type: type[E_co] + # bc-eval evaluators (evaluator/scores.py) uploaded for this category, and the one reported as CoreScore + evaluators: tuple[str, ...] + core_score: str + runner: Runner + # Whether evaluating builds/runs AL code and therefore needs a BC container + requires_container: bool + # Development scenarios (e.g. bug-fix) get direct container access; production-like ones (e.g. data-query) must + # use the exposed access methods, such as BC MCP, so the agent never sees the container credentials + pass_bc_credentials: bool + + @property + def requires_repo(self) -> bool: + return issubclass(self.entry_type, RepoGroundedEntry) + + def dataset_path(self, dataset_dir: Path) -> Path: + return dataset_dir / self.dataset_file + + def load_entries(self, dataset_dir: Path, entry_id: str | None = None) -> Sequence[E_co]: + return self.entry_type.load(self.dataset_path(dataset_dir), entry_id=entry_id) diff --git a/src/bcbench/categories/ext_request_advisor/__init__.py b/src/bcbench/categories/ext_request_advisor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/ext_request_advisor/definition.py b/src/bcbench/categories/ext_request_advisor/definition.py new file mode 100644 index 000000000..35b0fa1b3 --- /dev/null +++ b/src/bcbench/categories/ext_request_advisor/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import ExtRequestAdvisorEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.EXT_REQUEST_ADVISOR, + dataset_file="extensibility_request_advisor.jsonl", + entry_type=ExtRequestAdvisorEntry, + evaluators=("lm_checklist",), + core_score="test_passed", + runner="ubuntu-latest", + requires_container=False, + pass_bc_credentials=True, +) diff --git a/src/bcbench/categories/ext_request_implement/__init__.py b/src/bcbench/categories/ext_request_implement/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/ext_request_implement/definition.py b/src/bcbench/categories/ext_request_implement/definition.py new file mode 100644 index 000000000..f5f6df9c8 --- /dev/null +++ b/src/bcbench/categories/ext_request_implement/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import ExtRequestImplementEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.EXT_REQUEST_IMPLEMENT, + dataset_file="extensibility_request_implement.jsonl", + entry_type=ExtRequestImplementEntry, + evaluators=("lm_checklist",), + core_score="test_passed", + runner="ubuntu-latest", + requires_container=False, + pass_bc_credentials=True, +) diff --git a/src/bcbench/categories/ext_request_triage/__init__.py b/src/bcbench/categories/ext_request_triage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/ext_request_triage/definition.py b/src/bcbench/categories/ext_request_triage/definition.py new file mode 100644 index 000000000..b17b37b26 --- /dev/null +++ b/src/bcbench/categories/ext_request_triage/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import ExtRequestTriageEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.EXT_REQUEST_TRIAGE, + dataset_file="extensibility_request_triage.jsonl", + entry_type=ExtRequestTriageEntry, + evaluators=("lm_checklist",), + core_score="test_passed", + runner="ubuntu-latest", + requires_container=False, + pass_bc_credentials=True, +) diff --git a/src/bcbench/categories/nl2al/__init__.py b/src/bcbench/categories/nl2al/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/nl2al/definition.py b/src/bcbench/categories/nl2al/definition.py new file mode 100644 index 000000000..bf84b034c --- /dev/null +++ b/src/bcbench/categories/nl2al/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import NL2ALEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.NL2AL, + dataset_file="nl2al.jsonl", + entry_type=NL2ALEntry, + evaluators=("lm_checklist",), + core_score="test_passed", + runner="windows-latest", + requires_container=False, + pass_bc_credentials=True, +) diff --git a/src/bcbench/categories/test_generation/__init__.py b/src/bcbench/categories/test_generation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/bcbench/categories/test_generation/definition.py b/src/bcbench/categories/test_generation/definition.py new file mode 100644 index 000000000..c97e6e74d --- /dev/null +++ b/src/bcbench/categories/test_generation/definition.py @@ -0,0 +1,14 @@ +from bcbench.categories.definition import CategoryDefinition +from bcbench.dataset import TestGenEntry +from bcbench.types import EvaluationCategory + +DEFINITION = CategoryDefinition( + category=EvaluationCategory.TEST_GENERATION, + dataset_file="bcbench.jsonl", + entry_type=TestGenEntry, + evaluators=("resolution_rate", "build_rate", "pre_patch_failed_rate", "post_patch_passed_rate"), + core_score="ResolutionRate", + runner="GitHub-BCBench", + requires_container=True, + pass_bc_credentials=True, +) diff --git a/src/bcbench/cli_options.py b/src/bcbench/cli_options.py index a20cc1b1d..33cf21511 100644 --- a/src/bcbench/cli_options.py +++ b/src/bcbench/cli_options.py @@ -6,7 +6,8 @@ import typer from bcbench_core.container import ContainerConfig -from bcbench.types import AgentRuntimeConfig, EvaluationCategory +from bcbench.categories import category_definition +from bcbench.types import AgentRuntimeConfig, CopilotModelName, EvaluationCategory # Type aliases for cleaner command signatures # Note: Defaults are provided in function signatures, not here @@ -109,22 +110,11 @@ def resolve_evaluation_runtime( al_lsp=al_lsp, bc_mcp=bc_mcp, ) - if category.requires_container and runtime is None: + if category_definition(category).requires_container and runtime is None: raise typer.BadParameter(f"The {category.value} category requires a container", param_hint="--container-name") return runtime -CopilotModelName = Literal[ - "claude-sonnet-5", - "claude-opus-5", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - "gpt-5.3-codex", - "mai-code-1.1-flash", - "gemini-3.7-flash", -] - CopilotModel = Annotated[CopilotModelName, typer.Option(help="Copilot model to use")] ClaudeCodeModel = Annotated[ diff --git a/src/bcbench/commands/category.py b/src/bcbench/commands/category.py index 2c2d31523..02965d0ca 100644 --- a/src/bcbench/commands/category.py +++ b/src/bcbench/commands/category.py @@ -2,6 +2,7 @@ import typer +from bcbench.categories import category_definition from bcbench.cli_options import EvaluationCategoryOption from bcbench.github_actions import write_step_outputs from bcbench.types import EvaluationCategory @@ -23,11 +24,12 @@ def bceval_config(category: EvaluationCategoryOption) -> None: The lines are appended to $GITHUB_OUTPUT so they become GitHub Actions step outputs. Outside of Actions nothing is written. """ + definition = category_definition(category) outputs: dict[str, str] = { - "evaluators": ",".join(category.evaluators), - "core_score": category.core_score, + "evaluators": ",".join(definition.evaluators), + "core_score": definition.core_score, } - if ("lm_checklist" in category.evaluators) and (category.judge_model is not None): + if ("lm_checklist" in definition.evaluators) and (category.judge_model is not None): outputs["judge_model"] = category.judge_model write_step_outputs(outputs) @@ -35,10 +37,11 @@ def bceval_config(category: EvaluationCategoryOption) -> None: @category_app.command("runtime-config") def runtime_config(category: EvaluationCategoryOption) -> None: """Emit the GitHub Actions runner label and environment requirements for a category.""" + definition = category_definition(category) write_step_outputs( { - "runner": category.runner, - "requires-container": str(category.requires_container).lower(), - "requires-repo": str(category.requires_repo).lower(), + "runner": definition.runner, + "requires-container": str(definition.requires_container).lower(), + "requires-repo": str(definition.requires_repo).lower(), } ) diff --git a/src/bcbench/commands/contamination.py b/src/bcbench/commands/contamination.py index b3d1d6a61..c2d891737 100644 --- a/src/bcbench/commands/contamination.py +++ b/src/bcbench/commands/contamination.py @@ -10,6 +10,7 @@ import typer from bcbench_core.filesystem import prepare_run_dir +from bcbench.categories.bug_fix.definition import DEFINITION as BUG_FIX from bcbench.cli_options import CopilotModel, EvaluationCategoryOption, OutputDir, RunId from bcbench.config import get_config from bcbench.contamination.filepath_identification import FilePathIdentificationResult @@ -35,7 +36,7 @@ def filepath_identification( if category is not EvaluationCategory.BUG_FIX: raise typer.BadParameter("filepath-identification currently supports only bug-fix category", param_hint="--category") - entry: BugFixEntry = BugFixEntry.load(category.dataset_path, entry_id=entry_id)[0] + entry: BugFixEntry = BUG_FIX.load_entries(_config.paths.dataset_dir, entry_id)[0] run_dir = prepare_run_dir(output_dir, run_id) run_filepath_identification(entry=entry, model=model, result_dir=run_dir) diff --git a/src/bcbench/commands/dataset.py b/src/bcbench/commands/dataset.py index 3f3092749..83d4929a6 100644 --- a/src/bcbench/commands/dataset.py +++ b/src/bcbench/commands/dataset.py @@ -6,7 +6,9 @@ import typer +from bcbench.categories import category_definition from bcbench.cli_options import EvaluationCategoryOption +from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, CodeReviewEntry, RepoGroundedEntry from bcbench.dataset.dataset_entry import NL2ALEntry, _BugFixTestGenBase from bcbench.github_actions import write_step_outputs @@ -25,8 +27,9 @@ def list_entries( test_run: Annotated[bool, typer.Option(help="Indicate this is a test run (with 2 entries)")] = False, ) -> None: """List dataset entry IDs.""" - entry_cls = category.entry_class - resolved_path = category.dataset_path + definition = category_definition(category) + entry_cls = definition.entry_type + resolved_path = definition.dataset_path(get_config().paths.dataset_dir) if modified_only: import subprocess @@ -73,7 +76,7 @@ def view_entry( from rich.panel import Panel from rich.table import Table - entry: BaseDatasetEntry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry: BaseDatasetEntry = category_definition(category).load_entries(get_config().paths.dataset_dir, entry_id)[0] console = Console() info_table = Table(show_header=False, box=None) @@ -176,7 +179,7 @@ def version( github_output: Annotated[str | None, typer.Option(help="Write the version to GITHUB_OUTPUT with this key name")] = None, ) -> None: """Print an entry's environment_setup_version (the BC sandbox version).""" - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(get_config().paths.dataset_dir, entry_id)[0] print(entry.environment_setup_version) if github_output: write_step_outputs({github_output: entry.environment_setup_version}) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 318f980f6..6bb9431dc 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -9,6 +9,7 @@ from bcbench_core.filesystem import prepare_run_dir from bcbench.agent import BCalBackendConfig, get_claude_version, get_pr_review_version, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent +from bcbench.categories import category_definition from bcbench.cli_options import ( ClaudeCodeModel, ContainerCompany, @@ -76,7 +77,7 @@ def evaluate_copilot( al_lsp=al_lsp, bc_mcp=bc_mcp, ) - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0] run_dir = prepare_run_dir(output_dir, run_id) logger.info(f"Running evaluation on entry {entry_id} with GitHub Copilot CLI") @@ -101,6 +102,7 @@ def evaluate_copilot( category=category, model=ctx.model, output_dir=ctx.result_dir, + pass_bc_credentials=category_definition(category).pass_bc_credentials, runtime=runtime, ), ) @@ -146,7 +148,7 @@ def evaluate_claude_code( al_lsp=al_lsp, bc_mcp=bc_mcp, ) - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0] run_dir = prepare_run_dir(output_dir, run_id) logger.info(f"Running evaluation on entry {entry_id} with Claude Code") @@ -171,6 +173,7 @@ def evaluate_claude_code( category=category, model=ctx.model, output_dir=ctx.result_dir, + pass_bc_credentials=category_definition(category).pass_bc_credentials, runtime=runtime, ), ) @@ -200,7 +203,7 @@ def evaluate_pr_review( To only generate review.json without scoring, use 'bcbench run pr-review' instead. """ category = EvaluationCategory.CODE_REVIEW - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0] run_dir = prepare_run_dir(output_dir, run_id) logger.info(f"Running evaluation on entry {entry_id} with the BC-ALAgents review engine") @@ -247,7 +250,7 @@ def evaluate_bcal( To only run the agent to generate AL code without building, use 'bcbench run bcal' instead. """ category = EvaluationCategory.NL2AL - entry: NL2ALEntry = cast(NL2ALEntry, category.entry_class.load(category.dataset_path, entry_id=entry_id)[0]) + entry: NL2ALEntry = cast(NL2ALEntry, category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0]) run_dir = prepare_run_dir(output_dir, run_id) backend_config = BCalBackendConfig( command=llm_command, @@ -314,7 +317,7 @@ def evaluate_mock( """ Evaluate mock agent on single dataset entry for testing purposes. """ - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0] run_dir = prepare_run_dir(output_dir, run_id) logger.info(f"Running evaluation on entry {entry_id} with mock agent") diff --git a/src/bcbench/commands/redteam.py b/src/bcbench/commands/redteam.py index fb5d1a7e2..c530abd30 100644 --- a/src/bcbench/commands/redteam.py +++ b/src/bcbench/commands/redteam.py @@ -65,6 +65,7 @@ def scan( uv run bcbench redteam scan --language en --risk-category code_vulnerability uv run bcbench redteam scan --language es --seeds dataset/redteam/attack_objectives.json --attack-strategy base64 """ + from bcbench.categories.nl2al.definition import DEFINITION as NL2AL from bcbench.redteam import build_bcal_target, run_scan # Upstream treats seeds and risk categories as alternative objective sources, so exactly one is required. @@ -75,7 +76,9 @@ def scan( output.parent.mkdir(parents=True, exist_ok=True) # Only support NL2AL for now, we will think about extensibility later. + # bcal always requires --page/--audience, but red teaming has no dataset entry of its own at the moment. scan_target = build_bcal_target( + entry=NL2AL.load_entries(_config.paths.dataset_dir)[0], package_cache_path=_config.paths.evaluation_results_path / "redteam" / ALPACKAGES_DIRNAME, export_base=output.parent / "bcal-exports", backend_config=BCalBackendConfig( diff --git a/src/bcbench/commands/result.py b/src/bcbench/commands/result.py index 499cae818..5786ac86c 100644 --- a/src/bcbench/commands/result.py +++ b/src/bcbench/commands/result.py @@ -7,6 +7,7 @@ import typer +from bcbench.categories import category_definition from bcbench.cli_options import EvaluationCategoryOption, OutputDir, RunId from bcbench.config import get_config from bcbench.dataset.dataset_entry import INSTANCE_ID_PATTERN @@ -73,7 +74,7 @@ def result_summarize( logger.error("No results found in the result files") raise typer.Exit(code=1) - write_bceval_results(results, run_dir, run_id, bceval_output, category, git_ref=git_ref) + write_bceval_results(results, run_dir, run_id, bceval_output, category_definition(category).load_entries(_config.paths.dataset_dir), git_ref=git_ref) summary = EvaluationResultSummary.from_results(results, run_id=run_id) diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index 09a671635..dbce4baea 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -6,6 +6,7 @@ import typer from bcbench.agent import BCalBackendConfig, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent +from bcbench.categories import category_definition from bcbench.cli_options import ( ClaudeCodeModel, ContainerCompany, @@ -70,7 +71,7 @@ def run_copilot( al_lsp=al_lsp, bc_mcp=bc_mcp, ) - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0] category.pipeline.setup_workspace(entry, repo_path) run_copilot_agent( @@ -79,6 +80,7 @@ def run_copilot( model=model, category=category, output_dir=output_dir, + pass_bc_credentials=category_definition(category).pass_bc_credentials, runtime=runtime, ) @@ -121,7 +123,7 @@ def run_claude( al_lsp=al_lsp, bc_mcp=bc_mcp, ) - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0] category.pipeline.setup_workspace(entry, repo_path) run_claude_code( @@ -130,6 +132,7 @@ def run_claude( model=model, category=category, output_dir=output_dir, + pass_bc_credentials=category_definition(category).pass_bc_credentials, runtime=runtime, ) @@ -157,7 +160,7 @@ def run_pr_review( uv run bcbench run pr-review synthetic__style-018 --repo-path /path/to/testbed """ category = EvaluationCategory.CODE_REVIEW - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + entry = category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0] category.pipeline.setup_workspace(entry, repo_path) run_pr_review_agent( @@ -187,7 +190,7 @@ def run_bcal( uv run bcbench run bcal nl2al__job-budget-report-1 """ category = EvaluationCategory.NL2AL - entry: NL2ALEntry = cast(NL2ALEntry, category.entry_class.load(category.dataset_path, entry_id=entry_id)[0]) + entry: NL2ALEntry = cast(NL2ALEntry, category_definition(category).load_entries(_config.paths.dataset_dir, entry_id)[0]) category.pipeline.setup_workspace(entry, repo_path) run_bcal_agent( diff --git a/src/bcbench/config.py b/src/bcbench/config.py index dcac660f9..45b4561c6 100644 --- a/src/bcbench/config.py +++ b/src/bcbench/config.py @@ -11,8 +11,8 @@ from dotenv import load_dotenv from pydantic import AliasPath, BaseModel, ConfigDict, Field -from bcbench.cli_options import CopilotModelName from bcbench.paths import SHARED_CONFIG_FILE +from bcbench.types import CopilotModelName __all__ = ["Config", "get_config"] diff --git a/src/bcbench/redteam.py b/src/bcbench/redteam.py index ef2266f76..1ffd72bae 100644 --- a/src/bcbench/redteam.py +++ b/src/bcbench/redteam.py @@ -13,7 +13,6 @@ from bcbench.agent.bcal import BCalBackendConfig, run_bcal_prompt from bcbench.dataset.dataset_entry import NL2ALEntry -from bcbench.types import EvaluationCategory logger = logging.getLogger(__name__) @@ -61,12 +60,9 @@ def _message_content(message: object) -> str: return content -def build_bcal_target(package_cache_path: Path, export_base: Path, backend_config: BCalBackendConfig) -> RedTeamCallback: +def build_bcal_target(entry: NL2ALEntry, package_cache_path: Path, export_base: Path, backend_config: BCalBackendConfig) -> RedTeamCallback: """Wrap the nl2al (BCal) as a red-team target callback.""" - category = EvaluationCategory.NL2AL - # bcal always requires --page/--audience, but red teaming has no dataset entry of its own at the moment. - entry = category.entry_class.load(category.dataset_path)[0] _ensure_package_cache(package_cache_path, entry.environment_setup_version) # Azure's simple string callback converts exceptions into assistant responses. Its full @@ -82,7 +78,7 @@ async def bcal_target( query = _message_content(messages[-1]) export_folder = export_base / f"query-{uuid.uuid4().hex[:8]}" - response = await asyncio.to_thread(run_bcal_prompt, cast(NL2ALEntry, entry), query, package_cache_path, export_folder, backend_config) + response = await asyncio.to_thread(run_bcal_prompt, entry, query, package_cache_path, export_folder, backend_config) return { "messages": [{"role": "assistant", "content": response}], "stream": stream, diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index a5e76f820..adf8c42d1 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -4,13 +4,14 @@ import json import logging +from collections.abc import Sequence from pathlib import Path from typing import Any from bcbench.dataset import BaseDatasetEntry from bcbench.results.base import BaseEvaluationResult from bcbench.results.summary import get_benchmark_version -from bcbench.types import EvaluationCategory, ExpectedOutput, ExperimentConfiguration +from bcbench.types import ExpectedOutput, ExperimentConfiguration logger = logging.getLogger(__name__) @@ -35,12 +36,10 @@ def write_bceval_results( out_dir: Path, run_id: str, output_filename: str, - category: EvaluationCategory, + dataset_entries: Sequence[BaseDatasetEntry], git_ref: str | None = None, ) -> None: """Write results into a JSONL file for bceval consumption.""" - entry_cls = category.entry_class - dataset_entries: list[BaseDatasetEntry] = entry_cls.load(category.dataset_path) benchmark_version = get_benchmark_version() output_file = out_dir / output_filename diff --git a/src/bcbench/types.py b/src/bcbench/types.py index a0bd71d1a..8acc26607 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -259,54 +259,6 @@ class EvaluationCategory(StrEnum): # Triage a single extensibility request: emit managed labels, an advisory comment, and open/closed state. EXT_REQUEST_TRIAGE = "extensibility-request-triage" - @property - def dataset_path(self) -> Path: - from bcbench.config import get_config - - match self: - case EvaluationCategory.BUG_FIX: - return get_config().paths.dataset_dir / "bcbench.jsonl" - case EvaluationCategory.TEST_GENERATION: - return get_config().paths.dataset_dir / "bcbench.jsonl" - case EvaluationCategory.CODE_REVIEW: - return get_config().paths.dataset_dir / "codereview.jsonl" - case EvaluationCategory.NL2AL: - return get_config().paths.dataset_dir / "nl2al.jsonl" - case EvaluationCategory.DATA_QUERY: - return get_config().paths.dataset_dir / "dataquery.jsonl" - case EvaluationCategory.EXT_REQUEST_ADVISOR: - return get_config().paths.dataset_dir / "extensibility_request_advisor.jsonl" - case EvaluationCategory.EXT_REQUEST_IMPLEMENT: - return get_config().paths.dataset_dir / "extensibility_request_implement.jsonl" - case EvaluationCategory.EXT_REQUEST_TRIAGE: - return get_config().paths.dataset_dir / "extensibility_request_triage.jsonl" - - raise ValueError(f"Unknown evaluation category: {self}") - - @property - def entry_class(self) -> type[BaseDatasetEntry]: - from bcbench.dataset import BugFixEntry, CodeReviewEntry, DataQueryEntry, ExtRequestAdvisorEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, NL2ALEntry, TestGenEntry - - match self: - case EvaluationCategory.BUG_FIX: - return BugFixEntry - case EvaluationCategory.TEST_GENERATION: - return TestGenEntry - case EvaluationCategory.CODE_REVIEW: - return CodeReviewEntry - case EvaluationCategory.NL2AL: - return NL2ALEntry - case EvaluationCategory.DATA_QUERY: - return DataQueryEntry - case EvaluationCategory.EXT_REQUEST_ADVISOR: - return ExtRequestAdvisorEntry - case EvaluationCategory.EXT_REQUEST_IMPLEMENT: - return ExtRequestImplementEntry - case EvaluationCategory.EXT_REQUEST_TRIAGE: - return ExtRequestTriageEntry - - raise ValueError(f"Unknown evaluation category: {self}") - @property def result_class(self) -> type[BaseEvaluationResult]: from bcbench.results.base import ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult @@ -434,102 +386,6 @@ def judge_model(self) -> str | None: raise ValueError(f"Unknown evaluation category: {self}") - @property - def evaluators(self) -> list[str]: - """ - Names of bc-eval evaluators (from evaluator/scores.py) to run for this category. - - Used for uploading evaluation results to long term storage. - """ - match self: - case EvaluationCategory.BUG_FIX: - return ["resolution_rate", "build_rate"] - case EvaluationCategory.TEST_GENERATION: - return ["resolution_rate", "build_rate", "pre_patch_failed_rate", "post_patch_passed_rate"] - case EvaluationCategory.CODE_REVIEW: - return ["precision_score", "recall_score", "f1_score", "valid_review_output"] - case EvaluationCategory.NL2AL: - return ["lm_checklist"] - case EvaluationCategory.DATA_QUERY: - return ["resolution_rate", "build_rate"] - case EvaluationCategory.EXT_REQUEST_ADVISOR: - return ["lm_checklist"] - case EvaluationCategory.EXT_REQUEST_IMPLEMENT: - return ["lm_checklist"] - case EvaluationCategory.EXT_REQUEST_TRIAGE: - return ["lm_checklist"] - - raise ValueError(f"Unknown evaluation category: {self}") - - @property - def core_score(self) -> str: - """Name of the evaluator whose value is considered as CoreScore, required by bc-eval.""" - match self: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: - return "ResolutionRate" - case EvaluationCategory.CODE_REVIEW: - return "F1Score" - case EvaluationCategory.NL2AL | EvaluationCategory.EXT_REQUEST_ADVISOR | EvaluationCategory.EXT_REQUEST_IMPLEMENT | EvaluationCategory.EXT_REQUEST_TRIAGE: - return "test_passed" - case EvaluationCategory.DATA_QUERY: - return "ResolutionRate" - - raise ValueError(f"Unknown evaluation category: {self}") - - @property - def requires_container(self) -> bool: - """Whether evaluating this category builds/runs AL code and therefore needs a BC container.""" - match self: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: - return True - case EvaluationCategory.CODE_REVIEW | EvaluationCategory.NL2AL | EvaluationCategory.EXT_REQUEST_ADVISOR | EvaluationCategory.EXT_REQUEST_IMPLEMENT | EvaluationCategory.EXT_REQUEST_TRIAGE: - return False - - raise ValueError(f"Unknown evaluation category: {self}") - - @property - def pass_on_bc_container_credentials(self) -> bool: - # Categories that simulate development scenarios (e.g. Bug Fix) should have direct container access - # While categories that simulate production scenarios (e.g. Data Query) should only use exposed access methods, e.g. BC MCP - match self: - case EvaluationCategory.DATA_QUERY: - return False - case ( - EvaluationCategory.BUG_FIX - | EvaluationCategory.TEST_GENERATION - | EvaluationCategory.CODE_REVIEW - | EvaluationCategory.NL2AL - | EvaluationCategory.EXT_REQUEST_ADVISOR - | EvaluationCategory.EXT_REQUEST_IMPLEMENT - | EvaluationCategory.EXT_REQUEST_TRIAGE - ): - return True - - raise ValueError(f"Unknown evaluation category: {self}") - - @property - def requires_repo(self) -> bool: - """Whether evaluating this category works on a cloned dataset repository.""" - from bcbench.dataset import RepoGroundedEntry - - return issubclass(self.entry_class, RepoGroundedEntry) - - @property - def runner(self) -> str: - """GitHub Actions runner label for evaluating this category. - - Only categories that require building BaseApp needs self-hosted runners. - """ - match self: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: - return "GitHub-BCBench" - case EvaluationCategory.CODE_REVIEW | EvaluationCategory.EXT_REQUEST_ADVISOR | EvaluationCategory.EXT_REQUEST_IMPLEMENT | EvaluationCategory.EXT_REQUEST_TRIAGE: - return "ubuntu-latest" - case EvaluationCategory.NL2AL: - return "windows-latest" - - raise ValueError(f"Unknown evaluation category: {self}") - type TestGenerationInput = Literal["problem-statement", "gold-patch", "both"] @@ -685,3 +541,15 @@ def get_container(self) -> ContainerConfig: if self.container is None: raise ValueError(f"Container configuration is required for {self.category.value} evaluation") return self.container + + +CopilotModelName = Literal[ + "claude-sonnet-5", + "claude-opus-5", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-5.3-codex", + "mai-code-1.1-flash", + "gemini-3.7-flash", +] diff --git a/tests/test_bcquality_article_coverage.py b/tests/test_bcquality_article_coverage.py index ff9eaea4e..14f2881e6 100644 --- a/tests/test_bcquality_article_coverage.py +++ b/tests/test_bcquality_article_coverage.py @@ -12,6 +12,8 @@ enumerate_inventory, resolve_bcquality_root, ) +from bcbench.categories import category_definition +from bcbench.config import get_config from bcbench.dataset import CodeReviewEntry, ReviewComment from bcbench.dataset.codereview import ArticleId, CodeReviewEntryMetadata from bcbench.types import EvaluationCategory @@ -181,7 +183,7 @@ class TestDatasetArticleSlugs: @pytest.mark.e2e def test_declared_slugs_exist_in_bcquality(self): bcquality_root = _require_bcquality_root() - entries = CodeReviewEntry.load(EvaluationCategory.CODE_REVIEW.dataset_path) + entries = CodeReviewEntry.load(category_definition(EvaluationCategory.CODE_REVIEW).dataset_path(get_config().paths.dataset_dir)) declared = collect_declared_articles(entries) report = build_coverage_report(entries, inventory=enumerate_inventory(bcquality_root)) unknown = {article: declared[article] for article in report.unknown_articles} diff --git a/tests/test_categories.py b/tests/test_categories.py new file mode 100644 index 000000000..8c4e8d723 --- /dev/null +++ b/tests/test_categories.py @@ -0,0 +1,25 @@ +from pathlib import Path + +import pytest + +from bcbench.categories import category_definition +from bcbench.dataset import RepoGroundedEntry +from bcbench.types import EvaluationCategory + + +@pytest.mark.parametrize("category", list(EvaluationCategory)) +def test_definition_belongs_to_its_category(category: EvaluationCategory): + assert category_definition(category).category is category + + +def test_dataset_path_joins_the_dataset_dir(): + definition = category_definition(EvaluationCategory.CODE_REVIEW) + + assert definition.dataset_path(Path("data")) == Path("data") / "codereview.jsonl" + + +@pytest.mark.parametrize("category", list(EvaluationCategory)) +def test_requires_repo_follows_the_entry_type(category: EvaluationCategory): + definition = category_definition(category) + + assert definition.requires_repo is issubclass(definition.entry_type, RepoGroundedEntry) diff --git a/tests/test_category_command.py b/tests/test_category_command.py index 09af5aa11..c07f987c7 100644 --- a/tests/test_category_command.py +++ b/tests/test_category_command.py @@ -1,5 +1,6 @@ from typer.testing import CliRunner +from bcbench.categories import category_definition from bcbench.cli import app from bcbench.types import EvaluationCategory @@ -67,8 +68,8 @@ def test_bceval_config_supports_every_category(tmp_path, monkeypatch): assert result.exit_code == 0, f"{category}: {result.stdout}" contents = output_file.read_text(encoding="utf-8") - assert f"evaluators={','.join(category.evaluators)}" in contents - assert f"core_score={category.core_score}" in contents + assert f"evaluators={','.join(category_definition(category).evaluators)}" in contents + assert f"core_score={category_definition(category).core_score}" in contents def test_list_prints_every_category_one_per_line(): @@ -90,9 +91,9 @@ def test_runtime_config_supports_every_category(tmp_path, monkeypatch): assert result.exit_code == 0, f"{category}: {result.stdout}" contents = output_file.read_text(encoding="utf-8") - assert f"runner={category.runner}" in contents - assert f"requires-container={str(category.requires_container).lower()}" in contents - assert f"requires-repo={str(category.requires_repo).lower()}" in contents + assert f"runner={category_definition(category).runner}" in contents + assert f"requires-container={str(category_definition(category).requires_container).lower()}" in contents + assert f"requires-repo={str(category_definition(category).requires_repo).lower()}" in contents def test_runtime_config_marks_code_review_as_containerless_on_hosted_runner(tmp_path, monkeypatch): diff --git a/tests/test_claude_code_agent.py b/tests/test_claude_code_agent.py index cb1095f16..281d93ea4 100644 --- a/tests/test_claude_code_agent.py +++ b/tests/test_claude_code_agent.py @@ -40,6 +40,7 @@ def test_claude_code_excludes_user_settings_and_auto_memory(tmp_path: Path, monk category=EvaluationCategory.BUG_FIX, repo_path=repo_path, output_dir=output_dir, + pass_bc_credentials=True, ) assert mock_run.call_args.args[0] == [ diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 82d1ed237..01c517fcd 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -12,6 +12,8 @@ from typer.main import get_command from typer.testing import CliRunner +from bcbench import cli_options +from bcbench.categories.definition import CategoryDefinition from bcbench.cli import _redteam_extra_installed, app from bcbench.cli_options import resolve_agent_runtime, resolve_evaluation_runtime from bcbench.commands import evaluate as evaluate_commands @@ -128,8 +130,7 @@ def execute(self, context, _agent_runner): captured["context"] = context with ( - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=tmp_path / "nl2al.jsonl"), - patch.object(EvaluationCategory, "entry_class", new_callable=PropertyMock, return_value=EntryClass), + patch.object(evaluate_commands, "category_definition", return_value=SimpleNamespace(load_entries=lambda _dir, entry_id: EntryClass.load(None, entry_id))), patch.object(EvaluationCategory, "pipeline", new_callable=PropertyMock, return_value=Pipeline()), ): evaluate_commands.evaluate_bcal( @@ -177,12 +178,13 @@ def setup_workspace(self, _entry, _repo_path): def execute(self, context, agent_runner): agent_runner(context) - return entry, SimpleNamespace( - dataset_path=tmp_path / "dataset.jsonl", - entry_class=EntryClass, - pipeline=Pipeline(), - requires_container=False, - ) + definition = SimpleNamespace(load_entries=lambda _dir, entry_id: EntryClass.load(None, entry_id), requires_container=False, pass_bc_credentials=True) + with ( + patch.object(run_commands, "category_definition", return_value=definition), + patch.object(evaluate_commands, "category_definition", return_value=definition), + patch.object(cli_options, "category_definition", return_value=definition), + ): + yield entry, SimpleNamespace(pipeline=Pipeline()) @pytest.mark.parametrize( @@ -290,7 +292,7 @@ def test_result_summarize_creates_all_outputs(sample_results_directory, problem_ with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=dataset_path), + patch.object(CategoryDefinition, "dataset_path", return_value=dataset_path), patch.object(evaluate_commands, "get_copilot_version", side_effect=AssertionError("Must use artifact version")), patch.object(evaluate_commands, "get_claude_version", side_effect=AssertionError("Must use artifact version")), patch.object(evaluate_commands, "get_pr_review_version", side_effect=AssertionError("Must use artifact version")), @@ -330,7 +332,7 @@ def test_result_summarize_verifies_summary_calculations(sample_results_directory with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=dataset_path), + patch.object(CategoryDefinition, "dataset_path", return_value=dataset_path), ): result = runner.invoke( app, @@ -412,7 +414,7 @@ def test_result_summarize_with_custom_pattern(sample_results_directory, problem_ with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=dataset_path), + patch.object(CategoryDefinition, "dataset_path", return_value=dataset_path), ): result = runner.invoke( app, @@ -435,7 +437,7 @@ def test_result_summarize_with_custom_pattern(sample_results_directory, problem_ @pytest.mark.integration def test_dataset_list_displays_all_entries(sample_dataset_file_for_cli): - with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): + with patch.object(CategoryDefinition, "dataset_path", return_value=sample_dataset_file_for_cli): result = runner.invoke( app, [ @@ -453,7 +455,7 @@ def test_dataset_list_displays_all_entries(sample_dataset_file_for_cli): @pytest.mark.integration def test_dataset_bc_version_returns_entry_version(sample_dataset_file_for_cli): - with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): + with patch.object(CategoryDefinition, "dataset_path", return_value=sample_dataset_file_for_cli): result = runner.invoke( app, [ @@ -474,7 +476,7 @@ def test_dataset_bc_version_returns_entry_version(sample_dataset_file_for_cli): def test_dataset_list_missing_file_fails_gracefully(tmp_path): nonexistent_path = tmp_path / "nonexistent.jsonl" - with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=nonexistent_path): + with patch.object(CategoryDefinition, "dataset_path", return_value=nonexistent_path): result = runner.invoke( app, [ @@ -491,7 +493,7 @@ def test_dataset_list_empty_file_shows_zero_entries(tmp_path): empty_dataset = tmp_path / "empty.jsonl" empty_dataset.write_text("") - with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=empty_dataset): + with patch.object(CategoryDefinition, "dataset_path", return_value=empty_dataset): result = runner.invoke( app, [ @@ -509,7 +511,7 @@ def test_dataset_list_single_entry(tmp_path): entry = create_dataset_entry(instance_id="microsoftInternal__NAV-100") dataset_path = create_dataset_file(tmp_path, [entry]) - with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=dataset_path): + with patch.object(CategoryDefinition, "dataset_path", return_value=dataset_path): result = runner.invoke( app, [ @@ -525,7 +527,7 @@ def test_dataset_list_single_entry(tmp_path): @pytest.mark.integration def test_dataset_list_verifies_entry_format(sample_dataset_file_for_cli): - with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): + with patch.object(CategoryDefinition, "dataset_path", return_value=sample_dataset_file_for_cli): result = runner.invoke( app, [ diff --git a/tests/test_codereview.py b/tests/test_codereview.py index 87adb9e74..c8dcee011 100644 --- a/tests/test_codereview.py +++ b/tests/test_codereview.py @@ -8,6 +8,7 @@ import pytest from bcbench_core.exceptions import AgentError +from bcbench.categories import category_definition from bcbench.config import get_config from bcbench.dataset import CodeReviewEntry from bcbench.dataset.codereview import ReviewComment, Severity @@ -919,7 +920,7 @@ def test_pipeline_instantiates(self): assert pipeline is not None def test_entry_class_is_codereview(self): - assert EvaluationCategory.CODE_REVIEW.entry_class == CodeReviewEntry + assert category_definition(EvaluationCategory.CODE_REVIEW).entry_type is CodeReviewEntry def test_context_does_not_require_container(self, tmp_path): entry = create_codereview_entry() diff --git a/tests/test_codereview_fixtures.py b/tests/test_codereview_fixtures.py index 1a2024145..3bbbddbdb 100644 --- a/tests/test_codereview_fixtures.py +++ b/tests/test_codereview_fixtures.py @@ -5,6 +5,8 @@ from bcbench_core.git import apply_patch, init_repo from unidiff import PatchSet +from bcbench.categories import category_definition +from bcbench.config import get_config from bcbench.dataset.codereview import CodeReviewEntry, Severity from bcbench.types import EvaluationCategory @@ -21,7 +23,7 @@ def _unique_object(pairs): @pytest.fixture(scope="module") def materialized_entries(tmp_path_factory): - dataset = EvaluationCategory.CODE_REVIEW.dataset_path + dataset = category_definition(EvaluationCategory.CODE_REVIEW).dataset_path(get_config().paths.dataset_dir) entries = {entry.instance_id: entry for entry in CodeReviewEntry.load(dataset)} raw_entries = {json.loads(line)["instance_id"]: line for line in dataset.read_text(encoding="utf-8").splitlines() if line.strip()} materialized = {} diff --git a/tests/test_copilot_agent.py b/tests/test_copilot_agent.py index 826ca785b..012fe4edc 100644 --- a/tests/test_copilot_agent.py +++ b/tests/test_copilot_agent.py @@ -40,6 +40,7 @@ def test_copilot_does_not_enable_hooks_memory_or_unrestricted_urls(tmp_path: Pat category=EvaluationCategory.BUG_FIX, repo_path=repo_path, output_dir=output_dir, + pass_bc_credentials=True, ) assert mock_run.call_args.args[0] == [ @@ -97,6 +98,7 @@ def test_copilot_session_options_preserve_plugin_order_and_explicit_directory_gr category=EvaluationCategory.BUG_FIX, repo_path=repo_path, output_dir=output_dir, + pass_bc_credentials=True, ) assert mock_run.call_args.args[0] == [ @@ -155,7 +157,7 @@ def test_copilot_timeout_adapts_core_metrics_and_preserves_experiment_metadata(c run.side_effect = subprocess.TimeoutExpired(["copilot"], timeout) with pytest.raises(AgentTimeoutError, match="Copilot CLI timed out") as error: - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, timeout=timeout) + run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, pass_bc_credentials=True, timeout=timeout) assert error.value.metrics is not None assert error.value.metrics.execution_time == timeout @@ -171,7 +173,7 @@ def test_copilot_process_errors_propagate_and_stop_gateway(configured_copilot_ru run.side_effect = failure with pytest.raises(CopilotProcessError) as error: - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir) + run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, pass_bc_credentials=True) assert error.value.__cause__ is failure gateway.stop.assert_called_once_with() @@ -181,7 +183,7 @@ def test_missing_copilot_stops_gateway_without_invoking_a_process(configured_cop repo_path, output_dir, gateway, run = configured_copilot_run with patch("bcbench_core.agent.copilot.agent._find_copilot", return_value=None), pytest.raises(CopilotProcessError, match="not found"): - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir) + run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, pass_bc_credentials=True) run.assert_not_called() gateway.stop.assert_called_once_with() @@ -190,6 +192,6 @@ def test_missing_copilot_stops_gateway_without_invoking_a_process(configured_cop def test_successful_copilot_run_stops_gateway(configured_copilot_run): repo_path, output_dir, gateway, _ = configured_copilot_run - run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir) + run_copilot_agent(create_dataset_entry(), "test-model", EvaluationCategory.BUG_FIX, repo_path, output_dir, pass_bc_credentials=True) gateway.stop.assert_called_once_with() diff --git a/tests/test_custom_instructions.py b/tests/test_custom_instructions.py index 716382bc5..4599f4431 100644 --- a/tests/test_custom_instructions.py +++ b/tests/test_custom_instructions.py @@ -9,6 +9,7 @@ import pytest +from bcbench.categories import category_definition from bcbench.dataset import BugFixEntry, RepoGroundedEntry from bcbench.operations.instruction_operations import ( INSTRUCTION_SOURCE_FILE, @@ -74,7 +75,7 @@ def test_repo_grounded_profile_is_derived_from_repo(): def test_every_category_entry_class_names_a_customization_profile(): for category in EvaluationCategory: - entry = category.entry_class.model_construct(repo="microsoft/BCApps") + entry = category_definition(category).entry_type.model_construct(repo="microsoft/BCApps") assert entry.customization_profile, f"{category} has no customization profile" diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 2b238173a..68b159ca9 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -8,6 +8,7 @@ import pytest from bcbench_core.agent.metrics import AgentMetrics +from bcbench.categories import category_definition from bcbench.commands.evaluate import MockEvaluationPipeline from bcbench.dataset import BaseDatasetEntry, BugFixEntry, NL2ALEntry from bcbench.evaluate.base import AgentRunner, EvaluationPipeline @@ -128,7 +129,7 @@ def _entry_for_category(category: EvaluationCategory) -> BaseDatasetEntry: @pytest.mark.parametrize("category", [EvaluationCategory.BUG_FIX, EvaluationCategory.TEST_GENERATION, EvaluationCategory.NL2AL, EvaluationCategory.EXT_REQUEST_IMPLEMENT]) def test_al_evaluation_pipelines_pass_recommended_diff_exclusions(tmp_path, category): - entry = category.entry_class.model_validate(_entry_for_category(category).model_dump()) + entry = category_definition(category).entry_type.model_validate(_entry_for_category(category).model_dump()) ctx = create_evaluation_context(tmp_path, entry=entry, category=category) pipeline = category.pipeline module = type(pipeline).__module__ diff --git a/tests/test_pr_review_metrics.py b/tests/test_pr_review_metrics.py index fda46c382..9cd35b388 100644 --- a/tests/test_pr_review_metrics.py +++ b/tests/test_pr_review_metrics.py @@ -1,14 +1,12 @@ import json from pathlib import Path -from unittest.mock import patch import pytest from bcbench_core.exceptions import AgentError from bcbench.agent.pr_review.metrics import RUN_METRICS_FILE_NAME, build_pr_review_metrics -from bcbench.dataset.codereview import CodeReviewEntry from bcbench.results.bceval_export import write_bceval_results -from bcbench.types import AgentHarness, EvaluationCategory, PRReviewMetrics +from bcbench.types import AgentHarness, PRReviewMetrics from tests.conftest import create_codereview_entry, create_codereview_result @@ -112,8 +110,7 @@ def test_valid_engine_metrics_without_billing_preserve_unknown_credits_through_e result.save(tmp_path, "raw-result.jsonl") assert json.loads((tmp_path / "raw-result.jsonl").read_text(encoding="utf-8"))["metrics"]["ai_credits"] is None - with patch.object(CodeReviewEntry, "load", return_value=[create_codereview_entry()]): - write_bceval_results([result], tmp_path, "run", "export.jsonl", EvaluationCategory.CODE_REVIEW) + write_bceval_results([result], tmp_path, "run", "export.jsonl", [create_codereview_entry()]) metadata = json.loads((tmp_path / "export.jsonl").read_text(encoding="utf-8"))["metadata"] assert metadata["ai_credits"] is None diff --git a/tests/test_redteam.py b/tests/test_redteam.py index e9397addc..a52e58897 100644 --- a/tests/test_redteam.py +++ b/tests/test_redteam.py @@ -16,6 +16,8 @@ from rich.console import Console from typer.main import get_command +from tests.conftest import create_nl2al_entry + # Red teaming ships as the optional `redteam` extra, so skip when it is not installed. pytest.importorskip("azure.ai.evaluation.red_team") @@ -29,6 +31,7 @@ def bcal_target(tmp_path: Path) -> redteam.RedTeamCallback: with patch.object(redteam, "_ensure_package_cache"): return redteam.build_bcal_target( + entry=create_nl2al_entry(), package_cache_path=tmp_path / ".alpackages", export_base=tmp_path / "exports", backend_config=BCalBackendConfig(command="python bridge.py"), diff --git a/tests/test_result_writer.py b/tests/test_result_writer.py index 70e501f97..9fddd9a45 100644 --- a/tests/test_result_writer.py +++ b/tests/test_result_writer.py @@ -1,13 +1,12 @@ import json -from unittest.mock import PropertyMock, patch +from unittest.mock import patch import pytest from bcbench_core.agent.metrics import AgentMetrics -from bcbench.dataset.codereview import CodeReviewEntry -from bcbench.dataset.dataset_entry import BugFixEntry, _BugFixTestGenBase +from bcbench.dataset.dataset_entry import BugFixEntry, TestGenEntry, _BugFixTestGenBase from bcbench.results.bceval_export import write_bceval_results -from bcbench.types import AgentHarness, EvaluationCategory, ExperimentConfiguration, PRReviewMetrics +from bcbench.types import AgentHarness, ExperimentConfiguration, PRReviewMetrics from tests.conftest import VALID_INSTANCE_ID, create_bugfix_result, create_codereview_entry, create_codereview_result @@ -27,14 +26,13 @@ class TestWriteBcevalResults: ) def test_pr_review_credits_preserve_missing_zero_and_observed_values(self, tmp_path, metrics): result = create_codereview_result(agent_name=AgentHarness.PR_REVIEW, metrics=metrics) - with patch.object(CodeReviewEntry, "load", return_value=[create_codereview_entry()]): - write_bceval_results( - results=[result], - out_dir=tmp_path, - run_id="run", - output_filename="results.jsonl", - category=EvaluationCategory.CODE_REVIEW, - ) + write_bceval_results( + results=[result], + out_dir=tmp_path, + run_id="run", + output_filename="results.jsonl", + dataset_entries=[create_codereview_entry()], + ) metadata = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metadata"] assert metadata["ai_credits"] == (metrics.ai_credits if metrics is not None else None) @@ -49,14 +47,13 @@ def test_writes_bceval_results_with_all_fields(self, tmp_path, sample_dataset_fi with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[sample_bugfix_result_with_metrics], out_dir=output_dir, run_id="test_run_123", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), ) output_file = output_dir / "results.jsonl" @@ -87,14 +84,13 @@ def test_handles_none_prompt_tokens(self, tmp_path, sample_dataset_file, sample_ with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[sample_testgen_result], out_dir=output_dir, run_id="test_run_456", output_filename="results.jsonl", - category=EvaluationCategory.TEST_GENERATION, + dataset_entries=TestGenEntry.load(sample_dataset_file), ) output_file = output_dir / "results.jsonl" @@ -112,14 +108,13 @@ def test_handles_mixed_results(self, tmp_path, sample_dataset_file, sample_bugfi with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[sample_bugfix_result_with_metrics, sample_testgen_result], out_dir=output_dir, run_id="test_run_789", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), ) output_file = output_dir / "results.jsonl" @@ -144,14 +139,13 @@ def test_includes_expected_fields_in_bceval_format(self, tmp_path, sample_datase with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[sample_bugfix_result_with_metrics], out_dir=output_dir, run_id="test_run_abc", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), ) output_file = output_dir / "results.jsonl" @@ -178,14 +172,13 @@ def test_skips_results_without_matching_dataset_entry(self, tmp_path, sample_dat metrics=AgentMetrics(prompt_tokens=1000, completion_tokens=200), ) - with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file): - write_bceval_results( - results=[non_matching_result], - out_dir=output_dir, - run_id="test_run_xyz", - output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, - ) + write_bceval_results( + results=[non_matching_result], + out_dir=output_dir, + run_id="test_run_xyz", + output_filename="results.jsonl", + dataset_entries=BugFixEntry.load(sample_dataset_file), + ) output_file = output_dir / "results.jsonl" with output_file.open() as f: @@ -202,14 +195,13 @@ def test_handles_partial_none_metrics(self, tmp_path, sample_dataset_file, probl with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[result], out_dir=output_dir, run_id="test_run_partial", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), ) output_file = output_dir / "results.jsonl" @@ -234,7 +226,6 @@ def test_preserves_dict_expected_output_for_lm_checklist_style_categories(self, with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), patch.object(BugFixEntry, "get_expected_output", lambda self: checklist_payload), ): write_bceval_results( @@ -242,7 +233,7 @@ def test_preserves_dict_expected_output_for_lm_checklist_style_categories(self, out_dir=output_dir, run_id="test_run_checklist", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), ) with (output_dir / "results.jsonl").open() as f: @@ -258,14 +249,13 @@ def test_marks_run_as_baseline_when_no_experiment(self, tmp_path, sample_dataset with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[result], out_dir=output_dir, run_id="run_baseline", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), git_ref="main", ) @@ -285,14 +275,13 @@ def test_marks_run_as_baseline_when_experiment_is_empty(self, tmp_path, sample_d with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[result], out_dir=output_dir, run_id="run_empty_experiment", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), ) with (output_dir / "results.jsonl").open() as f: @@ -310,14 +299,13 @@ def test_marks_run_as_experiment_with_config(self, tmp_path, sample_dataset_file with ( patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), - patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), ): write_bceval_results( results=[result], out_dir=output_dir, run_id="run_experiment", output_filename="results.jsonl", - category=EvaluationCategory.BUG_FIX, + dataset_entries=BugFixEntry.load(sample_dataset_file), git_ref="feat/al-mcp", ) diff --git a/tests/test_setup_operations.py b/tests/test_setup_operations.py index 43c679d64..3591377ba 100644 --- a/tests/test_setup_operations.py +++ b/tests/test_setup_operations.py @@ -6,6 +6,8 @@ import pytest +from bcbench.categories import category_definition +from bcbench.config import get_config from bcbench.dataset import BugFixEntry from bcbench.operations.setup_operations import bootstrap_app_json, set_runtime_version, setup_repo_prebuild from bcbench.types import EvaluationCategory @@ -71,7 +73,7 @@ def test_setup_repo_prebuild_commits_scope_compatibility_baseline(tmp_path: Path def test_benchmark_patches_do_not_depend_on_removed_scope_lines() -> None: - for entry in BugFixEntry.load(EvaluationCategory.BUG_FIX.dataset_path): + for entry in BugFixEntry.load(category_definition(EvaluationCategory.BUG_FIX).dataset_path(get_config().paths.dataset_dir)): for patch in (entry.patch, entry.test_patch): assert re.search(r"(?im)^[-+ ]\s*Scope\s*=\s*OnPrem;", patch) is None, entry.instance_id diff --git a/tests/test_type_exhaustiveness.py b/tests/test_type_exhaustiveness.py index bb761e92e..f42c71128 100644 --- a/tests/test_type_exhaustiveness.py +++ b/tests/test_type_exhaustiveness.py @@ -4,6 +4,7 @@ from bcbench_core.agent.metrics import AgentMetrics from pydantic import TypeAdapter +from bcbench.categories import category_definition from bcbench.dataset import BugFixEntry, CodeReviewEntry, DataQueryEntry, ExtRequestAdvisorEntry, ExtRequestImplementEntry, ExtRequestTriageEntry, NL2ALEntry from bcbench.dataset.codereview import ReviewComment, Severity from bcbench.types import AgentHarness, AgentMetricsContract, AnyAgentMetrics, EvaluationCategory, PRReviewMetrics @@ -70,7 +71,7 @@ def test_all_categories_have_pipelines(): def test_all_categories_have_entry_classes(): for category in EvaluationCategory: - entry_cls = category.entry_class + entry_cls = category_definition(category).entry_type assert entry_cls is not None @@ -100,7 +101,7 @@ def test_all_categories_handled_in_get_expected_output( sample_ext_triage_entry: ExtRequestTriageEntry, ): for category in EvaluationCategory: - entry_cls = category.entry_class + entry_cls = category_definition(category).entry_type if entry_cls == CodeReviewEntry: # CodeReviewEntry has a different schema — test separately entry = CodeReviewEntry( @@ -141,19 +142,19 @@ def test_all_categories_handled_in_get_expected_output( def test_all_categories_have_evaluators(): for category in EvaluationCategory: - evaluators = category.evaluators - assert isinstance(evaluators, list) + evaluators = category_definition(category).evaluators + assert isinstance(evaluators, tuple) assert evaluators, f"{category} must declare at least one evaluator" assert all(isinstance(e, str) and e for e in evaluators) def test_all_categories_have_core_score(): for category in EvaluationCategory: - assert isinstance(category.core_score, str) - assert category.core_score, f"{category} must declare a non-empty core_score" + assert isinstance(category_definition(category).core_score, str) + assert category_definition(category).core_score, f"{category} must declare a non-empty core_score" def test_pass_on_bc_container_credentials_exhaustiveness(): categories_with_credentials_withheld = {EvaluationCategory.DATA_QUERY} for category in EvaluationCategory: - assert category.pass_on_bc_container_credentials is (category not in categories_with_credentials_withheld) + assert category_definition(category).pass_bc_credentials is (category not in categories_with_credentials_withheld)