From f63788045546f9d4ece171fddc70352087d9a354 Mon Sep 17 00:00:00 2001 From: Vanya Cohen <9630206+vanyacohen@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:58:35 -0700 Subject: [PATCH] [Benchmark] Add MET-Bench entity tracking tasks --- docs/en/METBench.md | 42 ++++++++ docs/en/index.rst | 1 + requirements.txt | 1 + tests/test_metbench.py | 117 +++++++++++++++++++++ vlmeval/dataset/METBENCH_LICENSE | 21 ++++ vlmeval/dataset/__init__.py | 3 + vlmeval/dataset/metbench.py | 164 +++++++++++++++++++++++++++++ vlmeval/dataset/metbench_core.py | 171 +++++++++++++++++++++++++++++++ 8 files changed, 520 insertions(+) create mode 100644 docs/en/METBench.md create mode 100644 tests/test_metbench.py create mode 100644 vlmeval/dataset/METBENCH_LICENSE create mode 100644 vlmeval/dataset/metbench.py create mode 100644 vlmeval/dataset/metbench_core.py diff --git a/docs/en/METBench.md b/docs/en/METBench.md new file mode 100644 index 000000000..28bb684cf --- /dev/null +++ b/docs/en/METBench.md @@ -0,0 +1,42 @@ +# MET-Bench + +[Paper](https://arxiv.org/abs/2502.10886) · [Website](https://vanyacohen.com/MET-Bench/) · [Reference evaluator](https://github.com/vanyacohen/MET-Bench) + +MET-Bench evaluates entity tracking in Minecraft, Chess, and Shell Game through parallel text and image inputs. Chess and Shell Game track ten actions from an initial state; Minecraft predicts the next state after an action. + +## Datasets + +| Domain | Hugging Face dataset | Text task | Image task | +|---|---|---|---| +| Minecraft | [🤗 Minecraft](https://huggingface.co/datasets/vanyacohen/MET-Bench-Minecraft) | `METBench_minecraft_text` | `METBench_minecraft_image` | +| Chess | [🤗 Chess](https://huggingface.co/datasets/vanyacohen/MET-Bench-Chess) | `METBench_chess_text` | `METBench_chess_image` | +| Shell Game | [🤗 Shell Game](https://huggingface.co/datasets/vanyacohen/MET-Bench-Shell) | `METBench_shell_text` | `METBench_shell_image` | + +Each domain contains 500 unique test inputs. Dataset revisions are pinned, and the text and image tasks use the same ordered example IDs. Text tasks download `evaluation_text_only`; image tasks download `evaluation`. Chess and Shell Game are deduplicated by initial state and ten-action prefix. Minecraft is deduplicated by initial state, action, and ordered candidate states. + +## Run + +After installing VLMEvalKit, select a model that supports multiple interleaved images: + +```bash +python run.py \ + --model YOUR_CONFIGURED_MODEL \ + --data METBench_minecraft_text METBench_minecraft_image \ + METBench_chess_text METBench_chess_image \ + METBench_shell_text METBench_shell_image \ + --work-dir results/metbench +``` + +The dataset supplies the benchmark's chain-of-thought prompts. Set temperature to zero and the output limit to 4,096 tokens in the model configuration for the reference evaluation. Credentials are read from the environment. + +For a small check, select two examples: + +```bash +python run.py \ + --model YOUR_CONFIGURED_MODEL \ + --data METBench_minecraft_image \ + --data-config '{"METBench_minecraft_image":{"class":"METBenchImage","dataset":"METBench_minecraft_image","limit":2}}' \ + --work-dir results/metbench-check +``` + +Each task produces an `_acc.csv` file. `Overall` reports accuracy in percent: correct answer choices for Minecraft and Shell Game, and correctly predicted board squares for Chess. `ci_lower` and `ci_upper` give 95% confidence bounds: a normal approximation using the standard error across board scores for Chess, and Wilson intervals across answer choices for Minecraft and Shell Game. Chess bounds are unavailable for a single example; `examples` gives the evaluated example count. Prompts and answer parsing match the reference evaluator and the lmms-eval integration. diff --git a/docs/en/index.rst b/docs/en/index.rst index 425c7de4d..f4437fc2a 100644 --- a/docs/en/index.rst +++ b/docs/en/index.rst @@ -26,6 +26,7 @@ We always welcome users' PRs (Pull Requests) and Issues to improve VLMEvalKit! Development.md ConfigSystem.md + METBench.md .. _Other Notes: .. toctree:: diff --git a/requirements.txt b/requirements.txt index e1361623a..2fa0c6379 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ apted>=1.0.3 bert_score cairosvg cd-fvd +chess>=1.11.2,<2 colormath>=3.0.0 datasets decord>=0.6.0 diff --git a/tests/test_metbench.py b/tests/test_metbench.py new file mode 100644 index 000000000..ca7391186 --- /dev/null +++ b/tests/test_metbench.py @@ -0,0 +1,117 @@ +"""MET-Bench's native dataset, prompt ordering, and result-file contracts.""" + +import io + +import pandas as pd +import pytest +from datasets import Dataset +from PIL import Image + +from vlmeval.dataset import SUPPORTED_DATASETS, build_dataset +from vlmeval.dataset.metbench import EVALUATION_RELEASES, METBenchImage, METBenchText +from vlmeval.smp import dump +from vlmeval.smp.file import INFER_FAIL_MSG + +FEN = 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1' + + +def image(index): + """Encode distinguishable frames to detect reordering and action-frame leakage.""" + buffer = io.BytesIO() + Image.new('RGB', (2, 2), (index, 0, 0)).save(buffer, format='PNG') + return {'bytes': buffer.getvalue(), 'path': None} + + +def rows(domain, modality): + """Make a full-size synthetic split with sparse source indices.""" + if domain == 'minecraft': + row = dict(initial_state='{"x":0}', action='Walk forward 1 block.', + candidate_states=['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}'], correct_choice=2) + if modality == 'image': + row.update(image_initial_state=image(0), image_action=image(99), + image_candidate_states=[image(i) for i in range(1, 5)]) + else: + row = dict(initial_state=FEN if domain == 'chess' else 1, + final_state=FEN if domain == 'chess' else 2, + actions=['g1f3' if domain == 'chess' else '1 swap 2'] * 10) + if modality == 'image': + row['image_actions'] = [image(i) for i in range(10)] + return Dataset.from_list([dict(row, example_id=f'{domain}-test-{3 * i}') for i in range(500)]) + + +@pytest.mark.parametrize('domain', ['minecraft', 'chess', 'shell']) +@pytest.mark.parametrize('modality', ['text', 'image']) +def test_native_dataset_contract(monkeypatch, tmp_path, domain, modality): + """Check discovery, pinned downloads, prompts, and scoring through native APIs.""" + calls = [] + + def load(repo, config, **kwargs): + calls.append((repo, config, kwargs)) + return rows(domain, modality) + + monkeypatch.setattr('vlmeval.dataset.metbench.load_dataset', load) + monkeypatch.setattr('vlmeval.dataset.metbench.LMUDataRoot', lambda: str(tmp_path)) + name = f'METBench_{domain}_{modality}' + assert name in SUPPORTED_DATASETS + registered = build_dataset(name) + assert len(registered.data) == 500 + cls = METBenchText if modality == 'text' else METBenchImage + task = cls(name, limit=2) + repo, revision = EVALUATION_RELEASES[domain] + assert calls[-1] == (repo, 'evaluation_text_only' if modality == 'text' else 'evaluation', + {'split': 'test', 'revision': revision}) + assert task.data['index'].tolist() == [0, 3] + prompt = task.build_prompt(task.data.iloc[1]) + images = [part['value'] for part in prompt if part['type'] == 'image'] + expected = list(range(5 if domain == 'minecraft' else 10)) if modality == 'image' else [] + assert [Image.open(path).getpixel((0, 0))[0] for path in images] == expected + assert task.dump_image(task.data.iloc[1]) == images + if modality == 'text': + assert len(prompt) == 1 + original_text = [part['value'] for part in prompt if part['type'] == 'text'] + task.by_index['3']['target'] = 'PRIVATE_TARGET' + assert [p['value'] for p in task.build_prompt(task.data.iloc[1]) if p['type'] == 'text'] == original_text + task.by_index['3']['target'] = task.examples[0]['target'] + path = str(tmp_path / 'predictions.xlsx') + predictions = task.data.copy() + predictions['prediction'] = [f'FINAL ANSWER: {task.examples[0]["target"]}', 'unparseable'] + dump(predictions.iloc[::-1], path) + result = task.evaluate(path).iloc[0] + assert result['Overall'] == 50.0 + assert result['ci_lower'] < 50 < result['ci_upper'] + assert result['examples'] == 2 + predictions['prediction'] = [None, 'unparseable'] + dump(predictions, path) + assert task.evaluate(path).iloc[0]['Overall'] == 0 + predictions['prediction'] = INFER_FAIL_MSG + dump(predictions, path) + with pytest.raises(ValueError, match='failed inference'): + task.evaluate(path) + dump(pd.concat([predictions.iloc[:1]] * 2), path) + with pytest.raises(ValueError, match='duplicate'): + task.evaluate(path) + dump(predictions.iloc[:1], path) + with pytest.raises(ValueError, match='selected examples'): + task.evaluate(path) + + +def test_chess_confidence_uses_whole_trials(monkeypatch, tmp_path): + """Correlated squares cannot inflate the effective example count.""" + import math + + monkeypatch.setattr('vlmeval.dataset.metbench.load_dataset', lambda *a, **k: rows('chess', 'text')) + monkeypatch.setattr('vlmeval.dataset.metbench.LMUDataRoot', lambda: str(tmp_path)) + task = METBenchText('METBench_chess_text', limit=100) + predictions = task.data.copy() + predictions['prediction'] = [f'FINAL ANSWER: {FEN}', 'unparseable'] * 50 + path = str(tmp_path / 'chess.xlsx') + dump(predictions, path) + result = task.evaluate(path).iloc[0] + margin = 100 * 1.959963984540054 * math.sqrt(0.25 / 99) + assert result['Overall'] == 50.0 + assert result['ci_lower'] == pytest.approx(50 - margin) + assert result['ci_upper'] == pytest.approx(50 + margin) + task = METBenchText('METBench_chess_text', limit=1) + dump(predictions.iloc[:1], path) + result = task.evaluate(path).iloc[0] + assert pd.isna(result['ci_lower']) and pd.isna(result['ci_upper']) diff --git a/vlmeval/dataset/METBENCH_LICENSE b/vlmeval/dataset/METBENCH_LICENSE new file mode 100644 index 000000000..c9b04200a --- /dev/null +++ b/vlmeval/dataset/METBENCH_LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 vanyacohen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vlmeval/dataset/__init__.py b/vlmeval/dataset/__init__.py index 97fb67a53..4d8cd74c2 100644 --- a/vlmeval/dataset/__init__.py +++ b/vlmeval/dataset/__init__.py @@ -79,6 +79,7 @@ # Add by EASI team from .megabench import MEGABench from .memlens import MemLens +from .metbench import METBenchImage, METBenchText from .miabench import MIABench from .mindcubebench import MindCubeBench from .mlvu import MLVU, MLVU_MCQ, MLVU_OpenEnded @@ -294,6 +295,7 @@ def evaluate(self, eval_file, **judge_kwargs): # Add new supported dataset class here IMAGE_DATASET = [ + METBenchImage, ImageCaptionDataset, ImageYORNDataset, ImageMCQDataset, ImageVQADataset, MathVision, LENS, MMMUDataset, OCRBench, MathVista, LLaVABench, LLaVABench_KO, VGRPBench, MMVet, # noqa: E501 MTVQADataset, TableVQABench, MMLongBench, MemLens, MMLongBenchDoc, VCRDataset, MMDUDataset, DUDE, DocScope, @@ -350,6 +352,7 @@ def evaluate(self, eval_file, **judge_kwargs): VIDEO_DATASET += [SiteBenchVideo, VsiBench, VsiSuperRecall, VsiSuperCount, MMSIVideoBench, STIBench, DSRBench] # noqa: E501 TEXT_DATASET = [ + METBenchText, TextMCQDataset, SGI_Bench_Wet_Experiment, SGI_Bench_Dry_Experiment, SGI_Bench_Deep_Research, SGI_Bench_Idea_Generation, XSTestDataset, FlamesDataset, MedXpertQAText diff --git a/vlmeval/dataset/metbench.py b/vlmeval/dataset/metbench.py new file mode 100644 index 000000000..6f420dc60 --- /dev/null +++ b/vlmeval/dataset/metbench.py @@ -0,0 +1,164 @@ +"""MET-Bench datasets for VLMEvalKit.""" + +import hashlib +import io +import math +from pathlib import Path + +import pandas as pd +from datasets import Image as DatasetImage +from datasets import List, load_dataset + +from vlmeval.dataset.image_base import ImageBaseDataset +from vlmeval.dataset.metbench_core import build_messages, clustered_ratio_interval, score +from vlmeval.smp import LMUDataRoot, dump, load +from vlmeval.smp.file import INFER_FAIL_MSG + +EVALUATION_RELEASES = { + "minecraft": ( + "vanyacohen/MET-Bench-Minecraft", + "ae0b474dd22986b8b292c2a42f5e81cf8675661e", + ), + "chess": ("vanyacohen/MET-Bench-Chess", "6fd525cd537a9c25efe64ae88102c0ab68f94b1d"), + "shell": ("vanyacohen/MET-Bench-Shell", "75725717b63cf6e54a5b8abd22e6ad96b47a12e3"), +} + + +class METBenchImage(ImageBaseDataset): + """Evaluate paired MET-Bench tasks with the released prompts and metrics.""" + + TYPE = "VQA" + MODALITY = "IMAGE" + modality = "image" + force_use_dataset_prompt = True + + @classmethod + def supported_datasets(cls): + return [ + f"METBench_{domain}_{cls.modality}" + for domain in ("minecraft", "chess", "shell") + ] + + def __init__(self, dataset="METBench_minecraft_image", limit=500): + if dataset not in self.supported_datasets(): + raise ValueError(f"Unsupported MET-Bench task: {dataset}") + if not isinstance(limit, int) or not 1 <= limit <= 500: + raise ValueError("limit must be between 1 and 500") + self.dataset_name = dataset + self.domain = dataset.split("_")[1] + self.img_root = str(Path(LMUDataRoot()) / "images" / "METBench") + self.meta_only = True + self.skip_noimg = False + repo, revision = EVALUATION_RELEASES[self.domain] + config = "evaluation_text_only" if self.modality == "text" else "evaluation" + data = load_dataset(repo, config, split="test", revision=revision) + if len(data) != 500: + raise ValueError("Expected the released 500-example evaluation split") + if self.modality == "image": + if self.domain == "minecraft": + columns = { + "image_initial_state": DatasetImage(decode=False), + "image_action": DatasetImage(decode=False), + "image_candidate_states": List(DatasetImage(decode=False)), + } + else: + columns = {"image_actions": List(DatasetImage(decode=False))} + for name, feature in columns.items(): + data = data.cast_column(name, feature) + self.examples = [] + for row in data.select(range(limit)): + row["metbench_domain"] = self.domain + row["target"] = row["correct_choice" if self.domain == "minecraft" else "final_state"] + row["source_row"] = int(row["example_id"].rsplit("-", 1)[1]) + self.examples.append(row) + self.by_index = {str(row["source_row"]): row for row in self.examples} + self.data = pd.DataFrame( + { + "index": row["source_row"], + "question": f"MET-Bench {self.domain} ({self.modality})", + "answer": str(row["target"]), + "example_id": row["example_id"], + } + for row in self.examples + ) + + def build_prompt(self, line): + """Keep every image in its original position in the user message.""" + if isinstance(line, int): + line = self.data.iloc[line] + row = self.by_index[str(line["index"])] + content = [] + for part in build_messages(row, self.modality)[0]["content"]: + if part["type"] == "text": + content.append({"type": "text", "value": part["text"]}) + else: + buffer = io.BytesIO() + part["url"].save(buffer, format="PNG") + data = buffer.getvalue() + path = Path(self.img_root) / (hashlib.sha256(data).hexdigest() + ".png") + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists(): + path.write_bytes(data) + content.append({"type": "image", "value": str(path.resolve())}) + return content + + def dump_image(self, line): + """Return the ordered image paths for framework inspection tools.""" + return [p["value"] for p in self.build_prompt(line) if p["type"] == "image"] + + def evaluate(self, eval_file, **judge_kwargs): + """Score completed model predictions with MET-Bench's domain scorer.""" + predictions = load(eval_file) + if predictions["index"].duplicated().any(): + raise ValueError("Prediction file contains duplicate example indices") + if set(predictions["index"].astype(str)) != set(self.by_index): + raise ValueError("Prediction file does not match the selected examples") + if ( + predictions["prediction"] + .astype(str) + .str.contains(INFER_FAIL_MSG, regex=False) + .any() + ): + raise ValueError("Retry failed inference requests before scoring MET-Bench") + scores = [] + for _, prediction in predictions.iterrows(): + response = prediction["prediction"] + if pd.isna(response): + response = "" + scores.append( + score(self.by_index[str(prediction["index"])], str(response)) + ) + accuracy = sum(scores) / len(scores) + if self.domain == "chess": + lower, upper = clustered_ratio_interval([(value, 1) for value in scores]) + else: + trials = len(scores) + z = 1.959963984540054 + denominator = 1 + z * z / trials + center = (accuracy + z * z / (2 * trials)) / denominator + margin = z * math.sqrt(accuracy * (1 - accuracy) / trials + z * z / (4 * trials * trials)) / denominator + lower, upper = max(0.0, center - margin), min(1.0, center + margin) + result = pd.DataFrame( + [ + { + "domain": self.domain, + "modality": self.modality, + "Overall": 100 * accuracy, + "ci_lower": 100 * lower if lower is not None else None, + "ci_upper": 100 * upper if upper is not None else None, + "examples": len(predictions), + } + ] + ) + dump(result, str(Path(eval_file).with_suffix("")) + "_acc.csv") + return result + + +class METBenchText(METBenchImage): + """The text modality of the same MET-Bench examples.""" + + MODALITY = "TEXT" + modality = "text" + + def __init__(self, dataset="METBench_minecraft_text", limit=500): + super().__init__(dataset, limit=limit) diff --git a/vlmeval/dataset/metbench_core.py b/vlmeval/dataset/metbench_core.py new file mode 100644 index 000000000..00bdb1416 --- /dev/null +++ b/vlmeval/dataset/metbench_core.py @@ -0,0 +1,171 @@ +"""MET-Bench prompts and scoring, shared by the evaluation integrations. + +Adapted from https://github.com/vanyacohen/MET-Bench (MIT License). +""" + +import io +import math +import re +from typing import Any + +from PIL import Image + + +def _image(value: Any) -> Image.Image: + """Accept decoded dataset images or their Arrow bytes/path representation.""" + if isinstance(value, Image.Image): + return value.convert("RGB") + source = io.BytesIO(value["bytes"]) if value.get("bytes") is not None else value["path"] + with Image.open(source) as image: + return image.convert("RGB") + + +def _sequence_content(doc: dict[str, Any], modality: str) -> list[dict[str, Any]]: + domain = doc["metbench_domain"] + if domain == "chess": + instructions = "You are a helpful assistant that tracks chess moves in a game and produces the final FEN.\n" + question = "What is the final FEN? Think step by step then output the final FEN as FINAL ANSWER: [FEN HERE]." + caption = "The move is from the green square to the red square." + else: + instructions = ( + "The shell game is a classic game where a ball is hidden under one of three shells. " + "You are a helpful assistant that tracks the position of the ball during the swaps and" + " determines the final position of the ball. We label the shells 1, 2, 3. The ball " + "starts under one of the numbered shells which we call the initial state, and each " + "move is shell swap of shell x and y written 'x swap y'.\n" + ) + question = ( + "What is the final position of the ball? Think step by step then output the final ball" + " location as FINAL ANSWER: [1, 2, or 3]." + ) + caption = "The shells with the numbers highlighted green are being swapped." + text = f"{instructions}\n{question}\nThe initial state is: {doc['initial_state']}\nHere are the moves played:\n" + content = [] + for index, action in enumerate(doc["actions"]): + if modality == "image": + content.append({"type": "text", "text": text}) + content.append({"type": "image", "url": _image(doc["image_actions"][index])}) + text = caption + "\n" + else: + text += f"{action}\n" + content.append({"type": "text", "text": text + f"\n{question}"}) + return content + + +def _minecraft_content(doc: dict[str, Any], modality: str) -> list[dict[str, Any]]: + action = doc["action"] + count = len(doc["candidate_states"]) + if modality == "text": + instructions = ( + "You are evaluating a Minecraft gameplay trajectory. You will see a JSON game-state " + "snapshot and an action that was performed. Your task is to identify which of the " + "candidate game states is the correct next state after the action was taken." + ) + question = ( + f'Which game state (1-{count}) is the correct next state after the action "{action}"? ' + f'Think step by step about how the game state should change given the action, then ' + f'output your answer as FINAL ANSWER: [1-{count}].' + ) + text = ( + f"{instructions}\n\nAction performed: {action}\n\nInput " + f"state:\n{doc['initial_state']}\n\nCandidate next states:" + ) + for index, state in enumerate(doc["candidate_states"], start=1): + text += f"\n\nChoice {index}:\n{state}" + return [{"type": "text", "text": text + f"\n\n{question}"}] + instructions = ( + "You are evaluating a Minecraft gameplay trajectory. You will see a first-person " + "screenshot from the game and an action that was performed. Your task is to identify " + "which of the candidate images shows the correct next frame after the action was " + "taken." + ) + question = ( + f'Which image (1-{count}) shows the correct next frame after the action "{action}"? ' + f'Think step by step about how the scene should change given the action, then output ' + f'your answer as FINAL ANSWER: [1-{count}].' + ) + content = [ + {"type": "text", "text": f"{instructions}\n\nAction performed: {action}\n\nInput frame:"}, + {"type": "image", "url": _image(doc["image_initial_state"])}, + {"type": "text", "text": "\nCandidate next frames:"}, + ] + for index, image in enumerate(doc["image_candidate_states"], start=1): + content.extend([{"type": "text", "text": f"\nChoice {index}:"}, {"type": "image", "url": _image(image)}]) + content.append({"type": "text", "text": f"\n{question}"}) + return content + + +def _parse_choice(response: str, maximum: int) -> int | None: + if not response: + return None + text = str(response).strip() + markers = list(re.finditer(r"final\s+answer\s*:\s*", text, re.IGNORECASE)) + if markers: + text = text[markers[-1].end():].strip() + text = text.splitlines()[0] if text else "" + text = text.replace(r"\boxed", "") + text = re.sub(r"[`*{}\[\]]", "", text).strip() + match = re.fullmatch( + r"(?:(?:choice|candidate|shell)\s*#?\s*)?([1-" + str(maximum) + r"])[.!]?", text, re.IGNORECASE) + return int(match.group(1)) if match else None + + +def _chess_accuracy(target: str, response: str) -> float: + # Task discovery does not need the Chess scoring dependency. + import chess + + truth = chess.Board(fen=str(target).strip().split("\n")[0].replace("`", "").replace("*", "")) + for line in reversed(str(response).splitlines()): + if "final answer" not in line.lower(): + continue + for token in ("\\boxed{", "\\boxed"): + line = line.replace(token, " ") + for char in "`*[](){}'\",;:.": + line = line.replace(char, " ") + tokens = line.split() + candidates = [] + for index in range(len(tokens) - 5): + try: + candidates.append(chess.Board(fen=" ".join(tokens[index:index + 6]))) + except ValueError: + continue + if len(candidates) == 1: + predicted = candidates[0] + return sum(truth.piece_at(square) == predicted.piece_at(square) for square in range(64)) / 64 + return 0.0 + + +def build_messages(doc: dict[str, Any], modality: str) -> list[dict[str, Any]]: + """Build one user turn, preserving the order of text and image inputs.""" + if modality not in {"text", "image"}: + raise ValueError(f"Unsupported modality: {modality}") + builder = _minecraft_content if doc["metbench_domain"] == "minecraft" else _sequence_content + return [{"role": "user", "content": builder(doc, modality)}] + + +def score(doc: dict[str, Any], response: str) -> float: + """Return board-square accuracy for Chess or answer accuracy otherwise.""" + if doc["metbench_domain"] == "chess": + return _chess_accuracy(doc["target"], response) + maximum = 4 if doc["metbench_domain"] == "minecraft" else 3 + return float(_parse_choice(response, maximum) == int(doc["target"])) + + +def clustered_ratio_interval(counts): + """Return a 95% normal interval for a ratio, clustered by example. + + Each pair contains correct and total states for one example. For Chess's + 64-square score this is the mean board score plus or minus 1.96 standard + errors, using the sample variance across boards. Unequal denominators use + the cluster delta method. Fewer than two contributing examples has no CI. + """ + counts = [(correct, total) for correct, total in counts if total > 0] + n = len(counts) + if n < 2: + return None, None + total = sum(total for _, total in counts) + accuracy = sum(correct for correct, _ in counts) / total + residual_ss = sum((correct - accuracy * size) ** 2 for correct, size in counts) + stderr = math.sqrt(n * residual_ss / (n - 1)) / total + margin = 1.959963984540054 * stderr + return max(0.0, accuracy - margin), min(1.0, accuracy + margin)