From f068c917bd1e2178db4df00b4d488bf7158cd980 Mon Sep 17 00:00:00 2001 From: HQHJT999 Date: Tue, 4 Aug 2026 20:57:01 +0800 Subject: [PATCH 1/2] feat(benchmark): add robust psychometric test assembly --- benchmarks/AssessmentEngineering/README.md | 14 + .../AssessmentEngineering/README_zh-CN.md | 11 + .../RobustTestAssembly/README.md | 30 ++ .../RobustTestAssembly/README_zh-CN.md | 30 ++ .../RobustTestAssembly/Task.md | 68 +++ .../RobustTestAssembly/Task_zh-CN.md | 127 +++++ .../frontier_eval/agent_files.txt | 7 + .../frontier_eval/artifact_files.txt | 2 + .../frontier_eval/candidate_destination.txt | 1 + .../frontier_eval/constraints.txt | 12 + .../frontier_eval/copy_files.txt | 6 + .../frontier_eval/eval_command.txt | 1 + .../frontier_eval/initial_program.txt | 1 + .../frontier_eval/readonly_files.txt | 14 + .../references/anchor_solutions_v1.json | 262 ++++++++++ .../references/baseline_init_v1.py | 324 ++++++++++++ .../references/design_notes.md | 30 ++ .../RobustTestAssembly/scripts/init.py | 325 ++++++++++++ .../verification/build_anchor_v1.py | 70 +++ .../verification/clean_candidate_v1.py | 344 +++++++++++++ .../verification/evaluator.py | 336 +++++++++++++ .../verification/generator.py | 312 ++++++++++++ .../verification/problem_view.py | 30 ++ .../verification/scoring.py | 269 ++++++++++ .../verification/test_task_v1.py | 471 ++++++++++++++++++ 25 files changed, 3097 insertions(+) create mode 100644 benchmarks/AssessmentEngineering/README.md create mode 100644 benchmarks/AssessmentEngineering/README_zh-CN.md create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/README.md create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/README_zh-CN.md create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/Task.md create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/Task_zh-CN.md create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/agent_files.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/artifact_files.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/candidate_destination.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/constraints.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_command.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/initial_program.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/references/anchor_solutions_v1.json create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/references/design_notes.md create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/scripts/init.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/verification/build_anchor_v1.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/verification/clean_candidate_v1.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/verification/generator.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/verification/problem_view.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/verification/scoring.py create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py diff --git a/benchmarks/AssessmentEngineering/README.md b/benchmarks/AssessmentEngineering/README.md new file mode 100644 index 00000000..d937fdbb --- /dev/null +++ b/benchmarks/AssessmentEngineering/README.md @@ -0,0 +1,14 @@ +# AssessmentEngineering + +This domain contains lightweight, deterministic, and fully offline benchmark tasks inspired by +psychometrics and operational assessment design. + +The tasks translate real assessment requirements—measurement precision, content coverage, +administration time, fairness risk, item exposure, and test security—into reproducible engineering +optimization problems with explicit feasibility constraints. + +## Task + +- `RobustTestAssembly`: assemble a fixed-length test form from a synthetic item bank while matching + exact domain quotas, respecting time, DIF-risk, exposure, and shared-material constraints, and + optimizing measurement information across multiple ability levels. diff --git a/benchmarks/AssessmentEngineering/README_zh-CN.md b/benchmarks/AssessmentEngineering/README_zh-CN.md new file mode 100644 index 00000000..eedea59a --- /dev/null +++ b/benchmarks/AssessmentEngineering/README_zh-CN.md @@ -0,0 +1,11 @@ +# AssessmentEngineering（测评工程） + +本领域包含轻量、确定性且完全离线的测评工程 benchmark，设计灵感来自心理测量和正式测验开发。 + +这些任务将测量精度、内容覆盖、作答时间、公平性风险、题目曝光和测验安全等现实要求， +转化为具有明确可行约束、能够稳定复现的工程优化问题。 + +## 任务 + +- `RobustTestAssembly`：从合成题库中组建固定长度的测验，精确满足各领域题量要求， + 遵守时间、DIF 风险、题目曝光和共享材料冲突限制，并优化多个能力水平上的测量信息。 diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/README.md b/benchmarks/AssessmentEngineering/RobustTestAssembly/README.md new file mode 100644 index 00000000..eff9c678 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/README.md @@ -0,0 +1,30 @@ +# RobustTestAssembly + +This is a fully offline and deterministic benchmark for psychometric test assembly. + +The candidate selects a fixed number of items from a synthetic item bank while satisfying domain quotas, time limits, DIF-risk limits, exposure limits, and shared-material constraints. It should also improve measurement information across multiple ability levels. + +## Benchmark ID + +`AssessmentEngineering/RobustTestAssembly` + +## Local evaluation + +Run from the task directory: + +`python verification/evaluator.py scripts/init.py` + +The initial program should score 50 and remain feasible in all 10 scenarios. + +Run the complete test suite with: + +`python verification/test_task_v1.py` + +## Runtime requirements + +- Linux +- Python 3.10 or newer +- Python standard library only +- No GPU, network access, external data, or API key required + +All item banks and risk indicators are synthetic. This benchmark evaluates optimization and constraint handling; it does not validate the fairness, validity, or clinical use of a real assessment. diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/README_zh-CN.md b/benchmarks/AssessmentEngineering/RobustTestAssembly/README_zh-CN.md new file mode 100644 index 00000000..747e0f20 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/README_zh-CN.md @@ -0,0 +1,30 @@ +# RobustTestAssembly（稳健心理测验组卷） + +这是一个完全离线、可重复运行的心理测量组卷 benchmark。 + +候选程序需要从合成题库中选择固定数量的题目，同时满足领域配额、作答时间、DIF 风险、题目曝光和材料冲突等限制，并尽可能提高不同能力水平上的测量信息。 + +## Benchmark ID + +`AssessmentEngineering/RobustTestAssembly` + +## 本地评测 + +在任务目录运行： + +`python verification/evaluator.py scripts/init.py` + +初始程序应得到 50 分，并在 10 个场景中全部合法。 + +完整自检命令： + +`python verification/test_task_v1.py` + +## 运行环境 + +- Linux +- Python 3.10 及以上 +- 只使用 Python 标准库 +- 不需要 GPU、网络、外部数据或 API Key + +所有题库和风险指标均为合成数据。本任务评测组卷优化和约束处理能力，不代表真实测验已经具有效度、公平性或临床价值。 diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/Task.md b/benchmarks/AssessmentEngineering/RobustTestAssembly/Task.md new file mode 100644 index 00000000..27f5798e --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/Task.md @@ -0,0 +1,68 @@ +# Task: Robust Psychometric Test Assembly + +## Background + +Operational test assembly must balance measurement quality with content coverage, administration time, fairness risk, item exposure, and test security. + +The candidate selects 24 items from a synthetic bank of 80 items while satisfying every hard constraint and improving information across several ability levels. + +## Input + +The problem JSON contains the required test length, exact domain quotas, ability points and weights, operational limits, and the candidate item bank. + +Each item includes an integer ID, domain, content strand, completion time, discrimination, difficulty, synthetic DIF risk, exposure, and an optional enemy group. + +Scenario names, random seeds, and feedback labels are removed before the problem is passed to the candidate. + +## Output + +The candidate must write a JSON object containing an integer list named `selected_ids`. + +Example: `{"selected_ids": [1, 2, 3]}` + +## Hard constraints + +1. Select exactly `test_length` items. +2. Use only known item IDs and do not repeat an ID. +3. Match every value in `domain_targets` exactly. +4. Do not exceed `max_items_per_enemy_group`. +5. Do not exceed `max_total_time`. +6. Do not exceed `max_mean_dif`. +7. Do not exceed `max_mean_exposure`. +8. Finish within the evaluator time limit. + +## Measurement information + +The benchmark uses a simplified two-parameter logistic model. + +`P(theta) = 1 / (1 + exp(-a * (theta - b)))` + +`I(theta) = a^2 * P(theta) * (1 - P(theta))` + +The evaluator computes the mean selected-item information at four ability points. + +## Scoring + +The raw objective combines weighted information, worst-point information, profile balance, DIF quality, exposure quality, time efficiency, and content-strand coverage. + +The frozen initial solution defines 50 points in every scenario. Better solutions score above 50 and worse solutions score below 50. + +The final robust score is 75 percent mean scenario score and 25 percent twentieth-percentile scenario score. + +If any scenario violates a hard constraint, the formal combined score is zero. A diagnostic score is still returned. + +## Scenarios + +The evaluator contains six development scenarios and four validation scenarios. + +Scenarios vary domain quotas, ability-profile weights, time limits, DIF limits, and exposure limits. Fixed seeds make every run reproducible. + +## Runtime + +The candidate must be deterministic, self-contained, and offline. Each scenario has a 10-second candidate timeout. + +The benchmark uses only the Python standard library and requires no GPU, external dataset, network access, or API key. + +## Scope + +All item banks and risk indicators are synthetic. The benchmark evaluates optimization and constraint handling; it does not validate the reliability, validity, fairness, clinical utility, or legal compliance of a real assessment. diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/Task_zh-CN.md b/benchmarks/AssessmentEngineering/RobustTestAssembly/Task_zh-CN.md new file mode 100644 index 00000000..19c3cf4a --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/Task_zh-CN.md @@ -0,0 +1,127 @@ +# 任务：稳健心理测验组卷 + +## 任务背景 + +正式测验组卷不能只挑选单项质量最高的题目，还必须同时考虑内容覆盖、作答时间、公平性风险、题目曝光和材料重复等现实限制。 + +本任务要求候选程序从一份包含 80 道合成题目的题库中，选择 24 道题组成测验。算法需要在满足全部硬性约束的前提下，提高测验在多个能力水平上的测量信息。 + +## 输入数据 + +候选程序会收到一个 JSON 格式的问题文件，主要包含： + +- `test_length`：需要选择的题目总数； +- `domain_targets`：记忆、推理、注意和执行功能四个领域的题量要求； +- `theta_points`：需要考察的能力水平； +- `theta_weights`：不同能力水平在当前场景中的重要程度； +- `max_total_time`：允许的总作答时间； +- `max_mean_dif`：允许的平均 DIF 风险上限； +- `max_mean_exposure`：允许的平均题目曝光上限； +- `max_items_per_enemy_group`：同一材料冲突组最多可选择的题目数； +- `items`：候选题目列表。 + +每道题包含题号、所属领域、内容类别、作答时间、区分度、难度、DIF 风险、曝光率和材料冲突组等属性。 + +场景名称、随机种子和反馈类型不会提供给候选程序，避免算法直接记忆固定答案。 + +## 候选程序输出 + +候选程序通过 `--problem` 读取问题文件，通过 `--output` 写出答案文件。 + +答案必须是一个 JSON 对象，其中 `selected_ids` 是由整数题号组成的列表。例如： + +`{"selected_ids": [1, 2, 3]}` + +实际列表长度必须等于场景中的 `test_length`。 + +## 硬性约束 + +一份组卷方案必须同时满足以下要求： + +1. 选择题数必须等于 `test_length`； +2. 所有题号都必须真实存在； +3. 同一道题不能重复选择； +4. 四个领域的题量必须与 `domain_targets` 完全一致； +5. 同一 `enemy_group` 中的入选题数不能超过允许上限； +6. 总作答时间不能超过 `max_total_time`； +7. 平均 DIF 风险不能超过 `max_mean_dif`； +8. 平均曝光率不能超过 `max_mean_exposure`； +9. 候选程序必须在评分器规定的时间内结束。 + +## 测量信息计算 + +任务使用简化的二参数逻辑模型计算题目信息量。 + +对于能力水平 `theta`，题目答对概率为： + +`P(theta) = 1 / (1 + exp(-a * (theta - b)))` + +其中 `a` 是题目区分度，`b` 是题目难度。 + +题目在该能力水平上的信息量为： + +`I(theta) = a^2 * P(theta) * (1 - P(theta))` + +评分器会计算所有入选题目在四个能力水平上的平均信息量，形成整份测验的信息曲线。 + +## 评分方法 + +合法方案的原始目标由以下部分加权组成： + +- 50%：按照 `theta_weights` 计算的加权测量信息； +- 22%：四个能力水平中最低的测量信息； +- 8%：信息曲线的均衡程度； +- 8%：平均 DIF 风险质量； +- 5%：平均曝光率质量； +- 3%：剩余作答时间效率； +- 4%：内容细分类别覆盖率。 + +每个场景都使用冻结的初始方案作为 50 分参照线。候选方案优于参照方案时得分高于 50，低于参照方案时得分低于 50。 + +十个场景的最终稳健得分由两部分组成： + +- 75%：所有场景的平均分； +- 25%：场景分数的第 20 百分位。 + +第 20 百分位用于惩罚只在部分场景表现良好、但在其他场景明显退化的算法。 + +如果任意一个场景违反硬性约束，正式总分归零；评分器仍会返回诊断分，帮助定位问题。 + +## 场景与泛化 + +评分器包含 10 个固定且可复现的场景： + +- 6 个开发场景：返回较详细的评分反馈； +- 4 个验证场景：只返回汇总信息，用于检查算法是否只针对开发场景过度调整。 + +不同场景会改变： + +- 四个领域的题量配额； +- 不同能力水平的权重； +- 总作答时间上限； +- 平均 DIF 风险上限； +- 平均题目曝光上限。 + +所有场景都由固定随机种子生成，因此同一版本的 benchmark 每次运行结果一致。 + +候选程序不会收到场景名称、随机种子或反馈类型，只能依据公开题目属性和组卷要求制定通用策略。 + +## 运行限制 + +候选程序必须保持确定性，并且只能离线运行。 + +程序不得读取或修改评分器、题库生成器、参照答案、结果文件、环境密钥或其他非公开信息。 + +每个场景的候选程序运行时间上限为 10 秒。超时、运行失败、输出文件缺失或 JSON 格式错误，都会使该场景无效。 + +任务只依赖 Python 标准库，不需要 GPU、网络、外部数据集或额外的软件环境。 + +## 解释边界 + +本任务中的题库、DIF 风险、曝光率和材料冲突关系均为合成数据。 + +该 benchmark 评测的是算法在多目标组卷、硬性约束处理和跨场景稳健性方面的能力。 + +评分结果不能用于证明某份真实测验已经具有效度、信度、公平性、临床价值或法律合规性。 + +任务使用的是简化二参数逻辑模型，没有模拟真实题目校准误差、多维能力结构、作答过程数据或真实机构的题库安全制度。 diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/agent_files.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/agent_files.txt new file mode 100644 index 00000000..e8a78865 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/agent_files.txt @@ -0,0 +1,7 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +scripts/init.py +references/design_notes.md +frontier_eval/constraints.txt diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/artifact_files.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..76dc893a --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/artifact_files.txt @@ -0,0 +1,2 @@ +metrics.json +artifacts.json diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/candidate_destination.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/candidate_destination.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/candidate_destination.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/constraints.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/constraints.txt new file mode 100644 index 00000000..24c81729 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/constraints.txt @@ -0,0 +1,12 @@ +Robust psychometric test assembly constraints: + +1) Edit only `scripts/init.py` inside the EVOLVE-BLOCK markers. +2) Keep the command-line interface working: + `--problem --output `. +3) Write one JSON object containing `selected_ids`, which must be a list of integer item IDs. +4) Select exactly the required number of items and match every domain target exactly. +5) Do not select duplicate, unknown, or mutually conflicting items. +6) Respect the total-time, mean-DIF-risk, and mean-exposure limits in every scenario. +7) Keep the algorithm deterministic, self-contained, and offline. +8) Do not read, modify, or replace evaluator, generator, anchor, result, or environment-secret files. +9) The candidate must finish each scenario within the evaluator time limit. diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt new file mode 100644 index 00000000..0f5a883a --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt @@ -0,0 +1,6 @@ +scripts/init.py +verification/evaluator.py +verification/generator.py +verification/scoring.py +verification/problem_view.py +references/anchor_solutions_v1.json diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_command.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_command.txt new file mode 100644 index 00000000..527d23b5 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +{python} verification/evaluator.py {candidate} --metrics-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/initial_program.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/initial_program.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..871e81a0 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt @@ -0,0 +1,14 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +verification/evaluator.py +verification/generator.py +verification/scoring.py +verification/problem_view.py +verification/test_task_v1.py +verification/build_anchor_v1.py +verification/clean_candidate_v1.py +references/anchor_solutions_v1.json +references/baseline_init_v1.py +references/design_notes.md diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/references/anchor_solutions_v1.json b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/anchor_solutions_v1.json new file mode 100644 index 00000000..11f49366 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/anchor_solutions_v1.json @@ -0,0 +1,262 @@ +{ + "dev_balanced": [ + 10, + 13, + 17, + 11, + 2, + 20, + 39, + 28, + 37, + 21, + 31, + 24, + 43, + 42, + 50, + 46, + 48, + 57, + 68, + 73, + 78, + 72, + 69, + 79 + ], + "dev_low_ability_focus": [ + 13, + 2, + 20, + 10, + 7, + 4, + 6, + 21, + 40, + 32, + 27, + 35, + 25, + 47, + 43, + 53, + 56, + 58, + 59, + 66, + 69, + 61, + 73, + 75 + ], + "dev_high_ability_focus": [ + 18, + 15, + 9, + 11, + 19, + 38, + 27, + 37, + 31, + 40, + 21, + 33, + 43, + 46, + 52, + 48, + 56, + 78, + 62, + 70, + 80, + 75, + 74, + 72 + ], + "dev_fast_screening": [ + 2, + 8, + 16, + 4, + 13, + 11, + 37, + 24, + 30, + 33, + 38, + 26, + 46, + 47, + 41, + 56, + 51, + 49, + 57, + 67, + 72, + 76, + 77, + 65 + ], + "dev_fairness_sensitive": [ + 9, + 20, + 19, + 14, + 6, + 7, + 38, + 35, + 29, + 37, + 27, + 24, + 46, + 48, + 42, + 52, + 56, + 60, + 62, + 70, + 63, + 74, + 75, + 66 + ], + "dev_security_sensitive": [ + 14, + 11, + 20, + 4, + 9, + 17, + 28, + 30, + 38, + 36, + 23, + 40, + 51, + 50, + 48, + 54, + 56, + 59, + 77, + 78, + 76, + 64, + 79, + 74 + ], + "validation_mixed_a": [ + 16, + 13, + 7, + 19, + 10, + 3, + 2, + 37, + 21, + 23, + 32, + 33, + 56, + 50, + 51, + 46, + 43, + 42, + 68, + 76, + 62, + 70, + 72, + 66 + ], + "validation_mixed_b": [ + 9, + 15, + 19, + 12, + 8, + 22, + 29, + 37, + 26, + 23, + 39, + 27, + 58, + 55, + 57, + 48, + 60, + 51, + 76, + 63, + 80, + 61, + 79, + 78 + ], + "validation_mixed_c": [ + 13, + 16, + 7, + 12, + 10, + 5, + 39, + 27, + 26, + 35, + 33, + 30, + 47, + 55, + 45, + 57, + 42, + 51, + 53, + 67, + 62, + 78, + 70, + 71 + ], + "validation_mixed_d": [ + 14, + 2, + 20, + 5, + 3, + 18, + 21, + 26, + 37, + 33, + 28, + 30, + 57, + 58, + 52, + 48, + 44, + 61, + 79, + 73, + 80, + 69, + 67, + 72 + ] +} \ No newline at end of file diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py new file mode 100644 index 00000000..87d6e791 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py @@ -0,0 +1,324 @@ +# EVOLVE-BLOCK-START +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + + +def item_information( + item: dict[str, Any], + theta: float, +) -> float: + """计算一道题在某个能力水平上的测量信息量。""" + discrimination = float(item["discrimination"]) + difficulty = float(item["difficulty"]) + + value = discrimination * (theta - difficulty) + value = max(-60.0, min(60.0, value)) + + probability = 1.0 / (1.0 + math.exp(-value)) + + return ( + discrimination + * discrimination + * probability + * (1.0 - probability) + ) + + +def item_value( + item: dict[str, Any], + problem: dict[str, Any], +) -> float: + """给单道题计算一个简单的初始吸引力分数。""" + information = [ + item_information(item, theta) + for theta in problem["theta_points"] + ] + + weighted_information = sum( + weight * value + for weight, value in zip( + problem["theta_weights"], + information, + ) + ) + + weakest_information = min(information) + + time_penalty = ( + 3.0 + * item["time"] + / problem["max_total_time"] + ) + + dif_penalty = ( + 0.50 + * item["dif_risk"] + / problem["max_mean_dif"] + ) + + exposure_penalty = ( + 0.30 + * item["exposure"] + / problem["max_mean_exposure"] + ) + + return ( + weighted_information + + 0.20 * weakest_information + - time_penalty + - dif_penalty + - exposure_penalty + ) + + +def constraint_values( + selected: list[dict[str, Any]], +) -> tuple[float, float, float]: + total_time = sum(item["time"] for item in selected) + + mean_dif = sum( + item["dif_risk"] for item in selected + ) / len(selected) + + mean_exposure = sum( + item["exposure"] for item in selected + ) / len(selected) + + return total_time, mean_dif, mean_exposure + + +def violation_amount( + selected: list[dict[str, Any]], + problem: dict[str, Any], +) -> float: + total_time, mean_dif, mean_exposure = ( + constraint_values(selected) + ) + + return ( + max( + 0.0, + total_time / problem["max_total_time"] - 1.0, + ) + + max( + 0.0, + mean_dif / problem["max_mean_dif"] - 1.0, + ) + + max( + 0.0, + mean_exposure + / problem["max_mean_exposure"] + - 1.0, + ) + ) + + +def solution_quality( + selected: list[dict[str, Any]], + problem: dict[str, Any], +) -> float: + information_by_theta = [ + sum( + item_information(item, theta) + for item in selected + ) + for theta in problem["theta_points"] + ] + + weighted_information = sum( + weight * value + for weight, value in zip( + problem["theta_weights"], + information_by_theta, + ) + ) + + return ( + weighted_information + + 0.20 * min(information_by_theta) + ) + + +def respects_enemy_groups( + selected: list[dict[str, Any]], +) -> bool: + seen: set[str] = set() + + for item in selected: + group = item["enemy_group"] + + if group is None: + continue + + if group in seen: + return False + + seen.add(group) + + return True + + +def select_items( + problem: dict[str, Any], +) -> list[int]: + selected: list[dict[str, Any]] = [] + + # 第一阶段：每个领域分别进行简单贪心选择。 + for domain, required_count in ( + problem["domain_targets"].items() + ): + candidates = [ + item + for item in problem["items"] + if item["domain"] == domain + ] + + chosen: list[dict[str, Any]] = [] + used_enemy_groups: set[str] = set() + + while len(chosen) < required_count: + available = [] + + for item in candidates: + if item in chosen: + continue + + group = item["enemy_group"] + + if ( + group is not None + and group in used_enemy_groups + ): + continue + + available.append(item) + + if not available: + raise RuntimeError( + f"领域 {domain} 没有足够的合法题目" + ) + + best_item = max( + available, + key=lambda item: ( + item_value(item, problem), + -item["time"], + -item["dif_risk"], + -item["exposure"], + -item["id"], + ), + ) + + chosen.append(best_item) + + if best_item["enemy_group"] is not None: + used_enemy_groups.add( + best_item["enemy_group"] + ) + + selected.extend(chosen) + + # 第二阶段：如果整体限制超标，就进行同领域换题。 + for _ in range(200): + current_violation = violation_amount( + selected, + problem, + ) + + if current_violation <= 1e-12: + break + + current_quality = solution_quality( + selected, + problem, + ) + selected_ids = { + item["id"] for item in selected + } + + best_swap = None + + for index, old_item in enumerate(selected): + for new_item in problem["items"]: + if new_item["domain"] != old_item["domain"]: + continue + + if new_item["id"] in selected_ids: + continue + + trial = selected.copy() + trial[index] = new_item + + if not respects_enemy_groups(trial): + continue + + new_violation = violation_amount( + trial, + problem, + ) + + if new_violation >= current_violation: + continue + + quality_loss = ( + current_quality + - solution_quality(trial, problem) + ) + + swap_key = ( + new_violation, + quality_loss, + new_item["id"], + ) + + if ( + best_swap is None + or swap_key < best_swap[0] + ): + best_swap = ( + swap_key, + index, + new_item, + ) + + if best_swap is None: + break + + _, index, new_item = best_swap + selected[index] = new_item + + return [item["id"] for item in selected] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--problem", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + problem = json.loads( + Path(args.problem).read_text( + encoding="utf-8", + ) + ) + + selected_ids = select_items(problem) + + Path(args.output).write_text( + json.dumps( + {"selected_ids": selected_ids}, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() +# EVOLVE-BLOCK-END diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/references/design_notes.md b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/design_notes.md new file mode 100644 index 00000000..1813fb8e --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/design_notes.md @@ -0,0 +1,30 @@ +# Design Notes + +## Purpose + +This benchmark evaluates robust psychometric test assembly under simultaneous measurement, content, time, fairness-risk, exposure, and security constraints. + +## Synthetic design + +The item banks are synthetic so the task is deterministic, redistributable, fully offline, and free of confidential assessment content. + +## Scenario design + +Ten fixed scenarios vary domain quotas, ability-profile weights, time limits, DIF-risk limits, and exposure limits. Six are development scenarios and four are validation scenarios. + +## Baseline and optimization gap + +The frozen baseline scores 50.0 and is feasible in all ten scenarios. + +An independent local-search fixture that does not import the formal scoring module scores 72.605462 while remaining feasible in all ten scenarios. This verifies meaningful optimization headroom. + +## Reproducibility + +- Scenario generation uses fixed seeds. +- Scenario names and seeds are hidden from the candidate. +- The evaluator and baseline are deterministic. +- Evaluation uses only the Python standard library. + +## Limitations + +The benchmark uses a simplified 2PL information model and synthetic scalar proxies for DIF risk and exposure. It does not model empirical calibration uncertainty, multidimensional IRT, real response data, or institutional test-security procedures. diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/scripts/init.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/scripts/init.py new file mode 100644 index 00000000..725f564e --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/scripts/init.py @@ -0,0 +1,325 @@ +# EVOLVE-BLOCK-START +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + + +def item_information( + item: dict[str, Any], + theta: float, +) -> float: + """计算一道题在某个能力水平上的测量信息量。""" + discrimination = float(item["discrimination"]) + difficulty = float(item["difficulty"]) + + value = discrimination * (theta - difficulty) + value = max(-60.0, min(60.0, value)) + + probability = 1.0 / (1.0 + math.exp(-value)) + + return ( + discrimination + * discrimination + * probability + * (1.0 - probability) + ) + + +def item_value( + item: dict[str, Any], + problem: dict[str, Any], +) -> float: + """给单道题计算一个简单的初始吸引力分数。""" + information = [ + item_information(item, theta) + for theta in problem["theta_points"] + ] + + weighted_information = sum( + weight * value + for weight, value in zip( + problem["theta_weights"], + information, + ) + ) + + weakest_information = min(information) + + time_penalty = ( + 3.0 + * item["time"] + / problem["max_total_time"] + ) + + dif_penalty = ( + 0.50 + * item["dif_risk"] + / problem["max_mean_dif"] + ) + + exposure_penalty = ( + 0.30 + * item["exposure"] + / problem["max_mean_exposure"] + ) + + return ( + weighted_information + + 0.20 * weakest_information + - time_penalty + - dif_penalty + - exposure_penalty + ) + + +def constraint_values( + selected: list[dict[str, Any]], +) -> tuple[float, float, float]: + total_time = sum(item["time"] for item in selected) + + mean_dif = sum( + item["dif_risk"] for item in selected + ) / len(selected) + + mean_exposure = sum( + item["exposure"] for item in selected + ) / len(selected) + + return total_time, mean_dif, mean_exposure + + +def violation_amount( + selected: list[dict[str, Any]], + problem: dict[str, Any], +) -> float: + total_time, mean_dif, mean_exposure = ( + constraint_values(selected) + ) + + return ( + max( + 0.0, + total_time / problem["max_total_time"] - 1.0, + ) + + max( + 0.0, + mean_dif / problem["max_mean_dif"] - 1.0, + ) + + max( + 0.0, + mean_exposure + / problem["max_mean_exposure"] + - 1.0, + ) + ) + + +def solution_quality( + selected: list[dict[str, Any]], + problem: dict[str, Any], +) -> float: + information_by_theta = [ + sum( + item_information(item, theta) + for item in selected + ) + for theta in problem["theta_points"] + ] + + weighted_information = sum( + weight * value + for weight, value in zip( + problem["theta_weights"], + information_by_theta, + ) + ) + + return ( + weighted_information + + 0.20 * min(information_by_theta) + ) + + +def respects_enemy_groups( + selected: list[dict[str, Any]], +) -> bool: + seen: set[str] = set() + + for item in selected: + group = item["enemy_group"] + + if group is None: + continue + + if group in seen: + return False + + seen.add(group) + + return True + + +def select_items( + problem: dict[str, Any], +) -> list[int]: + selected: list[dict[str, Any]] = [] + + # 第一阶段：每个领域分别进行简单贪心选择。 + for domain, required_count in ( + problem["domain_targets"].items() + ): + candidates = [ + item + for item in problem["items"] + if item["domain"] == domain + ] + + chosen: list[dict[str, Any]] = [] + used_enemy_groups: set[str] = set() + + while len(chosen) < required_count: + available = [] + + for item in candidates: + if item in chosen: + continue + + group = item["enemy_group"] + + if ( + group is not None + and group in used_enemy_groups + ): + continue + + available.append(item) + + if not available: + raise RuntimeError( + f"领域 {domain} 没有足够的合法题目" + ) + + best_item = max( + available, + key=lambda item: ( + item_value(item, problem), + -item["time"], + -item["dif_risk"], + -item["exposure"], + -item["id"], + ), + ) + + chosen.append(best_item) + + if best_item["enemy_group"] is not None: + used_enemy_groups.add( + best_item["enemy_group"] + ) + + selected.extend(chosen) + + # 第二阶段：如果整体限制超标，就进行同领域换题。 + for _ in range(200): + current_violation = violation_amount( + selected, + problem, + ) + + if current_violation <= 1e-12: + break + + current_quality = solution_quality( + selected, + problem, + ) + selected_ids = { + item["id"] for item in selected + } + + best_swap = None + + for index, old_item in enumerate(selected): + for new_item in problem["items"]: + if new_item["domain"] != old_item["domain"]: + continue + + if new_item["id"] in selected_ids: + continue + + trial = selected.copy() + trial[index] = new_item + + if not respects_enemy_groups(trial): + continue + + new_violation = violation_amount( + trial, + problem, + ) + + if new_violation >= current_violation: + continue + + quality_loss = ( + current_quality + - solution_quality(trial, problem) + ) + + swap_key = ( + new_violation, + quality_loss, + new_item["id"], + ) + + if ( + best_swap is None + or swap_key < best_swap[0] + ): + best_swap = ( + swap_key, + index, + new_item, + ) + + if best_swap is None: + break + + _, index, new_item = best_swap + selected[index] = new_item + + return [item["id"] for item in selected] + + +# EVOLVE-BLOCK-END + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--problem", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + problem = json.loads( + Path(args.problem).read_text( + encoding="utf-8", + ) + ) + + selected_ids = select_items(problem) + + Path(args.output).write_text( + json.dumps( + {"selected_ids": selected_ids}, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/build_anchor_v1.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/build_anchor_v1.py new file mode 100644 index 00000000..af341071 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/build_anchor_v1.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +from generator import SCENARIO_SPECS, generate_scenario +from problem_view import candidate_view + + +TASK_ROOT = Path(__file__).resolve().parents[1] +CANDIDATE_PATH = TASK_ROOT / "references" / "baseline_init_v1.py" +OUTPUT_PATH = TASK_ROOT / "references" / "anchor_solutions_v1.json" + + +def main() -> None: + anchors: dict[str, list[int]] = {} + + for scenario_name in SCENARIO_SPECS: + problem = generate_scenario(scenario_name) + + with tempfile.TemporaryDirectory() as directory: + directory_path = Path(directory) + problem_path = directory_path / "problem.json" + solution_path = directory_path / "solution.json" + + problem_path.write_text( + json.dumps(candidate_view(problem), indent=2), + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(CANDIDATE_PATH), + "--problem", + str(problem_path), + "--output", + str(solution_path), + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + + if result.returncode != 0: + raise RuntimeError( + f"{scenario_name} 运行失败：{result.stderr}" + ) + + solution = json.loads( + solution_path.read_text(encoding="utf-8") + ) + + anchors[scenario_name] = solution["selected_ids"] + + OUTPUT_PATH.write_text( + json.dumps(anchors, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + print(f"已保存 {len(anchors)} 个参照答案") + print(f"位置：{OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/clean_candidate_v1.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/clean_candidate_v1.py new file mode 100644 index 00000000..64f9dae1 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/clean_candidate_v1.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import argparse +import importlib.util +import json +import math +from collections import Counter +from pathlib import Path +from typing import Any + + +TASK_ROOT = Path(__file__).resolve().parents[1] +BASELINE_PATH = ( + TASK_ROOT + / "references" + / "baseline_init_v1.py" +) + + +def load_baseline_module() -> Any: + """读取公开的普通初始算法，不读取裁判。""" + spec = importlib.util.spec_from_file_location( + "frozen_baseline", + BASELINE_PATH, + ) + + if spec is None or spec.loader is None: + raise RuntimeError("无法读取普通初始算法") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + return module + + +def item_information( + item: dict[str, Any], + theta: float, +) -> float: + """估计一道题在某个能力水平上的测量信息量。""" + discrimination = float(item["discrimination"]) + difficulty = float(item["difficulty"]) + + value = discrimination * (theta - difficulty) + value = max(-60.0, min(60.0, value)) + + probability = 1.0 / ( + 1.0 + math.exp(-value) + ) + + return ( + discrimination + * discrimination + * probability + * (1.0 - probability) + ) + + +def is_valid( + problem: dict[str, Any], + selected_ids: list[int], +) -> bool: + """根据公开规则判断一套组卷方案是否合法。""" + if len(selected_ids) != problem["test_length"]: + return False + + if len(selected_ids) != len(set(selected_ids)): + return False + + items_by_id = { + item["id"]: item + for item in problem["items"] + } + + if any( + item_id not in items_by_id + for item_id in selected_ids + ): + return False + + selected = [ + items_by_id[item_id] + for item_id in selected_ids + ] + + domain_counts = Counter( + item["domain"] for item in selected + ) + + for domain, target in ( + problem["domain_targets"].items() + ): + if domain_counts.get(domain, 0) != target: + return False + + enemy_counts = Counter( + item["enemy_group"] + for item in selected + if item["enemy_group"] is not None + ) + + if any( + count + > problem["max_items_per_enemy_group"] + for count in enemy_counts.values() + ): + return False + + total_time = sum( + item["time"] for item in selected + ) + + mean_dif = sum( + item["dif_risk"] for item in selected + ) / len(selected) + + mean_exposure = sum( + item["exposure"] for item in selected + ) / len(selected) + + return ( + total_time <= problem["max_total_time"] + and mean_dif <= problem["max_mean_dif"] + and mean_exposure + <= problem["max_mean_exposure"] + ) + + +def proxy_quality( + problem: dict[str, Any], + selected_ids: list[int], +) -> float: + """ + 使用独立设计的近似评价规则。 + + 它和正式裁判的权重并不相同， + 只依据公开的心理测量目标判断方案。 + """ + items_by_id = { + item["id"]: item + for item in problem["items"] + } + + selected = [ + items_by_id[item_id] + for item_id in selected_ids + ] + + information_curve = [ + sum( + item_information(item, theta) + for item in selected + ) / len(selected) + for theta in problem["theta_points"] + ] + + weighted_information = sum( + weight * information + for weight, information in zip( + problem["theta_weights"], + information_curve, + ) + ) + + worst_information = min(information_curve) + + mean_information = ( + sum(information_curve) + / len(information_curve) + ) + + profile_balance = ( + worst_information / mean_information + if mean_information > 0 + else 0.0 + ) + + total_time = sum( + item["time"] for item in selected + ) + + mean_dif = sum( + item["dif_risk"] for item in selected + ) / len(selected) + + mean_exposure = sum( + item["exposure"] for item in selected + ) / len(selected) + + available_strands = { + (item["domain"], item["content_strand"]) + for item in problem["items"] + } + + selected_strands = { + (item["domain"], item["content_strand"]) + for item in selected + } + + strand_coverage = ( + len(selected_strands) + / len(available_strands) + ) + + time_efficiency = max( + 0.0, + 1.0 + - total_time / problem["max_total_time"], + ) + + # 这些权重故意不使用正式裁判的权重。 + return ( + 0.48 * weighted_information + + 0.25 * worst_information + + 0.08 * profile_balance + + 0.09 * (1.0 - mean_dif) + + 0.05 * (1.0 - mean_exposure) + + 0.03 * time_efficiency + + 0.02 * strand_coverage + ) + + +def improve_selection( + problem: dict[str, Any], +) -> list[int]: + baseline = load_baseline_module() + + selected_ids = list( + baseline.select_items(problem) + ) + + items_by_id = { + item["id"]: item + for item in problem["items"] + } + + current_quality = proxy_quality( + problem, + selected_ids, + ) + + for _ in range(100): + selected_set = set(selected_ids) + + best_ids: list[int] | None = None + best_quality = current_quality + + for position, old_id in enumerate( + selected_ids + ): + old_domain = ( + items_by_id[old_id]["domain"] + ) + + for new_item in problem["items"]: + new_id = new_item["id"] + + # 只在同一领域内换题， + # 避免破坏领域数量要求。 + if ( + new_item["domain"] + != old_domain + ): + continue + + if new_id in selected_set: + continue + + trial_ids = selected_ids.copy() + trial_ids[position] = new_id + + if not is_valid( + problem, + trial_ids, + ): + continue + + trial_quality = proxy_quality( + problem, + trial_ids, + ) + + if ( + trial_quality + > best_quality + 1e-12 + ): + best_quality = trial_quality + best_ids = trial_ids + + elif ( + abs( + trial_quality + - best_quality + ) + <= 1e-12 + and best_ids is not None + and tuple(trial_ids) + < tuple(best_ids) + ): + best_ids = trial_ids + + if best_ids is None: + break + + selected_ids = best_ids + current_quality = best_quality + + return selected_ids + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--problem", + required=True, + ) + parser.add_argument( + "--output", + required=True, + ) + args = parser.parse_args() + + problem = json.loads( + Path(args.problem).read_text( + encoding="utf-8", + ) + ) + + selected_ids = improve_selection(problem) + + Path(args.output).write_text( + json.dumps( + { + "selected_ids": selected_ids, + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py new file mode 100644 index 00000000..6b4c7962 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +from generator import SCENARIO_SPECS, generate_scenario +from problem_view import candidate_view +from scoring import score_selection + + +TASK_ROOT = Path(__file__).resolve().parents[1] +ANCHOR_PATH = ( + TASK_ROOT + / "references" + / "anchor_solutions_v1.json" +) + + +def write_json( + path: str, + payload: dict[str, Any], +) -> None: + Path(path).write_text( + json.dumps( + payload, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +def percentile( + values: list[float], + fraction: float, +) -> float: + ordered = sorted(values) + + if len(ordered) == 1: + return ordered[0] + + position = fraction * (len(ordered) - 1) + lower = int(math.floor(position)) + upper = int(math.ceil(position)) + + if lower == upper: + return ordered[lower] + + weight = position - lower + + return ( + ordered[lower] * (1.0 - weight) + + ordered[upper] * weight + ) + + +def run_candidate( + candidate_path: Path, + problem: dict[str, Any], +) -> tuple[Any, dict[str, Any]]: + with tempfile.TemporaryDirectory() as directory: + directory_path = Path(directory) + problem_path = directory_path / "problem.json" + output_path = directory_path / "solution.json" + + problem_path.write_text( + json.dumps(candidate_view(problem), indent=2), + encoding="utf-8", + ) + + try: + result = subprocess.run( + [ + sys.executable, + str(candidate_path), + "--problem", + str(problem_path), + "--output", + str(output_path), + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + except subprocess.TimeoutExpired: + return [], { + "runtime_error": "候选程序运行超时" + } + + if result.returncode != 0: + return [], { + "runtime_error": "候选程序运行失败", + "stdout": result.stdout, + "stderr": result.stderr, + } + + try: + solution = json.loads( + output_path.read_text( + encoding="utf-8", + ) + ) + selected_ids = solution["selected_ids"] + except Exception as error: + return [], { + "runtime_error": ( + f"无法读取候选输出：{error}" + ), + "stdout": result.stdout, + "stderr": result.stderr, + } + + return selected_ids, {} + + +def scenario_score( + candidate_objective: float, + anchor_objective: float, +) -> float: + difference = ( + candidate_objective + - anchor_objective + ) + + # 与固定初始答案相同为50分； + # 优于它超过50，差于它低于50。 + return max( + 0.0, + min( + 100.0, + 50.0 + + 45.0 + * math.tanh(difference / 0.08), + ), + ) + + +def evaluate( + candidate_path: Path, +) -> tuple[dict[str, Any], dict[str, Any]]: + anchor_solutions = json.loads( + ANCHOR_PATH.read_text(encoding="utf-8") + ) + + rows: list[dict[str, Any]] = [] + + for scenario_name, scenario_spec in ( + SCENARIO_SPECS.items() + ): + problem = generate_scenario(scenario_name) + + selected_ids, runtime_details = run_candidate( + candidate_path, + problem, + ) + + candidate_result = score_selection( + problem, + selected_ids, + ) + + if runtime_details: + candidate_result["valid"] = False + candidate_result["errors"].append( + runtime_details["runtime_error"] + ) + + anchor_result = score_selection( + problem, + anchor_solutions[scenario_name], + ) + + formal_score = ( + scenario_score( + candidate_result["objective"], + anchor_result["objective"], + ) + if candidate_result["valid"] + else 0.0 + ) + + diagnostic_score = max( + 0.0, + scenario_score( + candidate_result["objective"], + anchor_result["objective"], + ) + - 8.0 + * len(candidate_result["errors"]) + ) + + row: dict[str, Any] = { + "scenario": scenario_name, + "feedback": scenario_spec["feedback"], + "valid": candidate_result["valid"], + "score": formal_score, + "diagnostic_score": diagnostic_score, + "errors": candidate_result["errors"], + } + + if scenario_spec["feedback"]: + row["candidate"] = candidate_result + row["anchor"] = anchor_result + else: + row["candidate_summary"] = { + "objective": round( + candidate_result["objective"], + 6, + ), + "selected_count": ( + candidate_result[ + "selected_count" + ] + ), + "total_time": ( + candidate_result["total_time"] + ), + "mean_dif": round( + candidate_result["mean_dif"], + 6, + ), + "mean_exposure": round( + candidate_result[ + "mean_exposure" + ], + 6, + ), + } + + rows.append(row) + + formal_scores = [ + float(row["score"]) + for row in rows + ] + + diagnostic_scores = [ + float(row["diagnostic_score"]) + for row in rows + ] + + all_valid = all( + bool(row["valid"]) + for row in rows + ) + + mean_formal = ( + sum(formal_scores) / len(formal_scores) + ) + p20_formal = percentile( + formal_scores, + 0.20, + ) + + mean_diagnostic = ( + sum(diagnostic_scores) + / len(diagnostic_scores) + ) + p20_diagnostic = percentile( + diagnostic_scores, + 0.20, + ) + + robust_formal = ( + 0.75 * mean_formal + + 0.25 * p20_formal + ) + + robust_diagnostic = ( + 0.75 * mean_diagnostic + + 0.25 * p20_diagnostic + ) + + metrics = { + "combined_score": ( + round(robust_formal, 6) + if all_valid + else 0.0 + ), + "diagnostic_score": round( + robust_diagnostic, + 6, + ), + "valid": 1.0 if all_valid else 0.0, + "feasible_scenarios": float( + sum(bool(row["valid"]) for row in rows) + ), + "num_scenarios": float(len(rows)), + "mean_scenario_score": round( + mean_formal, + 6, + ), + "p20_scenario_score": round( + p20_formal, + 6, + ), + } + + artifacts = { + "candidate_path": str(candidate_path), + "feedback_rows": rows, + } + + return metrics, artifacts + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("candidate") + parser.add_argument( + "--metrics-out", + default="metrics.json", + ) + parser.add_argument( + "--artifacts-out", + default="artifacts.json", + ) + args = parser.parse_args() + + metrics, artifacts = evaluate( + Path(args.candidate).resolve() + ) + + write_json(args.metrics_out, metrics) + write_json(args.artifacts_out, artifacts) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/generator.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/generator.py new file mode 100644 index 00000000..c99d545a --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/generator.py @@ -0,0 +1,312 @@ +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path +from typing import Any + + +DOMAINS = ( + "memory", + "reasoning", + "attention", + "executive", +) + +THETA_POINTS = (-1.5, -0.5, 0.5, 1.5) + + +SCENARIO_SPECS: dict[str, dict[str, Any]] = { + "dev_balanced": { + "seed": 6101, + "feedback": True, + "domain_targets": { + "memory": 6, + "reasoning": 6, + "attention": 6, + "executive": 6, + }, + "theta_weights": [0.20, 0.30, 0.30, 0.20], + "max_total_time": 78, + "max_mean_dif": 0.42, + "max_mean_exposure": 0.68, + }, + "dev_low_ability_focus": { + "seed": 6102, + "feedback": True, + "domain_targets": { + "memory": 7, + "reasoning": 6, + "attention": 6, + "executive": 5, + }, + "theta_weights": [0.45, 0.30, 0.15, 0.10], + "max_total_time": 80, + "max_mean_dif": 0.40, + "max_mean_exposure": 0.68, + }, + "dev_high_ability_focus": { + "seed": 6103, + "feedback": True, + "domain_targets": { + "memory": 5, + "reasoning": 7, + "attention": 5, + "executive": 7, + }, + "theta_weights": [0.10, 0.15, 0.30, 0.45], + "max_total_time": 80, + "max_mean_dif": 0.40, + "max_mean_exposure": 0.68, + }, + "dev_fast_screening": { + "seed": 6104, + "feedback": True, + "domain_targets": { + "memory": 6, + "reasoning": 6, + "attention": 7, + "executive": 5, + }, + "theta_weights": [0.20, 0.30, 0.30, 0.20], + "max_total_time": 65, + "max_mean_dif": 0.43, + "max_mean_exposure": 0.70, + }, + "dev_fairness_sensitive": { + "seed": 6105, + "feedback": True, + "domain_targets": { + "memory": 6, + "reasoning": 6, + "attention": 6, + "executive": 6, + }, + "theta_weights": [0.25, 0.25, 0.25, 0.25], + "max_total_time": 80, + "max_mean_dif": 0.27, + "max_mean_exposure": 0.68, + }, + "dev_security_sensitive": { + "seed": 6106, + "feedback": True, + "domain_targets": { + "memory": 6, + "reasoning": 6, + "attention": 6, + "executive": 6, + }, + "theta_weights": [0.20, 0.30, 0.30, 0.20], + "max_total_time": 80, + "max_mean_dif": 0.42, + "max_mean_exposure": 0.43, + }, + "validation_mixed_a": { + "seed": 7101, + "feedback": False, + "domain_targets": { + "memory": 7, + "reasoning": 5, + "attention": 6, + "executive": 6, + }, + "theta_weights": [0.35, 0.30, 0.20, 0.15], + "max_total_time": 73, + "max_mean_dif": 0.34, + "max_mean_exposure": 0.55, + }, + "validation_mixed_b": { + "seed": 7102, + "feedback": False, + "domain_targets": { + "memory": 5, + "reasoning": 7, + "attention": 6, + "executive": 6, + }, + "theta_weights": [0.15, 0.20, 0.30, 0.35], + "max_total_time": 72, + "max_mean_dif": 0.35, + "max_mean_exposure": 0.54, + }, + "validation_mixed_c": { + "seed": 7103, + "feedback": False, + "domain_targets": { + "memory": 6, + "reasoning": 6, + "attention": 7, + "executive": 5, + }, + "theta_weights": [0.25, 0.35, 0.25, 0.15], + "max_total_time": 68, + "max_mean_dif": 0.32, + "max_mean_exposure": 0.58, + }, + "validation_mixed_d": { + "seed": 7104, + "feedback": False, + "domain_targets": { + "memory": 6, + "reasoning": 6, + "attention": 5, + "executive": 7, + }, + "theta_weights": [0.15, 0.25, 0.30, 0.30], + "max_total_time": 70, + "max_mean_dif": 0.33, + "max_mean_exposure": 0.52, + }, +} + + +def generate_item_bank(seed: int) -> list[dict[str, Any]]: + """生成包含80道模拟题的稳定题库。""" + rng = random.Random(seed) + items: list[dict[str, Any]] = [] + + for domain_index, domain in enumerate(DOMAINS): + for local_index in range(20): + item_id = domain_index * 20 + local_index + 1 + + # 前16道题两两共享材料，因此同组最多选一道； + # 最后4道题没有材料冲突。 + if local_index < 16: + enemy_group: str | None = ( + f"{domain}_material_{local_index // 2 + 1}" + ) + else: + enemy_group = None + + item = { + "id": item_id, + "domain": domain, + "content_strand": ( + f"{domain}_strand_{local_index % 4 + 1}" + ), + "time": rng.randint(2, 5), + "discrimination": round( + rng.uniform(0.65, 2.15), + 4, + ), + "difficulty": round( + rng.uniform(-2.4, 2.4), + 4, + ), + "dif_risk": round( + min( + 0.95, + max( + 0.01, + rng.betavariate(1.6, 4.2), + ), + ), + 4, + ), + "exposure": round( + min( + 0.98, + max( + 0.02, + rng.betavariate(2.0, 3.0), + ), + ), + 4, + ), + "enemy_group": enemy_group, + } + items.append(item) + + return items + + +def generate_scenario(name: str) -> dict[str, Any]: + if name not in SCENARIO_SPECS: + available = ", ".join(SCENARIO_SPECS) + raise ValueError( + f"未知场景：{name}。可选场景：{available}" + ) + + spec = SCENARIO_SPECS[name] + + return { + "problem_version": "v1-draft", + "scenario": name, + "seed": spec["seed"], + "feedback": spec["feedback"], + "test_length": sum( + spec["domain_targets"].values() + ), + "domain_targets": spec["domain_targets"], + "theta_points": list(THETA_POINTS), + "theta_weights": spec["theta_weights"], + "max_total_time": spec["max_total_time"], + "max_mean_dif": spec["max_mean_dif"], + "max_mean_exposure": spec["max_mean_exposure"], + "max_items_per_enemy_group": 1, + "items": generate_item_bank(spec["seed"]), + } + + +def print_summary() -> None: + rows = [] + + for name, spec in SCENARIO_SPECS.items(): + rows.append( + { + "scenario": name, + "seed": spec["seed"], + "feedback": spec["feedback"], + "test_length": sum( + spec["domain_targets"].values() + ), + "domain_targets": spec["domain_targets"], + "max_total_time": spec["max_total_time"], + "max_mean_dif": spec["max_mean_dif"], + "max_mean_exposure": ( + spec["max_mean_exposure"] + ), + } + ) + + print( + json.dumps( + rows, + ensure_ascii=False, + indent=2, + ) + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--summary", action="store_true") + parser.add_argument("--scenario") + parser.add_argument("--output") + args = parser.parse_args() + + if args.summary: + print_summary() + return + + if not args.scenario or not args.output: + parser.error( + "生成场景时必须同时提供 " + "--scenario 和 --output" + ) + + scenario = generate_scenario(args.scenario) + + Path(args.output).write_text( + json.dumps( + scenario, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/problem_view.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/problem_view.py new file mode 100644 index 00000000..45d57ee0 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/problem_view.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + + +PRIVATE_FIELDS = { + "problem_version", + "scenario", + "seed", + "feedback", +} + + +def candidate_view( + problem: dict[str, Any], +) -> dict[str, Any]: + """ + 生成交给候选程序的公开题目。 + + 裁判仍保留场景名称、随机种子和反馈类型， + 但候选程序只能看到完成组卷真正需要的信息。 + """ + public_problem = { + key: value + for key, value in problem.items() + if key not in PRIVATE_FIELDS + } + + return deepcopy(public_problem) diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/scoring.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/scoring.py new file mode 100644 index 00000000..c3780a08 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/scoring.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import math +from collections import Counter +from typing import Any + + +def item_information( + item: dict[str, Any], + theta: float, +) -> float: + """二参数逻辑模型下，题目在某能力水平的测量信息量。""" + discrimination = float(item["discrimination"]) + difficulty = float(item["difficulty"]) + + value = discrimination * (theta - difficulty) + value = max(-60.0, min(60.0, value)) + + probability = 1.0 / (1.0 + math.exp(-value)) + + return ( + discrimination + * discrimination + * probability + * (1.0 - probability) + ) + + +def validate_selection( + problem: dict[str, Any], + selected_ids: Any, +) -> tuple[list[dict[str, Any]], list[str], dict[str, Any]]: + errors: list[str] = [] + items_by_id = { + item["id"]: item + for item in problem["items"] + } + + if not isinstance(selected_ids, list): + selected_ids = [] + errors.append("selected_ids 必须是列表") + + clean_ids: list[int] = [] + + for item_id in selected_ids: + if isinstance(item_id, bool) or not isinstance(item_id, int): + errors.append(f"非法题号类型：{item_id!r}") + elif item_id not in items_by_id: + errors.append(f"未知题号：{item_id}") + else: + clean_ids.append(item_id) + + if len(selected_ids) != problem["test_length"]: + errors.append( + f"题目总数错误：要求 {problem['test_length']}，" + f"实际 {len(selected_ids)}" + ) + + if len(clean_ids) != len(set(clean_ids)): + errors.append("存在重复题号") + + unique_ids = list(dict.fromkeys(clean_ids)) + selected = [ + items_by_id[item_id] + for item_id in unique_ids + ] + + domain_counts = Counter( + item["domain"] for item in selected + ) + + for domain, target in problem["domain_targets"].items(): + actual = domain_counts.get(domain, 0) + + if actual != target: + errors.append( + f"{domain} 数量错误：要求 {target}，实际 {actual}" + ) + + enemy_counts = Counter( + item["enemy_group"] + for item in selected + if item["enemy_group"] is not None + ) + + enemy_conflicts = { + group: count + for group, count in enemy_counts.items() + if count > problem["max_items_per_enemy_group"] + } + + if enemy_conflicts: + errors.append(f"存在材料冲突：{enemy_conflicts}") + + total_time = sum( + item["time"] for item in selected + ) + + if selected: + mean_dif = sum( + item["dif_risk"] for item in selected + ) / len(selected) + + mean_exposure = sum( + item["exposure"] for item in selected + ) / len(selected) + else: + mean_dif = 1.0 + mean_exposure = 1.0 + + if total_time > problem["max_total_time"]: + errors.append( + f"总时间超限：{total_time} > " + f"{problem['max_total_time']}" + ) + + if mean_dif > problem["max_mean_dif"]: + errors.append( + f"平均 DIF 风险超限：{mean_dif:.4f} > " + f"{problem['max_mean_dif']:.4f}" + ) + + if mean_exposure > problem["max_mean_exposure"]: + errors.append( + f"平均曝光率超限：{mean_exposure:.4f} > " + f"{problem['max_mean_exposure']:.4f}" + ) + + stats = { + "selected_count": len(selected_ids), + "domain_counts": dict(domain_counts), + "total_time": total_time, + "time_limit": problem["max_total_time"], + "mean_dif": mean_dif, + "dif_limit": problem["max_mean_dif"], + "mean_exposure": mean_exposure, + "exposure_limit": problem["max_mean_exposure"], + "enemy_conflicts": enemy_conflicts, + } + + return selected, errors, stats + + +def score_selection( + problem: dict[str, Any], + selected_ids: Any, +) -> dict[str, Any]: + selected, errors, stats = validate_selection( + problem, + selected_ids, + ) + + if not selected: + return { + "valid": False, + "errors": errors, + "objective": 0.0, + "components": {}, + **stats, + } + + information_curve = [ + sum( + item_information(item, theta) + for item in selected + ) / len(selected) + for theta in problem["theta_points"] + ] + + weighted_information = sum( + weight * information + for weight, information in zip( + problem["theta_weights"], + information_curve, + ) + ) + + worst_information = min(information_curve) + mean_information = ( + sum(information_curve) + / len(information_curve) + ) + + profile_balance = ( + worst_information / mean_information + if mean_information > 0 + else 0.0 + ) + + available_strands = { + (item["domain"], item["content_strand"]) + for item in problem["items"] + } + + selected_strands = { + (item["domain"], item["content_strand"]) + for item in selected + } + + strand_coverage = ( + len(selected_strands) / len(available_strands) + ) + + time_efficiency = max( + 0.0, + 1.0 + - stats["total_time"] / problem["max_total_time"], + ) + + dif_quality = max( + 0.0, + 1.0 - stats["mean_dif"], + ) + + exposure_quality = max( + 0.0, + 1.0 - stats["mean_exposure"], + ) + + objective = ( + 0.50 * weighted_information + + 0.22 * worst_information + + 0.08 * profile_balance + + 0.08 * dif_quality + + 0.05 * exposure_quality + + 0.03 * time_efficiency + + 0.04 * strand_coverage + ) + + return { + "valid": len(errors) == 0, + "errors": errors, + "objective": objective, + "components": { + "information_curve": [ + round(value, 6) + for value in information_curve + ], + "weighted_information": round( + weighted_information, + 6, + ), + "worst_information": round( + worst_information, + 6, + ), + "profile_balance": round( + profile_balance, + 6, + ), + "strand_coverage": round( + strand_coverage, + 6, + ), + "time_efficiency": round( + time_efficiency, + 6, + ), + "dif_quality": round( + dif_quality, + 6, + ), + "exposure_quality": round( + exposure_quality, + 6, + ), + }, + **stats, + } diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py new file mode 100644 index 00000000..de337d4b --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py @@ -0,0 +1,471 @@ +from __future__ import annotations + +import json +import tempfile +from collections import defaultdict +from pathlib import Path +from typing import Any + +from evaluator import evaluate, run_candidate +from generator import SCENARIO_SPECS, generate_scenario +from scoring import score_selection + + +TASK_ROOT = Path(__file__).resolve().parents[1] +BASELINE_PATH = TASK_ROOT / "scripts" / "init.py" +CLEAN_PATH = ( + TASK_ROOT + / "verification" + / "clean_candidate_v1.py" +) +ANCHOR_PATH = ( + TASK_ROOT + / "references" + / "anchor_solutions_v1.json" +) + + +def require(condition: bool, message: str) -> None: + """条件不满足时，立即报告具体测试失败。""" + if not condition: + raise AssertionError(message) + + +def contains_error( + result: dict[str, Any], + keyword: str, +) -> bool: + return any( + keyword in str(error) + for error in result["errors"] + ) + + +def replace_with_wrong_domain( + problem: dict[str, Any], + anchor_ids: list[int], +) -> list[int]: + items_by_id = { + item["id"]: item + for item in problem["items"] + } + + selected_set = set(anchor_ids) + + old_id = next( + item_id + for item_id in anchor_ids + if items_by_id[item_id]["domain"] == "memory" + ) + + new_id = next( + item["id"] + for item in problem["items"] + if ( + item["domain"] == "reasoning" + and item["id"] not in selected_set + ) + ) + + trial = anchor_ids.copy() + trial[trial.index(old_id)] = new_id + return trial + + +def create_enemy_conflict( + problem: dict[str, Any], + anchor_ids: list[int], +) -> list[int]: + items_by_id = { + item["id"]: item + for item in problem["items"] + } + + groups: dict[str, list[int]] = defaultdict(list) + + for item in problem["items"]: + group = item["enemy_group"] + + if group is not None: + groups[group].append(item["id"]) + + for pair_ids in groups.values(): + if len(pair_ids) < 2: + continue + + pair = pair_ids[:2] + domain = items_by_id[pair[0]]["domain"] + + trial = anchor_ids.copy() + missing = [ + item_id + for item_id in pair + if item_id not in trial + ] + + removable = [ + item_id + for item_id in trial + if ( + items_by_id[item_id]["domain"] == domain + and item_id not in pair + ) + ] + + if len(removable) < len(missing): + continue + + for old_id, new_id in zip( + removable, + missing, + ): + trial[trial.index(old_id)] = new_id + + result = score_selection(problem, trial) + + if contains_error(result, "材料冲突"): + return trial + + raise AssertionError( + "测试程序未能构造材料冲突答案" + ) + + +def select_extreme_items( + problem: dict[str, Any], + field: str, +) -> list[int]: + selected_ids: list[int] = [] + + for domain, target in ( + problem["domain_targets"].items() + ): + candidates = [ + item + for item in problem["items"] + if item["domain"] == domain + ] + + candidates.sort( + key=lambda item: ( + -float(item[field]), + item["id"], + ) + ) + + selected_ids.extend( + item["id"] + for item in candidates[:target] + ) + + return selected_ids + + +def main() -> None: + anchors = json.loads( + ANCHOR_PATH.read_text(encoding="utf-8") + ) + + # 1. 同一场景必须每次生成完全相同的题库。 + generated_once = generate_scenario( + "dev_balanced" + ) + generated_twice = generate_scenario( + "dev_balanced" + ) + + require( + generated_once == generated_twice, + "题库生成不具有确定性", + ) + + # 2. 十个固定参照答案都必须合法。 + valid_anchor_count = 0 + + for scenario_name in SCENARIO_SPECS: + problem = generate_scenario(scenario_name) + result = score_selection( + problem, + anchors[scenario_name], + ) + + require( + result["valid"], + ( + f"参照答案在 {scenario_name} " + f"中不合法：{result['errors']}" + ), + ) + valid_anchor_count += 1 + + # 3. 普通算法对相同场景必须给出相同答案。 + baseline_problem = generate_scenario( + "dev_balanced" + ) + + first_ids, first_error = run_candidate( + BASELINE_PATH, + baseline_problem, + ) + second_ids, second_error = run_candidate( + BASELINE_PATH, + baseline_problem, + ) + + require( + not first_error and not second_error, + "普通算法运行失败", + ) + require( + first_ids == second_ids, + "普通算法不具有确定性", + ) + + # 4. 正式运行普通算法，应为有效的50分。 + baseline_metrics, _ = evaluate( + BASELINE_PATH + ) + + require( + baseline_metrics["valid"] == 1.0, + "普通算法被正式裁判判为无效", + ) + require( + abs( + baseline_metrics["combined_score"] + - 50.0 + ) + < 1e-9, + "普通算法没有保持50分参照线", + ) + + # 5. 独立改进算法必须明显优于普通算法。 + clean_metrics, _ = evaluate(CLEAN_PATH) + + require( + clean_metrics["valid"] == 1.0, + "独立改进算法存在无效场景", + ) + require( + clean_metrics["combined_score"] >= 65.0, + ( + "独立算法提升不足：" + f"{clean_metrics['combined_score']}" + ), + ) + + # 6. 精确检查几类错误答案。 + problem = generate_scenario( + "dev_balanced" + ) + anchor_ids = anchors["dev_balanced"] + + duplicate_result = score_selection( + problem, + [1] * problem["test_length"], + ) + require( + not duplicate_result["valid"] + and contains_error( + duplicate_result, + "重复题号", + ), + "裁判没有识别重复题号", + ) + + unknown_ids = anchor_ids.copy() + unknown_ids[-1] = 999 + + unknown_result = score_selection( + problem, + unknown_ids, + ) + require( + not unknown_result["valid"] + and contains_error( + unknown_result, + "未知题号", + ), + "裁判没有识别不存在的题号", + ) + + wrong_domain_ids = replace_with_wrong_domain( + problem, + anchor_ids, + ) + wrong_domain_result = score_selection( + problem, + wrong_domain_ids, + ) + require( + not wrong_domain_result["valid"] + and contains_error( + wrong_domain_result, + "数量错误", + ), + "裁判没有识别领域数量错误", + ) + + conflict_ids = create_enemy_conflict( + problem, + anchor_ids, + ) + conflict_result = score_selection( + problem, + conflict_ids, + ) + require( + not conflict_result["valid"] + and contains_error( + conflict_result, + "材料冲突", + ), + "裁判没有识别相似材料冲突", + ) + + fast_problem = generate_scenario( + "dev_fast_screening" + ) + slow_ids = select_extreme_items( + fast_problem, + "time", + ) + slow_result = score_selection( + fast_problem, + slow_ids, + ) + require( + not slow_result["valid"] + and contains_error( + slow_result, + "总时间超限", + ), + "裁判没有识别总时间超限", + ) + + fairness_problem = generate_scenario( + "dev_fairness_sensitive" + ) + high_dif_ids = select_extreme_items( + fairness_problem, + "dif_risk", + ) + high_dif_result = score_selection( + fairness_problem, + high_dif_ids, + ) + require( + not high_dif_result["valid"] + and contains_error( + high_dif_result, + "DIF", + ), + "裁判没有识别 DIF 风险超限", + ) + + security_problem = generate_scenario( + "dev_security_sensitive" + ) + high_exposure_ids = select_extreme_items( + security_problem, + "exposure", + ) + high_exposure_result = score_selection( + security_problem, + high_exposure_ids, + ) + require( + not high_exposure_result["valid"] + and contains_error( + high_exposure_result, + "曝光率", + ), + "裁判没有识别曝光率超限", + ) + + # 7. 进行一次端到端错误程序测试。 + with tempfile.TemporaryDirectory() as directory: + bad_candidate = ( + Path(directory) + / "bad_candidate.py" + ) + + bad_candidate.write_text( + """ +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("--problem", required=True) +parser.add_argument("--output", required=True) +args = parser.parse_args() + +problem = json.loads( + Path(args.problem).read_text() +) + +Path(args.output).write_text( + json.dumps({ + "selected_ids": [ + 1 + ] * problem["test_length"] + }) +) +""".strip(), + encoding="utf-8", + ) + + bad_metrics, _ = evaluate( + bad_candidate + ) + + require( + bad_metrics["valid"] == 0.0, + "端到端裁判错误地接受了非法程序", + ) + require( + bad_metrics["combined_score"] == 0.0, + "非法程序的正式得分没有归零", + ) + + summary = { + "all_tests_passed": True, + "deterministic_generator": True, + "deterministic_baseline": True, + "valid_anchor_scenarios": ( + valid_anchor_count + ), + "baseline_score": ( + baseline_metrics["combined_score"] + ), + "clean_candidate_score": ( + clean_metrics["combined_score"] + ), + "rejected_error_types": [ + "duplicate_item_ids", + "unknown_item_id", + "wrong_domain_counts", + "enemy_group_conflict", + "time_limit", + "dif_limit", + "exposure_limit", + "end_to_end_invalid_candidate", + ], + } + + print( + json.dumps( + summary, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() + + + +task="$HOME/projects/Frontier-Engineering/benchmarks/AssessmentEngineering/RobustTestAssembly" From 6b7d5d8b77b73f82dd54b4ecd88a05c47a1682bf Mon Sep 17 00:00:00 2001 From: HQHJT999 <279831547+HQHJT999@users.noreply.github.com> Date: Sat, 22 Aug 2026 07:09:05 +0800 Subject: [PATCH 2/2] fix: address reviewer feedback for RobustTestAssembly --- .../frontier_eval/copy_files.txt | 1 + .../frontier_eval/eval_cwd.txt | 1 + .../frontier_eval/readonly_files.txt | 2 +- .../references/baseline_init_v1.py | 3 +- .../clean_candidate_v1.py | 0 .../verification/evaluator.py | 59 +++++++- .../verification/test_task_v1.py | 129 ++++++++++++++---- 7 files changed, 163 insertions(+), 32 deletions(-) create mode 100644 benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_cwd.txt rename benchmarks/AssessmentEngineering/RobustTestAssembly/{verification => references}/clean_candidate_v1.py (100%) diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt index 0f5a883a..60989cdc 100644 --- a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/copy_files.txt @@ -4,3 +4,4 @@ verification/generator.py verification/scoring.py verification/problem_view.py references/anchor_solutions_v1.json +references/baseline_init_v1.py diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_cwd.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt index 871e81a0..72321967 100644 --- a/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/frontier_eval/readonly_files.txt @@ -8,7 +8,7 @@ verification/scoring.py verification/problem_view.py verification/test_task_v1.py verification/build_anchor_v1.py -verification/clean_candidate_v1.py +references/clean_candidate_v1.py references/anchor_solutions_v1.json references/baseline_init_v1.py references/design_notes.md diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py index 87d6e791..725f564e 100644 --- a/benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/baseline_init_v1.py @@ -295,6 +295,8 @@ def select_items( return [item["id"] for item in selected] +# EVOLVE-BLOCK-END + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--problem", required=True) @@ -321,4 +323,3 @@ def main() -> None: if __name__ == "__main__": main() -# EVOLVE-BLOCK-END diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/clean_candidate_v1.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/references/clean_candidate_v1.py similarity index 100% rename from benchmarks/AssessmentEngineering/RobustTestAssembly/verification/clean_candidate_v1.py rename to benchmarks/AssessmentEngineering/RobustTestAssembly/references/clean_candidate_v1.py diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py index 6b4c7962..535e49eb 100644 --- a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/evaluator.py @@ -3,6 +3,10 @@ import argparse import json import math +try: + import resource +except ImportError: + resource = None import subprocess import sys import tempfile @@ -15,11 +19,30 @@ TASK_ROOT = Path(__file__).resolve().parents[1] +BASELINE_PATH = TASK_ROOT / "references" / "baseline_init_v1.py" ANCHOR_PATH = ( TASK_ROOT / "references" / "anchor_solutions_v1.json" ) +START_MARKER = "# EVOLVE-BLOCK-START" +END_MARKER = "# EVOLVE-BLOCK-END" + + +def frozen_regions(path: Path) -> tuple[str, str] | None: + before, start, remainder = path.read_text( + encoding="utf-8" + ).partition(START_MARKER) + _, end, after = remainder.partition(END_MARKER) + return (before, after) if start and end else None + + +def limit_resources() -> None: + if resource is None: + return + resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) + resource.setrlimit(resource.RLIMIT_AS, (1 << 30, 1 << 30)) + resource.setrlimit(resource.RLIMIT_NPROC, (32, 32)) def write_json( @@ -88,6 +111,7 @@ def run_candidate( text=True, timeout=10, check=False, + preexec_fn=limit_resources if resource is not None else None, ) except subprocess.TimeoutExpired: return [], { @@ -145,6 +169,13 @@ def scenario_score( def evaluate( candidate_path: Path, ) -> tuple[dict[str, Any], dict[str, Any]]: + boundary_error = ( + "candidate modified code outside EVOLVE-BLOCK" + if frozen_regions(candidate_path) + != frozen_regions(BASELINE_PATH) + else "" + ) + anchor_solutions = json.loads( ANCHOR_PATH.read_text(encoding="utf-8") ) @@ -156,10 +187,30 @@ def evaluate( ): problem = generate_scenario(scenario_name) - selected_ids, runtime_details = run_candidate( - candidate_path, - problem, - ) + if boundary_error: + selected_ids, runtime_details = [], { + "runtime_error": boundary_error + } + else: + selected_ids, runtime_details = run_candidate( + candidate_path, + problem, + ) + + if not runtime_details: + repeated_ids, repeated_details = run_candidate( + candidate_path, + problem, + ) + + if repeated_details: + runtime_details = repeated_details + elif selected_ids != repeated_ids: + runtime_details = { + "runtime_error": ( + "candidate output is nondeterministic" + ) + } candidate_result = score_selection( problem, diff --git a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py index de337d4b..b8ef211c 100644 --- a/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py +++ b/benchmarks/AssessmentEngineering/RobustTestAssembly/verification/test_task_v1.py @@ -6,7 +6,12 @@ from pathlib import Path from typing import Any -from evaluator import evaluate, run_candidate +from evaluator import ( + END_MARKER, + START_MARKER, + evaluate, + run_candidate, +) from generator import SCENARIO_SPECS, generate_scenario from scoring import score_selection @@ -15,7 +20,7 @@ BASELINE_PATH = TASK_ROOT / "scripts" / "init.py" CLEAN_PATH = ( TASK_ROOT - / "verification" + / "references" / "clean_candidate_v1.py" ) ANCHOR_PATH = ( @@ -221,6 +226,95 @@ def main() -> None: "普通算法不具有确定性", ) + baseline_source = BASELINE_PATH.read_text( + encoding="utf-8" + ) + + with tempfile.TemporaryDirectory( + dir=TASK_ROOT + ) as directory: + directory_path = Path(directory) + + inside_candidate = directory_path / "inside.py" + inside_candidate.write_text( + baseline_source.replace( + "def item_information(", + "# allowed change\ndef item_information(", + 1, + ), + encoding="utf-8", + ) + inside_metrics, _ = evaluate(inside_candidate) + + require( + inside_metrics["valid"] == 1.0, + "change inside EVOLVE-BLOCK was rejected", + ) + + outside_candidate = directory_path / "outside.py" + outside_candidate.write_text( + baseline_source.replace( + "parser = argparse.ArgumentParser()", + "parser = argparse.ArgumentParser(description='changed')", + 1, + ), + encoding="utf-8", + ) + outside_metrics, _ = evaluate(outside_candidate) + + require( + outside_metrics["valid"] == 0.0 + and outside_metrics["combined_score"] == 0.0, + "change outside EVOLVE-BLOCK was accepted", + ) + + nondeterministic_candidate = ( + directory_path / "nondeterministic.py" + ) + nondeterministic_candidate.write_text( + baseline_source.replace( + ' return [item["id"] for item in selected]', + """ state_path = Path(__file__).with_suffix(".state") + if state_path.exists(): + state_path.unlink() + selected.reverse() + else: + state_path.touch() + + return [item["id"] for item in selected]""", + 1, + ), + encoding="utf-8", + ) + nondeterministic_metrics, _ = evaluate( + nondeterministic_candidate + ) + + require( + nondeterministic_metrics["valid"] == 0.0 + and nondeterministic_metrics["combined_score"] == 0.0, + "nondeterministic candidate was accepted", + ) + + clean_candidate = directory_path / "clean.py" + clean_block = CLEAN_PATH.read_text( + encoding="utf-8" + ).split("\ndef main() -> None:", 1)[0] + baseline_after = baseline_source.partition( + END_MARKER + )[2] + clean_candidate.write_text( + START_MARKER + + "\n" + + clean_block + + "\n\ndef select_items(problem: dict[str, Any]) -> list[int]:\n" + + " return improve_selection(problem)\n\n" + + END_MARKER + + baseline_after, + encoding="utf-8", + ) + clean_metrics, _ = evaluate(clean_candidate) + # 4. 正式运行普通算法，应为有效的50分。 baseline_metrics, _ = evaluate( BASELINE_PATH @@ -240,8 +334,6 @@ def main() -> None: ) # 5. 独立改进算法必须明显优于普通算法。 - clean_metrics, _ = evaluate(CLEAN_PATH) - require( clean_metrics["valid"] == 1.0, "独立改进算法存在无效场景", @@ -391,28 +483,11 @@ def main() -> None: ) bad_candidate.write_text( - """ -import argparse -import json -from pathlib import Path - -parser = argparse.ArgumentParser() -parser.add_argument("--problem", required=True) -parser.add_argument("--output", required=True) -args = parser.parse_args() - -problem = json.loads( - Path(args.problem).read_text() -) - -Path(args.output).write_text( - json.dumps({ - "selected_ids": [ - 1 - ] * problem["test_length"] - }) -) -""".strip(), + baseline_source.replace( + ' return [item["id"] for item in selected]', + ' return [1] * problem["test_length"]', + 1, + ), encoding="utf-8", ) @@ -451,6 +526,8 @@ def main() -> None: "dif_limit", "exposure_limit", "end_to_end_invalid_candidate", + "outside_evolve_block", + "nondeterministic_candidate", ], }