From 253f491d404f7a11eea80ac28b0bcbcef84049aa Mon Sep 17 00:00:00 2001 From: Yunn1633 <165199018+Yunn1633@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:48:20 +0800 Subject: [PATCH] feat: add WNTR pump scheduling benchmark --- .../PumpScheduling/README.md | 56 ++++ .../PumpScheduling/README_zh-CN.md | 24 ++ .../PumpScheduling/THIRD_PARTY_NOTICES.md | 8 + .../WaterDistribution/PumpScheduling/Task.md | 75 +++++ .../PumpScheduling/Task_zh-CN.md | 56 ++++ .../frontier_eval/agent_files.txt | 9 + .../frontier_eval/artifact_files.txt | 1 + .../frontier_eval/candidate_destination.txt | 1 + .../frontier_eval/constraints.txt | 7 + .../frontier_eval/copy_files.txt | 1 + .../frontier_eval/eval_command.txt | 1 + .../PumpScheduling/frontier_eval/eval_cwd.txt | 1 + .../frontier_eval/evaluate_submission.py | 24 ++ .../frontier_eval/initial_program.txt | 1 + .../frontier_eval/readonly_files.txt | 12 + .../PumpScheduling/frontier_eval/run_eval.sh | 4 + .../PumpScheduling/references/provenance.md | 8 + .../references/scenario_schema.json | 12 + .../references/scenarios_public.json | 5 + .../references/service_nodes.json | 4 + .../PumpScheduling/scripts/init.py | 24 ++ .../PumpScheduling/verification/baseline.py | 17 + .../verification/controller_worker.py | 45 +++ .../PumpScheduling/verification/evaluate.py | 7 + .../PumpScheduling/verification/evaluator.py | 41 +++ .../verification/requirements.txt | 4 + .../PumpScheduling/verification/rollout.py | 313 ++++++++++++++++++ .../verification/scenario_loader.py | 25 ++ .../PumpScheduling/verification/scoring.py | 20 ++ .../verification/tests/test_contract.py | 36 ++ .../tests/test_evaluator_output.py | 53 +++ .../verification/tests/test_scenarios.py | 40 +++ .../verification/tests/test_scoring.py | 15 + benchmarks/WaterDistribution/README.md | 8 + benchmarks/WaterDistribution/README_zh-CN.md | 8 + ...2026-07-13-wntr-compliance-fixes-design.md | 69 ++++ ...07-13-wntr-continuous-hydraulics-design.md | 94 ++++++ ...-07-13-wntr-doc-audit-compliance-design.md | 54 +++ .../2026-07-13-wntr-domain-index-design.md | 19 ++ .../2026-07-13-wntr-pump-scheduling-design.md | 254 ++++++++++++++ 40 files changed, 1456 insertions(+) create mode 100644 benchmarks/WaterDistribution/PumpScheduling/README.md create mode 100644 benchmarks/WaterDistribution/PumpScheduling/README_zh-CN.md create mode 100644 benchmarks/WaterDistribution/PumpScheduling/THIRD_PARTY_NOTICES.md create mode 100644 benchmarks/WaterDistribution/PumpScheduling/Task.md create mode 100644 benchmarks/WaterDistribution/PumpScheduling/Task_zh-CN.md create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/agent_files.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/artifact_files.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/candidate_destination.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/constraints.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/copy_files.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_command.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_cwd.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/evaluate_submission.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/initial_program.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/readonly_files.txt create mode 100755 benchmarks/WaterDistribution/PumpScheduling/frontier_eval/run_eval.sh create mode 100644 benchmarks/WaterDistribution/PumpScheduling/references/provenance.md create mode 100644 benchmarks/WaterDistribution/PumpScheduling/references/scenario_schema.json create mode 100644 benchmarks/WaterDistribution/PumpScheduling/references/scenarios_public.json create mode 100644 benchmarks/WaterDistribution/PumpScheduling/references/service_nodes.json create mode 100644 benchmarks/WaterDistribution/PumpScheduling/scripts/init.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/baseline.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/controller_worker.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/evaluate.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/evaluator.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/requirements.txt create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/rollout.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/scenario_loader.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/scoring.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_contract.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_evaluator_output.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scenarios.py create mode 100644 benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scoring.py create mode 100644 benchmarks/WaterDistribution/README.md create mode 100644 benchmarks/WaterDistribution/README_zh-CN.md create mode 100644 docs/superpowers/specs/2026-07-13-wntr-compliance-fixes-design.md create mode 100644 docs/superpowers/specs/2026-07-13-wntr-continuous-hydraulics-design.md create mode 100644 docs/superpowers/specs/2026-07-13-wntr-doc-audit-compliance-design.md create mode 100644 docs/superpowers/specs/2026-07-13-wntr-domain-index-design.md create mode 100644 docs/superpowers/specs/2026-07-13-wntr-pump-scheduling-design.md diff --git a/benchmarks/WaterDistribution/PumpScheduling/README.md b/benchmarks/WaterDistribution/PumpScheduling/README.md new file mode 100644 index 00000000..8b08eecd --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/README.md @@ -0,0 +1,56 @@ +# WaterDistribution/PumpScheduling + +A CPU-only closed-loop pump-scheduling benchmark on EPANET Net3 through WNTR. +The unified benchmark ID is `WaterDistribution/PumpScheduling`. + +## Environment + +From the repository root, create the repository-owned task runtime. The task is +tested with Python 3.11 on Linux. + +```bash +python3.11 -m venv .venvs/frontier-wntr +.venvs/frontier-wntr/bin/python -m pip install \ + -r benchmarks/WaterDistribution/PumpScheduling/verification/requirements.txt +``` + +## Direct evaluation + +From the benchmark directory: + +```bash +../../../.venvs/frontier-wntr/bin/python verification/evaluator.py \ + scripts/init.py --json-out metrics.json --artifacts-out artifacts.json +``` + +## Unified baseline evaluation + +From the repository root: + +```bash +python -m frontier_eval \ + task=unified \ + task.benchmark=WaterDistribution/PumpScheduling \ + task.runtime.python_path=uv-env:frontier-wntr \ + algorithm=openevolve \ + algorithm.iterations=0 +``` + +Baseline-only validation does not require a model API key. + +## Tests + +From the benchmark directory: + +```bash +../../../.venvs/frontier-wntr/bin/python -m pytest -q verification/tests +``` + +## Resources and execution assumptions + +The evaluator uses three documented public scenarios and three frozen hidden +variants. It requires CPU only, WNTR 1.4.0 and its EPANET runtime; no GPU, +Docker, external dataset, or network access is required. The evaluator executes +on the host. Candidate controllers run in an isolated subprocess with only the +JSON causal-observation interface, but the evaluator itself should still be run +only from a trusted benchmark checkout. diff --git a/benchmarks/WaterDistribution/PumpScheduling/README_zh-CN.md b/benchmarks/WaterDistribution/PumpScheduling/README_zh-CN.md new file mode 100644 index 00000000..9410aa53 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/README_zh-CN.md @@ -0,0 +1,24 @@ +# 配水系统泵站调度 + +基于 WNTR/EPANET Net3 的纯 CPU、闭环泵站调度基准。统一任务 ID 为 +`WaterDistribution/PumpScheduling`。安装依赖后直接评测： + +```bash +python -m pip install -r verification/requirements.txt +python verification/evaluator.py scripts/init.py --json-out metrics.json --artifacts-out artifacts.json +``` + +从仓库根目录运行统一基线评测： + +```bash +python -m frontier_eval \ + task=unified \ + task.benchmark=WaterDistribution/PumpScheduling \ + task.runtime.python_path=/path/to/wntr-python \ + algorithm=openevolve \ + algorithm.iterations=0 +``` + +任务仅需 CPU、WNTR 1.4.0 和 EPANET 运行时，不需要 GPU、Docker、外部数据集 +或网络。候选控制器在隔离子进程中仅通过 JSON 因果观测接口执行；评测器本身 +仍应只从可信的 benchmark checkout 运行。 diff --git a/benchmarks/WaterDistribution/PumpScheduling/THIRD_PARTY_NOTICES.md b/benchmarks/WaterDistribution/PumpScheduling/THIRD_PARTY_NOTICES.md new file mode 100644 index 00000000..2e263e40 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/THIRD_PARTY_NOTICES.md @@ -0,0 +1,8 @@ +# Third-party notices + +- WNTR is distributed under the BSD 3-Clause license. +- EPANET is public-domain software developed by the US EPA. +- Net3 is an EPANET example network distributed with WNTR. This task references + WNTR's installed copy and does not redistribute the network file. + +See `references/provenance.md` for exact versions and source links. diff --git a/benchmarks/WaterDistribution/PumpScheduling/Task.md b/benchmarks/WaterDistribution/PumpScheduling/Task.md new file mode 100644 index 00000000..f1168410 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/Task.md @@ -0,0 +1,75 @@ +# Net3 Pump Scheduling + +## Engineering problem and value + +Drinking-water utilities must operate pumps while maintaining service pressure +and storage reserves. Pump operation consumes electricity, creates demand peaks, +and incurs mechanical wear when speeds change. This benchmark asks for a causal +closed-loop controller that balances those costs against hydraulic feasibility +under changing demand, tariffs, starting storage, and small leaks. + +## Physical model + +The evaluator uses the EPANET Net3 example network distributed with WNTR 1.4.0. +Each deterministic scenario uses one continuous 24-hour EPANET hydraulic +session with hourly control intervals and five-minute hydraulic steps; the +network and its tank state are not recreated between intervals. +The controller sets the speeds of pumps `10` and `335`; link `330` follows the +existing Net3 relationship with pump `335`. The evaluator removes the original +Net3 controls so that the submitted controller supplies the hourly actions. + +## Editable program and input + +Edit only the code between `EVOLVE-BLOCK-START` and `EVOLVE-BLOCK-END` in +`scripts/init.py`, preserving `control(observation)`. The controller receives one +JSON-compatible dictionary per hour containing only causal information: + +| Field | Type | Meaning | +|---|---|---| +| `hour` | integer | Current interval, from 0 through 23 | +| `tank_levels_m` | mapping | Current pressure-head level in metres for tanks `1`, `2`, and `3` | +| `tariff` | number | Current hourly electricity-price coefficient | +| `demand_multiplier` | number | Current dimensionless network demand multiplier | +| `previous_action` | mapping | Previous speeds for pumps `10` and `335`; both are 0 initially | + +The candidate runs in an isolated subprocess and may use only this observation +and state derived from earlier observations in the same scenario. It cannot read +the verifier, benchmark files, or hidden scenarios. + +## Output + +Return a dictionary with exactly the string keys `10` and `335`. Each value is a +finite pump-speed multiplier in the closed interval `[0, 1]`. + +## Hard constraints + +Every public and hidden scenario must: + +- complete all 24 hydraulic intervals without controller, protocol, or EPANET + failure; +- keep pressure at or above 20 m at every declared service node: `153`, `15`, + `253`, `103`, `127`, `101`, `129`, `251`, `255`, and `105`; +- keep tanks `1`, `2`, and `3` within their Net3 minimum and maximum levels; +- finish each tank no more than 0.25 m below its scenario initial level. + +Any failed scenario makes the complete candidate invalid and gives it zero +aggregate score. + +## Objective and scoring + +For each valid scenario the evaluator measures hourly energy cost, peak pump +power in kW, total speed switching, and terminal storage deficit in metres. Each +metric is compared with the frozen shipped baseline using +`baseline_metric / candidate_metric`, clipped to `[0, 1.5]`. The scenario score +is: + +```text +100 * (0.55 * energy_ratio + + 0.20 * peak_power_ratio + + 0.10 * switching_ratio + + 0.15 * terminal_recovery_ratio) +``` + +The six scenario scores aggregate as `0.70 * mean + 0.30 * worst_case`. +`score` therefore has a theoretical range of 0 to 150 and higher is better. +The unified `combined_score` is `score / 100`; the shipped baseline scores 1.0. diff --git a/benchmarks/WaterDistribution/PumpScheduling/Task_zh-CN.md b/benchmarks/WaterDistribution/PumpScheduling/Task_zh-CN.md new file mode 100644 index 00000000..3e54da61 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/Task_zh-CN.md @@ -0,0 +1,56 @@ +# Net3 泵站调度 + +## 工程问题与价值 + +饮用水配水系统需要在保证用户压力和水箱储备的同时调度水泵。水泵运行会产生 +电费与功率峰值，频繁改变泵速还会增加机械磨损。本任务要求设计因果闭环控制器， +在需求、电价、初始储水量和小型泄漏变化下兼顾运行成本与水力可行性。 + +## 物理模型 + +评测使用 WNTR 1.4.0 自带的 EPANET Net3 示例网络。每个确定性场景在单个 +连续 24 小时 EPANET 水力会话中仿真，每小时控制一次并以五分钟水力步长 +推进；区间之间不重建管网或水箱状态。控制器设置水泵 `10` 和 +`335` 的速度；连接 `330` 按 Net3 中与泵 `335` 的关系切换。评测器移除 Net3 +原始控制规则，由候选控制器提供逐小时动作。 + +## 可编辑程序与输入 + +仅修改 `scripts/init.py` 中 `EVOLVE-BLOCK-START` 与 `EVOLVE-BLOCK-END` 之间的 +代码，并保留 `control(observation)`。每小时收到一个仅含因果信息的字典： + +| 字段 | 类型 | 含义 | +|---|---|---| +| `hour` | 整数 | 当前区间，取值 0 至 23 | +| `tank_levels_m` | 映射 | 水箱 `1`、`2`、`3` 当前以米计的压力水头水位 | +| `tariff` | 数值 | 当前小时电价系数 | +| `demand_multiplier` | 数值 | 当前无量纲网络需求倍率 | +| `previous_action` | 映射 | 水泵 `10`、`335` 上一小时速度；初始均为 0 | + +候选程序在隔离子进程中运行，只能使用当前观测以及同一场景中过去观测形成的状态， +不能读取验证器、benchmark 文件或隐藏场景。 + +## 输出 + +返回恰好包含字符串键 `10`、`335` 的字典；每个值必须是闭区间 `[0,1]` 内的 +有限泵速倍率。 + +## 硬约束 + +每个公开及隐藏场景均必须完成全部 24 个区间。指定服务节点 `153`、`15`、 +`253`、`103`、`127`、`101`、`129`、`251`、`255` 和 `105` 的压力均不得低于 +20 m；三个水箱不得超出 Net3 上下界，且每个水箱终态不得比该场景初态低超过 +0.25 m。任一场景失败都会使整个候选无效并得到零聚合分数。 + +## 目标与评分 + +每个有效场景测量逐小时能源成本、kW 峰值功率、泵速切换总量和以米计的终态储水 +缺口。各项以 `冻结基线指标 / 候选指标` 归一化并截断到 `[0,1.5]`，场景分数为： + +```text +100 * (0.55 * 能源比率 + 0.20 * 峰值功率比率 + + 0.10 * 切换比率 + 0.15 * 终态恢复比率) +``` + +六个场景按 `0.70 * 平均分 + 0.30 * 最差分` 聚合。`score` 理论范围为 0 至 +150，越高越好；统一评测的 `combined_score = score / 100`，初始基线为 1.0。 diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/agent_files.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/agent_files.txt new file mode 100644 index 00000000..26f2f17d --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/agent_files.txt @@ -0,0 +1,9 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +scripts/init.py +references/scenarios_public.json +references/service_nodes.json +references/provenance.md +frontier_eval/constraints.txt diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/artifact_files.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..6a107a0b --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/artifact_files.txt @@ -0,0 +1 @@ +# metrics.json and artifacts.json are handled by UnifiedTask. diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/candidate_destination.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/candidate_destination.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/candidate_destination.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/constraints.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/constraints.txt new file mode 100644 index 00000000..3cbabdb5 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/constraints.txt @@ -0,0 +1,7 @@ +PumpScheduling unified constraints: +1) Modify only code between EVOLVE-BLOCK-START and EVOLVE-BLOCK-END in scripts/init.py. +2) Preserve control(observation) and return exactly pump keys 10 and 335. +3) Pump speeds must be finite numbers in [0, 1]. +4) Use only the causal observation; do not access evaluator or hidden-case files. +5) All six scenarios must converge and satisfy pressure, tank-bound, and terminal-storage constraints. +6) Do not modify documentation, references, verification, tests, or frontier_eval metadata. diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/copy_files.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/copy_files.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/copy_files.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_command.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_command.txt new file mode 100644 index 00000000..fee2365a --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +{python} verification/evaluator.py {candidate} --json-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_cwd.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/evaluate_submission.py b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/evaluate_submission.py new file mode 100644 index 00000000..9e24f4bd --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/evaluate_submission.py @@ -0,0 +1,24 @@ +"""Python compatibility entry point for task-local evaluation.""" + +import subprocess +import sys +from pathlib import Path + + +def main(): + task_root = Path(__file__).parents[1] + candidate = sys.argv[1] if len(sys.argv) > 1 else "scripts/init.py" + command = [ + sys.executable, + "verification/evaluator.py", + candidate, + "--json-out", + "metrics.json", + "--artifacts-out", + "artifacts.json", + ] + return subprocess.call(command, cwd=task_root) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/initial_program.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/initial_program.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/readonly_files.txt b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..56d3f555 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/readonly_files.txt @@ -0,0 +1,12 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +THIRD_PARTY_NOTICES.md +references +verification +frontier_eval +frontier_eval/run_eval.sh +verification/evaluate.py +verification/evaluator.py +frontier_eval/evaluate_submission.py diff --git a/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/run_eval.sh b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/run_eval.sh new file mode 100755 index 00000000..ca9fe624 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/frontier_eval/run_eval.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +candidate="${1:-scripts/init.py}" +"${PYTHON:-python}" verification/evaluator.py "$candidate" --json-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/WaterDistribution/PumpScheduling/references/provenance.md b/benchmarks/WaterDistribution/PumpScheduling/references/provenance.md new file mode 100644 index 00000000..dbc52c76 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/references/provenance.md @@ -0,0 +1,8 @@ +# Provenance + +The hydraulic model is the Net3 example bundled with WNTR 1.4.0 and loaded by +`WaterNetworkModel("Net3")`. WNTR source and licensing: +https://github.com/USEPA/WNTR . EPANET source: +https://github.com/USEPA/EPANET2.2 . Scenario demand and tariff arrays are +original synthetic benchmark data committed in this directory; no external +operational data is included. diff --git a/benchmarks/WaterDistribution/PumpScheduling/references/scenario_schema.json b/benchmarks/WaterDistribution/PumpScheduling/references/scenario_schema.json new file mode 100644 index 00000000..1f05f40f --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/references/scenario_schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["id", "demand_multipliers", "tariff", "initial_tank_offsets_m", "leak"], + "properties": { + "id": {"type": "string"}, + "demand_multipliers": {"type": "array", "minItems": 24, "maxItems": 24, "items": {"type": "number", "minimum": 0.5, "maximum": 1.6}}, + "tariff": {"type": "array", "minItems": 24, "maxItems": 24, "items": {"type": "number", "minimum": 0.01}}, + "initial_tank_offsets_m": {"type": "object"}, + "leak": {"type": ["object", "null"]} + } +} diff --git a/benchmarks/WaterDistribution/PumpScheduling/references/scenarios_public.json b/benchmarks/WaterDistribution/PumpScheduling/references/scenarios_public.json new file mode 100644 index 00000000..80bf8091 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/references/scenarios_public.json @@ -0,0 +1,5 @@ +[ + {"id":"public_normal","demand_multipliers":[0.72,0.68,0.65,0.64,0.68,0.78,0.92,1.08,1.15,1.12,1.05,1.00,1.04,1.08,1.10,1.12,1.18,1.28,1.32,1.24,1.10,0.96,0.84,0.76],"tariff":[0.10,0.10,0.10,0.10,0.10,0.10,0.14,0.18,0.22,0.22,0.18,0.16,0.16,0.16,0.18,0.22,0.30,0.30,0.30,0.24,0.20,0.16,0.12,0.10],"initial_tank_offsets_m":{"1":0,"2":0,"3":0},"leak":null}, + {"id":"public_peak","demand_multipliers":[0.75,0.70,0.68,0.67,0.72,0.84,1.02,1.22,1.34,1.30,1.17,1.08,1.10,1.14,1.18,1.23,1.32,1.44,1.48,1.38,1.20,1.02,0.90,0.80],"tariff":[0.08,0.08,0.08,0.08,0.08,0.10,0.14,0.20,0.28,0.28,0.22,0.18,0.18,0.18,0.22,0.28,0.36,0.36,0.36,0.30,0.22,0.16,0.10,0.08],"initial_tank_offsets_m":{"1":0.2,"2":0.1,"3":0.1},"leak":null}, + {"id":"public_low_storage","demand_multipliers":[0.70,0.68,0.66,0.65,0.70,0.82,0.98,1.14,1.20,1.16,1.08,1.02,1.05,1.10,1.13,1.18,1.24,1.34,1.38,1.30,1.16,1.00,0.88,0.78],"tariff":[0.10,0.10,0.10,0.10,0.10,0.12,0.16,0.22,0.28,0.28,0.20,0.17,0.17,0.17,0.20,0.25,0.32,0.32,0.32,0.26,0.20,0.16,0.12,0.10],"initial_tank_offsets_m":{"1":-0.4,"2":-0.5,"3":-0.5},"leak":null} +] diff --git a/benchmarks/WaterDistribution/PumpScheduling/references/service_nodes.json b/benchmarks/WaterDistribution/PumpScheduling/references/service_nodes.json new file mode 100644 index 00000000..2792a752 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/references/service_nodes.json @@ -0,0 +1,4 @@ +{ + "service_nodes": ["153", "15", "253", "103", "127", "101", "129", "251", "255", "105"], + "minimum_pressure_m": 20.0 +} diff --git a/benchmarks/WaterDistribution/PumpScheduling/scripts/init.py b/benchmarks/WaterDistribution/PumpScheduling/scripts/init.py new file mode 100644 index 00000000..506593a7 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/scripts/init.py @@ -0,0 +1,24 @@ +"""Baseline controller for the Net3 pump-scheduling task.""" + + +# EVOLVE-BLOCK-START +def control(observation: dict) -> dict[str, float]: + """Return pump speeds using only causal information in ``observation``.""" + hour = int(observation["hour"]) + tank_1 = float(observation["tank_levels_m"]["1"]) + price = float(observation["tariff"]) + + # Pump 10 replenishes the network from the lake, mostly off peak. + pump_10 = 1.0 if hour in range(1, 21) else 0.0 + if min(observation["tank_levels_m"].values()) < 2.5: + pump_10 = 1.0 + + # Pump 335 and pipe 330 regulate tank 1 in the original Net3 controls. + if tank_1 < 5.25: + pump_335 = 1.0 + elif tank_1 > 5.80: + pump_335 = 0.0 + else: + pump_335 = 0.0 if price > 0.20 else float(observation["previous_action"]["335"]) + return {"10": pump_10, "335": pump_335} +# EVOLVE-BLOCK-END diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/baseline.py b/benchmarks/WaterDistribution/PumpScheduling/verification/baseline.py new file mode 100644 index 00000000..5e982d49 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/baseline.py @@ -0,0 +1,17 @@ +"""Frozen reference controller for evaluator normalization.""" + + +def control(observation: dict) -> dict[str, float]: + hour = int(observation["hour"]) + tank_1 = float(observation["tank_levels_m"]["1"]) + price = float(observation["tariff"]) + pump_10 = 1.0 if hour in range(1, 21) else 0.0 + if min(observation["tank_levels_m"].values()) < 2.5: + pump_10 = 1.0 + if tank_1 < 5.25: + pump_335 = 1.0 + elif tank_1 > 5.80: + pump_335 = 0.0 + else: + pump_335 = 0.0 if price > 0.20 else float(observation["previous_action"]["335"]) + return {"10": pump_10, "335": pump_335} diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/controller_worker.py b/benchmarks/WaterDistribution/PumpScheduling/verification/controller_worker.py new file mode 100644 index 00000000..f501fae7 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/controller_worker.py @@ -0,0 +1,45 @@ +"""Isolated JSON-lines worker for causal pump controllers.""" + +import json +import math +import sys +from pathlib import Path + +SAFE_BUILTINS = { + "abs": abs, "all": all, "any": any, "bool": bool, "dict": dict, + "enumerate": enumerate, "float": float, "int": int, "len": len, + "list": list, "max": max, "min": min, "pow": pow, "range": range, + "reversed": reversed, "round": round, "set": set, "sorted": sorted, + "str": str, "sum": sum, "tuple": tuple, "zip": zip, + "Exception": Exception, "ValueError": ValueError, +} + + +def _load_control(path): + source = Path(path).read_text(encoding="utf-8") + namespace = {"__builtins__": SAFE_BUILTINS, "__name__": "candidate", "math": math} + exec(compile(source, "candidate.py", "exec"), namespace, namespace) + control = namespace.get("control") + if not callable(control): + raise ValueError("candidate must define control(observation)") + return control + + +def main(): + try: + control = _load_control(sys.argv[1]) + print(json.dumps({"ready": True}), flush=True) + except Exception as exc: + print(json.dumps({"ready": False, "error": f"{type(exc).__name__}: {exc}"}), flush=True) + return 2 + for line in sys.stdin: + try: + action = control(json.loads(line)) + print(json.dumps({"ok": True, "action": action}, allow_nan=False), flush=True) + except Exception as exc: + print(json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"}), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/evaluate.py b/benchmarks/WaterDistribution/PumpScheduling/verification/evaluate.py new file mode 100644 index 00000000..a82e27db --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/evaluate.py @@ -0,0 +1,7 @@ +"""Compatibility entry point for the PumpScheduling evaluator.""" + +from evaluator import main + + +if __name__ == "__main__": + main() diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/evaluator.py b/benchmarks/WaterDistribution/PumpScheduling/verification/evaluator.py new file mode 100644 index 00000000..2b302d2b --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/evaluator.py @@ -0,0 +1,41 @@ +import argparse +import json +from pathlib import Path + +from rollout import load_controller, rollout +from scenario_loader import load_hidden, load_public +from scoring import aggregate, scenario_score + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("candidate") + parser.add_argument("--json-out", default="metrics.json") + parser.add_argument("--artifacts-out", default="artifacts.json") + args = parser.parse_args() + controller = load_controller(args.candidate) + scenarios = load_public() + load_hidden() + baseline = load_controller(str(Path(__file__).with_name("baseline.py"))) + references = {s["id"]: rollout(baseline, s) for s in scenarios} + details = [] + scores = [] + for scenario in scenarios: + metrics = rollout(controller, scenario) + score = scenario_score(metrics, references[scenario["id"]]) if references[scenario["id"]].get("valid") else 0.0 + scores.append(score) + details.append({"scenario": scenario["id"], "score": score, "metrics": metrics}) + score = aggregate(scores) + valid = all(x["metrics"].get("valid", False) for x in details) + payload = { + "combined_score": score / 100.0 if valid else 0.0, + "score": score, + "valid": valid, + "scenarios": details, + } + Path(args.json_out).write_text(json.dumps(payload, indent=2), encoding="utf-8") + Path(args.artifacts_out).write_text(json.dumps({"scenario_count": len(details)}, indent=2), encoding="utf-8") + print(json.dumps(payload)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/requirements.txt b/benchmarks/WaterDistribution/PumpScheduling/verification/requirements.txt new file mode 100644 index 00000000..87e41158 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/requirements.txt @@ -0,0 +1,4 @@ +wntr==1.4.0 +setuptools<81 +numpy>=1.26,<3 +pytest>=8,<9 diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/rollout.py b/benchmarks/WaterDistribution/PumpScheduling/verification/rollout.py new file mode 100644 index 00000000..c06039c0 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/rollout.py @@ -0,0 +1,313 @@ +import json +import math +import os +import selectors +import subprocess +import sys +import tempfile +from pathlib import Path + +import wntr +from wntr.epanet.toolkit import ENepanet +from wntr.epanet.util import EN, FlowUnits, HydParam, to_si + +PUMPS = ("10", "335") +TANKS = ("1", "2", "3") +_SERVICE_CONTRACT = json.loads( + (Path(__file__).parents[1] / "references" / "service_nodes.json").read_text( + encoding="utf-8" + ) +) +SERVICE_NODES = tuple(_SERVICE_CONTRACT["service_nodes"]) +MINIMUM_PRESSURE_M = float(_SERVICE_CONTRACT["minimum_pressure_m"]) +HOUR_S = 3600 +DURATION_S = 24 * HOUR_S +HYDRAULIC_STEP_S = 300 + + +def load_controller(path): + path = Path(path).resolve() + if not path.is_file(): + raise ValueError(f"candidate not found: {path}") + return path + + +class IsolatedController: + """Run a controller with only a JSON observation/action interface.""" + + def __init__(self, source, timeout_s): + self.temp = tempfile.TemporaryDirectory(prefix="wntr-controller-") + candidate = Path(self.temp.name) / "candidate.py" + candidate.write_bytes(Path(source).read_bytes()) + worker = Path(__file__).with_name("controller_worker.py") + env = {"PATH": os.environ.get("PATH", ""), "PYTHONHASHSEED": "0"} + self.process = subprocess.Popen( + [sys.executable, "-I", "-S", str(worker), str(candidate)], + cwd=self.temp.name, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + bufsize=1, + ) + ready = self._read(timeout_s) + if not ready.get("ready"): + error = ready.get("error", "candidate failed to load") + self.close() + raise ValueError(error) + + def _read(self, timeout_s): + selector = selectors.DefaultSelector() + try: + selector.register(self.process.stdout, selectors.EVENT_READ) + if not selector.select(timeout_s): + raise TimeoutError("control call timed out") + line = self.process.stdout.readline() + finally: + selector.close() + if not line: + raise ValueError("controller process exited") + return json.loads(line) + + def call(self, observation, timeout_s): + self.process.stdin.write(json.dumps(observation, allow_nan=False) + "\n") + self.process.stdin.flush() + response = self._read(timeout_s) + if not response.get("ok"): + raise ValueError(response.get("error", "controller call failed")) + return response.get("action") + + def close(self): + if getattr(self, "process", None) is not None: + self.process.kill() + self.process.wait(timeout=2) + self.process = None + self.temp.cleanup() + + +def _network(scenario): + """Construct one continuous 24-hour Net3 model for a scenario.""" + wn = wntr.network.WaterNetworkModel("Net3") + for name in list(wn.control_name_list): + wn.remove_control(name) + wn.options.time.duration = DURATION_S + wn.options.time.hydraulic_timestep = HYDRAULIC_STEP_S + wn.options.time.report_timestep = HYDRAULIC_STEP_S + wn.options.time.pattern_timestep = HOUR_S + wn.options.time.pattern_start = 0 + wn.options.hydraulic.demand_multiplier = 1.0 + + multipliers = scenario["demand_multipliers"] + for _, pattern in wn.patterns(): + original = list(pattern.multipliers) + pattern.multipliers = [ + original[hour % len(original)] * multipliers[hour] + for hour in range(24) + ] + + for name in TANKS: + tank = wn.get_node(name) + tank.init_level += scenario["initial_tank_offsets_m"].get(name, 0.0) + + leak = scenario.get("leak") + if leak: + wn.get_node(leak["junction"]).add_leak( + wn, + area=leak["area_m2"], + start_time=leak["start_hour"] * HOUR_S, + end_time=leak["end_hour"] * HOUR_S, + ) + return wn + + +class ContinuousHydraulics: + """Small EPANET toolkit adapter retaining one hydraulic session.""" + + def __init__(self, wn, directory): + self.wn = wn + self.directory = Path(directory) + self.inp = self.directory / "network.inp" + wntr.network.write_inpfile(wn, str(self.inp), version=2.2) + self.toolkit = ENepanet( + str(self.inp), + str(self.directory / "network.rpt"), + str(self.directory / "network.bin"), + version=2.2, + ) + self.toolkit.ENopen() + self.units = FlowUnits(self.toolkit.ENgetflowunits()) + self.node_index = { + name: self.toolkit.ENgetnodeindex(name) + for name in set(TANKS + SERVICE_NODES) + } + self.tank_elevation = { + name: wn.get_node(name).elevation for name in TANKS + } + self.link_index = { + name: self.toolkit.ENgetlinkindex(name) + for name in PUMPS + ("330",) + } + self.pump_nodes = { + name: ( + self.toolkit.ENgetnodeindex(wn.get_link(name).start_node_name), + self.toolkit.ENgetnodeindex(wn.get_link(name).end_node_name), + ) + for name in PUMPS + } + self.toolkit.ENopenH() + self.toolkit.ENinitH(0) + self.hydraulics_open = True + + def close(self): + if getattr(self, "toolkit", None) is None: + return + if getattr(self, "hydraulics_open", False): + self.toolkit.ENcloseH() + self.hydraulics_open = False + self.toolkit.ENclose() + self.toolkit = None + + def tank_levels(self): + return { + name: float( + to_si( + self.units, + self.toolkit.ENgetnodevalue(index, EN.HEAD), + HydParam.HydraulicHead, + ) + ) - self.tank_elevation[name] + for name, index in self.node_index.items() + if name in TANKS + } + + def pressures(self): + return { + name: float(to_si(self.units, self.toolkit.ENgetnodevalue(self.node_index[name], EN.PRESSURE), HydParam.Pressure)) + for name in SERVICE_NODES + } + + def apply_action(self, action): + for name in PUMPS: + index = self.link_index[name] + speed = action[name] + self.toolkit.ENsetlinkvalue(index, EN.SETTING, max(speed, 0.01)) + self.toolkit.ENsetlinkvalue(index, EN.STATUS, 1.0 if speed > 0.01 else 0.0) + self.toolkit.ENsetlinkvalue( + self.link_index["330"], EN.STATUS, 0.0 if action["335"] > 0.01 else 1.0 + ) + + def run(self, expected_time): + actual = self.toolkit.ENrunH() + if actual != expected_time: + raise RuntimeError(f"unexpected hydraulic time {actual}, expected {expected_time}") + + def next_step(self): + return self.toolkit.ENnextH() + + def pump_power_kw(self): + total = 0.0 + for name in PUMPS: + flow = float(to_si(self.units, self.toolkit.ENgetlinkvalue(self.link_index[name], EN.FLOW), HydParam.Flow)) + start, end = self.pump_nodes[name] + start_head = float(to_si(self.units, self.toolkit.ENgetnodevalue(start, EN.HEAD), HydParam.HydraulicHead)) + end_head = float(to_si(self.units, self.toolkit.ENgetnodevalue(end, EN.HEAD), HydParam.HydraulicHead)) + total += 1000.0 * 9.80665 * max(flow, 0.0) * max(end_head - start_head, 0.0) / 0.75 / 1000.0 + return total + + +def rollout(controller_path, scenario, call_timeout_s=1.0): + controller = None + hydraulics = None + try: + wn = _network(scenario) + initial = {name: wn.get_node(name).init_level for name in TANKS} + tank_bounds = { + name: (wn.get_node(name).min_level, wn.get_node(name).max_level) + for name in TANKS + } + previous = {name: 0.0 for name in PUMPS} + switching = 0.0 + minimum_pressure = float("inf") + hourly_energy_kwh = [0.0] * 24 + controller_calls = 0 + + controller = IsolatedController(controller_path, call_timeout_s) + with tempfile.TemporaryDirectory(prefix="wntr-pump-") as temp_dir: + hydraulics = ContinuousHydraulics(wn, temp_dir) + current_time = 0 + final_levels = dict(initial) + while current_time <= DURATION_S: + if current_time < DURATION_S and current_time % HOUR_S == 0: + hour = current_time // HOUR_S + levels = hydraulics.tank_levels() + observation = { + "hour": hour, + "tank_levels_m": levels, + "tariff": scenario["tariff"][hour], + "demand_multiplier": scenario["demand_multipliers"][hour], + "previous_action": dict(previous), + } + action = controller.call(observation, call_timeout_s) + controller_calls += 1 + if not isinstance(action, dict) or set(action) != set(PUMPS): + raise ValueError("invalid or timed-out action") + action = {name: float(action[name]) for name in PUMPS} + if any(not math.isfinite(value) or value < 0 or value > 1 for value in action.values()): + raise ValueError("pump speeds must be finite and in [0,1]") + switching += sum(abs(action[name] - previous[name]) for name in PUMPS) + hydraulics.apply_action(action) + previous = action + + hydraulics.run(current_time) + pressures = hydraulics.pressures() + minimum_pressure = min(minimum_pressure, min(pressures.values())) + final_levels = hydraulics.tank_levels() + for name, level in final_levels.items(): + lower, upper = tank_bounds[name] + if level < lower - 1e-4 or level > upper + 1e-4: + raise ValueError("tank bound violation") + + step_s = hydraulics.next_step() + if step_s == 0: + break + if step_s > HYDRAULIC_STEP_S: + raise RuntimeError(f"hydraulic step {step_s} exceeds {HYDRAULIC_STEP_S}") + if current_time < DURATION_S: + hour = min(current_time // HOUR_S, 23) + hourly_energy_kwh[hour] += hydraulics.pump_power_kw() * step_s / HOUR_S + current_time += step_s + + if current_time != DURATION_S: + raise RuntimeError(f"incomplete hydraulic simulation at {current_time}s") + if controller_calls != 24: + raise RuntimeError(f"controller called {controller_calls} times, expected 24") + + hourly_power_kw = hourly_energy_kwh + energy_cost = sum( + hourly_power_kw[hour] * scenario["tariff"][hour] + for hour in range(24) + ) + terminal_deficit = sum(max(0.0, initial[name] - final_levels[name]) for name in TANKS) + valid = minimum_pressure >= MINIMUM_PRESSURE_M and all( + final_levels[name] >= initial[name] - 0.25 for name in TANKS + ) + return { + "valid": valid, + "energy_cost": energy_cost, + "peak_power_kw": max(hourly_power_kw), + "switching": max(switching, 0.01), + "terminal_deficit_m": max(terminal_deficit, 0.01), + "minimum_pressure_m": minimum_pressure, + "final_tank_levels_m": final_levels, + } + except Exception as exc: + return {"valid": False, "error": f"{type(exc).__name__}: {exc}"} + finally: + if hydraulics is not None: + try: + hydraulics.close() + except Exception: + pass + if controller is not None: + controller.close() diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/scenario_loader.py b/benchmarks/WaterDistribution/PumpScheduling/verification/scenario_loader.py new file mode 100644 index 00000000..122baf38 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/scenario_loader.py @@ -0,0 +1,25 @@ +import json +from pathlib import Path + + +def load_public(): + path = Path(__file__).parents[1] / "references" / "scenarios_public.json" + return json.loads(path.read_text(encoding="utf-8")) + + +def load_hidden(): + """Frozen deterministic cases, kept in evaluator code to resist overfitting.""" + base = load_public() + cases = [] + for index, source in enumerate(base): + case = json.loads(json.dumps(source)) + case["id"] = f"hidden_{index + 1}" + case["demand_multipliers"] = [round(min(1.50, x * (1.01 + 0.01 * index)), 4) for x in source["demand_multipliers"]] + case["tariff"] = source["tariff"][3:] + source["tariff"][:3] + case["initial_tank_offsets_m"] = {"1": -0.15 * index, "2": -0.20, "3": -0.10} + if index == 1: + case["leak"] = {"junction": "101", "start_hour": 8, "end_hour": 14, "area_m2": 0.00001} + elif index == 2: + case["leak"] = {"junction": "153", "start_hour": 16, "end_hour": 20, "area_m2": 0.000008} + cases.append(case) + return cases diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/scoring.py b/benchmarks/WaterDistribution/PumpScheduling/verification/scoring.py new file mode 100644 index 00000000..4986e508 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/scoring.py @@ -0,0 +1,20 @@ +import math + + +def scenario_score(metrics, reference): + if not metrics.get("valid", False): + return 0.0 + ratios = { + "energy": reference["energy_cost"] / max(metrics["energy_cost"], 1e-9), + "peak": reference["peak_power_kw"] / max(metrics["peak_power_kw"], 1e-9), + "smooth": reference["switching"] / max(metrics["switching"], 1e-9), + "terminal": reference["terminal_deficit_m"] / max(metrics["terminal_deficit_m"], 1e-9), + } + bounded = {k: min(1.5, max(0.0, v)) for k, v in ratios.items()} + return 100.0 * (0.55 * bounded["energy"] + 0.20 * bounded["peak"] + 0.10 * bounded["smooth"] + 0.15 * bounded["terminal"]) + + +def aggregate(scores): + if not scores or any(not math.isfinite(x) or x <= 0 for x in scores): + return 0.0 + return 0.70 * sum(scores) / len(scores) + 0.30 * min(scores) diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_contract.py b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_contract.py new file mode 100644 index 00000000..867f5341 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_contract.py @@ -0,0 +1,36 @@ +import importlib.util +from pathlib import Path + +import pytest + +import sys +sys.path.insert(0, str(Path(__file__).parents[1])) +from rollout import IsolatedController + + +def test_baseline_contract(): + path = Path(__file__).parents[2] / "scripts" / "init.py" + spec = importlib.util.spec_from_file_location("candidate", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + observation = {"hour": 0, "tank_levels_m": {"1": 4.0, "2": 7.0, "3": 8.8}, "tariff": 0.1, "demand_multiplier": 1.0, "previous_action": {"10": 0.0, "335": 0.0}} + action = module.control(observation) + assert set(action) == {"10", "335"} + assert all(0 <= value <= 1 for value in action.values()) + + +def test_isolated_controller_accepts_baseline(): + path = Path(__file__).parents[2] / "scripts" / "init.py" + controller = IsolatedController(path, 1.0) + try: + observation = {"hour": 0, "tank_levels_m": {"1": 4.0, "2": 7.0, "3": 8.8}, "tariff": 0.1, "demand_multiplier": 1.0, "previous_action": {"10": 0.0, "335": 0.0}} + assert set(controller.call(observation, 1.0)) == {"10", "335"} + finally: + controller.close() + + +def test_isolated_controller_rejects_file_access(tmp_path): + candidate = tmp_path / "candidate.py" + candidate.write_text("open('/etc/passwd').read()\ndef control(observation): return {'10': 0, '335': 0}\n", encoding="utf-8") + with pytest.raises(ValueError, match="open"): + IsolatedController(candidate, 1.0) diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_evaluator_output.py b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_evaluator_output.py new file mode 100644 index 00000000..5a21c76d --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_evaluator_output.py @@ -0,0 +1,53 @@ +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1])) +import evaluator + + +def test_evaluator_uses_frozen_reference_and_explicit_metrics(monkeypatch, tmp_path): + candidate = tmp_path / "candidate.py" + candidate.write_text("def control(observation): return {'10': 0, '335': 0}\n", encoding="utf-8") + metrics_path = tmp_path / "metrics.json" + artifacts_path = tmp_path / "artifacts.json" + loaded = [] + + def fake_load(path): + resolved = Path(path).resolve() + loaded.append(resolved) + return resolved + + fake_metrics = { + "valid": True, + "energy_cost": 1.0, + "peak_power_kw": 1.0, + "switching": 1.0, + "terminal_deficit_m": 1.0, + } + monkeypatch.setattr(evaluator, "load_controller", fake_load) + monkeypatch.setattr(evaluator, "load_public", lambda: [{"id": "case"}]) + monkeypatch.setattr(evaluator, "load_hidden", lambda: []) + monkeypatch.setattr(evaluator, "rollout", lambda _controller, _scenario: dict(fake_metrics)) + monkeypatch.setattr( + sys, + "argv", + [ + "evaluator.py", + str(candidate), + "--json-out", + str(metrics_path), + "--artifacts-out", + str(artifacts_path), + ], + ) + + evaluator.main() + payload = json.loads(metrics_path.read_text(encoding="utf-8")) + + assert loaded[0] == candidate.resolve() + assert loaded[1].name == "baseline.py" + assert loaded[0] != loaded[1] + assert isinstance(payload["valid"], bool) + assert isinstance(payload["combined_score"], float) + assert payload["combined_score"] == 1.0 diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scenarios.py b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scenarios.py new file mode 100644 index 00000000..3332c6f9 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scenarios.py @@ -0,0 +1,40 @@ +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1])) +from scenario_loader import load_hidden, load_public +from rollout import ( + DURATION_S, + HYDRAULIC_STEP_S, + MINIMUM_PRESSURE_M, + SERVICE_NODES, + _network, +) + + +def test_scenario_counts_and_lengths(): + cases = load_public() + load_hidden() + assert len(cases) == 6 + assert len({case["id"] for case in cases}) == 6 + assert all(len(case["tariff"]) == len(case["demand_multipliers"]) == 24 for case in cases) + + +def test_public_service_node_contract_matches_evaluator(): + task_root = Path(__file__).parents[2] + declaration = json.loads( + (task_root / "references" / "service_nodes.json").read_text(encoding="utf-8") + ) + assert tuple(declaration["service_nodes"]) == SERVICE_NODES + assert declaration["minimum_pressure_m"] == MINIMUM_PRESSURE_M == 20.0 + for document in (task_root / "Task.md", task_root / "Task_zh-CN.md"): + text = document.read_text(encoding="utf-8") + assert all(f"`{node}`" in text for node in SERVICE_NODES) + + +def test_scenario_builds_one_continuous_day(): + wn = _network(load_public()[0]) + assert wn.options.time.duration == DURATION_S + assert wn.options.time.hydraulic_timestep == HYDRAULIC_STEP_S + assert wn.options.time.pattern_timestep == 3600 + assert not wn.control_name_list diff --git a/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scoring.py b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scoring.py new file mode 100644 index 00000000..00e3c6b4 --- /dev/null +++ b/benchmarks/WaterDistribution/PumpScheduling/verification/tests/test_scoring.py @@ -0,0 +1,15 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1])) +from scoring import aggregate, scenario_score + + +def test_invalid_scores_zero(): + assert scenario_score({"valid": False}, {}) == 0.0 + assert aggregate([100.0, 0.0]) == 0.0 + + +def test_reference_scores_100(): + metrics = {"valid": True, "energy_cost": 2, "peak_power_kw": 3, "switching": 4, "terminal_deficit_m": 5} + assert scenario_score(metrics, metrics) == 100.0 diff --git a/benchmarks/WaterDistribution/README.md b/benchmarks/WaterDistribution/README.md new file mode 100644 index 00000000..95a03b02 --- /dev/null +++ b/benchmarks/WaterDistribution/README.md @@ -0,0 +1,8 @@ +# Water Distribution + +Engineering optimization tasks for drinking-water distribution systems. + +## Tasks + +- [PumpScheduling](PumpScheduling/README.md): Closed-loop pump scheduling on + EPANET Net3 through WNTR under demand, tariff, storage, and leak variations. diff --git a/benchmarks/WaterDistribution/README_zh-CN.md b/benchmarks/WaterDistribution/README_zh-CN.md new file mode 100644 index 00000000..8393cd8b --- /dev/null +++ b/benchmarks/WaterDistribution/README_zh-CN.md @@ -0,0 +1,8 @@ +# 配水系统 + +面向饮用水配水系统的工程优化任务。 + +## 任务列表 + +- [PumpScheduling](PumpScheduling/README_zh-CN.md)：基于 WNTR 中 EPANET Net3 管网的 + 闭环水泵调度，考虑需求、电价、储水和泄漏变化。 diff --git a/docs/superpowers/specs/2026-07-13-wntr-compliance-fixes-design.md b/docs/superpowers/specs/2026-07-13-wntr-compliance-fixes-design.md new file mode 100644 index 00000000..c5ba1fa8 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-wntr-compliance-fixes-design.md @@ -0,0 +1,69 @@ +# WNTR PumpScheduling Compliance Fixes + +## Scope + +Repair only the five confirmed submission blockers in +`WaterDistribution/PumpScheduling`: frozen-reference scoring, strict readonly +audit compatibility, unified-run documentation, causal-observation enforcement, +and explicit unified metric fields. No unrelated task constraints or refactors +are introduced. + +## Scoring architecture + +The shipped baseline controller will be copied to +`verification/baseline.py`. The evaluator will always load this readonly file +for reference rollouts and will load the submitted candidate only from the path +provided on the command line. Because `verification/` is readonly in unified +evaluation, replacing `scripts/init.py` can no longer replace the reference. + +The evaluator will write numeric `valid`, `score`, and `combined_score` fields. +`score` remains the documented 0--150 task score. `combined_score` will be +`score / 100`, preserving the repository runner's current fallback semantics +while making the contract explicit. Invalid candidates receive zero. + +## Causal controller isolation + +Candidate source is copied to a temporary isolated directory and executed by a +persistent Python worker subprocess. The worker starts with an isolated Python +mode, a sanitized environment, a temporary working directory, and a restricted +import/open policy. It receives only JSON observations over standard input and +returns only JSON actions over standard output. + +The evaluator retains the one-second call deadline, validates the exact pump +keys and finite speed bounds, and terminates the worker after each scenario. +Attempts to import task verifier modules, inspect benchmark files, or use paths +outside the isolated directory invalidate the rollout. Standard-library modules +needed for ordinary controller logic remain usable. + +## Unified metadata and documentation + +Add `frontier_eval/run_eval.sh` as the short benchmark-local wrapper expected by +the repository readonly audit. Add it to `readonly_files.txt`; the frozen +baseline is already covered by the existing `verification` entry. + +Update the English README with: + +- dependency installation; +- direct evaluator command; +- unified benchmark ID `WaterDistribution/PumpScheduling`; +- exact zero-iteration unified command; +- required Python runtime override; +- CPU, WNTR/EPANET, host-process, and trusted-evaluator assumptions. + +Mirror operationally relevant commands in the optional Chinese README. + +## Error handling + +Candidate import errors, protocol violations, timeouts, prohibited file access, +invalid JSON, non-finite values, hydraulic failures, and constraint violations +produce an invalid scenario instead of crashing the evaluator. The evaluator +still writes `metrics.json` and `artifacts.json` and exits normally so the +unified runner can retain actionable diagnostics. + +## Verification + +Run, in order: + +1. task unit tests, including scoring-reference separation and isolation tests; +2. direct baseline evaluator; +3. unified OpenEvolve baseline with zero iterations; diff --git a/docs/superpowers/specs/2026-07-13-wntr-continuous-hydraulics-design.md b/docs/superpowers/specs/2026-07-13-wntr-continuous-hydraulics-design.md new file mode 100644 index 00000000..dedd4709 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-wntr-continuous-hydraulics-design.md @@ -0,0 +1,94 @@ +# WNTR continuous-hydraulics evaluator design + +## Goal + +Replace the PumpScheduling evaluator's 24 independent one-hour Net3 simulations +with one continuous 24-hour EPANET hydraulic session per scenario. Preserve the +candidate interface, causal hourly control cadence, hard thresholds, and scoring +formula while making hydraulic state, tank dynamics, and pattern time continuous. + +The task contract will explicitly define the pressure-constraint node set as +`153`, `15`, `253`, `103`, `127`, `101`, `129`, `251`, `255`, and `105`. + +## Continuous simulation architecture + +For each scenario, construct Net3 once, remove its original controls, set the +scenario's initial tank levels, configure the 24-hour duration and five-minute +hydraulic timestep, and schedule any leak on the same model. Open one EPANET +hydraulic session and retain it until the scenario ends. + +At each integer hour from 0 through 23: + +1. Read the current levels of tanks `1`, `2`, and `3` from the live hydraulic + state. +2. Build the documented causal observation and call the candidate controller. +3. Validate the exact pump-key set and finite `[0, 1]` speeds. +4. Apply pump `10` and `335` speeds and open/closed states, plus the documented + inverse state of link `330`. +5. Apply the scenario demand multiplier for that hour. +6. Advance the existing hydraulic session through that hour in five-minute + steps without rebuilding or reopening the network. + +The controller is called exactly 24 times. It cannot observe future tariffs, +demands, leaks, or hydraulic states. + +## Metrics and constraints + +At every hydraulic step, inspect pressure at all ten declared service nodes and +the bounds of all three tanks. Any convergence, protocol, pressure, or tank-bound +failure invalidates the scenario. + +Compute pump power at every hydraulic step from nonnegative flow and delivered +head using the existing fixed efficiency. Integrate power over step duration to +obtain hourly energy, multiply each hour by its tariff, and retain the existing +hourly-average peak-power interpretation. Switching remains the sum of hourly +speed changes. Terminal deficit and terminal feasibility are computed from the +state at the end of the same continuous session. + +The existing per-scenario ratio weights, clipping, six-scenario aggregation, and +`combined_score = score / 100` remain unchanged. The frozen baseline controller +and every candidate use the same continuous rollout function, so reference +metrics are regenerated during evaluation rather than carried over from the old +simulation method. + +## Contract and public data + +Update both Task documents to list the ten service-node IDs explicitly. Add the +same ordered set to a public reference file exposed through `agent_files.txt`. +Tests must compare that public declaration to the evaluator constant so the +contract and implementation cannot drift. + +Update the README only where reproduction or runtime descriptions change. The +benchmark remains CPU-only, host-executed, deterministic, and based on WNTR +1.4.0 with no new external data. + +## Error handling + +Candidate load errors, invalid actions, timeouts, EPANET errors, non-finite +hydraulic values, and incomplete 24-hour runs produce an invalid scenario with +an actionable bounded error message. Evaluation still writes numeric `valid`, +`score`, and `combined_score` fields and exits successfully for candidate-caused +failures. + +## Verification + +Add tests that verify: + +- one network construction and one hydraulic session per rollout; +- exactly 24 causal controller calls; +- continuous simulation time and five-minute state sampling; +- hourly action and demand-multiplier updates; +- leak activation over the documented absolute time window; +- pressure checks cover exactly the ten public service nodes; +- tank bounds and terminal constraints use the continuous final state; +- invalid candidate and hydraulic failures emit numeric invalid metrics; +- the baseline is feasible in all three public and three hidden scenarios. + +Then run, in order: + +1. task-local unit and contract tests; +2. direct baseline evaluation; +3. unified zero-iteration evaluation; +4. strict readonly metadata audit, interpreting unrelated repository failures + separately; +5. one authorized DeepSeek V4 Flash OpenEvolve iteration. diff --git a/docs/superpowers/specs/2026-07-13-wntr-doc-audit-compliance-design.md b/docs/superpowers/specs/2026-07-13-wntr-doc-audit-compliance-design.md new file mode 100644 index 00000000..3a4f6c40 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-wntr-doc-audit-compliance-design.md @@ -0,0 +1,54 @@ +# WNTR Documentation and Audit Compliance + +## Scope + +Close the two remaining explicit contribution gaps for +`WaterDistribution/PumpScheduling`: complete task-contract documentation and +accurate readonly metadata. Do not add engineering constraints beyond the +existing task contract. + +## Task contract documentation + +Expand `Task.md` and its Chinese translation with only the repository-required +content: + +- the operational pump-scheduling problem and its engineering/economic value; +- the WNTR 1.4.0 EPANET Net3 model and hourly closed-loop simulation; +- every causal observation field, type, meaning, and unit; +- the exact two-key pump-speed output contract; +- feasibility constraints already enforced by the evaluator; +- objective components, weights, reference normalization, clipping, and + cross-scenario aggregation. + +The documentation must match executable evaluator behavior exactly. + +## Accurate evaluator entry points + +Add real compatibility entry points rather than readonly declarations for +missing paths: + +- `verification/evaluate.py` delegates to `verification/evaluator.py`; +- `frontier_eval/evaluate_submission.py` executes the same task-local evaluator + contract as `frontier_eval/run_eval.sh`. + +Do not create a `parse_mdriver_result.py` placeholder because MallocLab result +parsing is unrelated to this Python benchmark. + +## Readonly audit correction + +The repository audit currently treats four task-family-specific filenames as if +every unified benchmark must provide all four. Change the audit so required +coverage remains mandatory, while a recommended path is checked only when that +path exists in the benchmark. This preserves readonly protection without +forcing unrelated or nonexistent wrappers into metadata. + +Remove nonexistent path declarations from the WNTR `readonly_files.txt` and +list the two real compatibility entry points explicitly. Existing directory +coverage remains intact. + +## Tests and acceptance + +Add tests that verify explicit numeric unified metrics, reference/candidate +separation, isolated file-access rejection, and both real wrappers. Run: + +1. all WNTR unit and contract tests; diff --git a/docs/superpowers/specs/2026-07-13-wntr-domain-index-design.md b/docs/superpowers/specs/2026-07-13-wntr-domain-index-design.md new file mode 100644 index 00000000..19ae6fa4 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-wntr-domain-index-design.md @@ -0,0 +1,19 @@ +# WNTR domain index compliance design + +## Scope + +Update only the WaterDistribution domain overview documents so they include the +required sub-task index. Do not change candidate isolation, evaluation behavior, +metadata, dependencies, or task-level documentation. + +## Changes + +- Add a `Tasks` section to `benchmarks/WaterDistribution/README.md` linking to + `PumpScheduling/README.md` with a one-sentence English description. +- Add the equivalent `任务列表` section to + `benchmarks/WaterDistribution/README_zh-CN.md`. + +## Verification + +Check that both relative links resolve and that the working tree contains only +the intended documentation changes after the design commit. diff --git a/docs/superpowers/specs/2026-07-13-wntr-pump-scheduling-design.md b/docs/superpowers/specs/2026-07-13-wntr-pump-scheduling-design.md new file mode 100644 index 00000000..930f3fb1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-wntr-pump-scheduling-design.md @@ -0,0 +1,254 @@ +# WNTR Pump Scheduling Benchmark Design + +## Objective + +Add `WaterDistribution/PumpScheduling` to Frontier-Engineering as a unified +task. The task evolves a deterministic pump controller for EPANET Net3. The +controller must reduce electricity cost and peak power while maintaining water +pressure and tank recovery across multiple demand, tariff, and leak scenarios. + +## Server Isolation + +All work remains inside an isolated workspace: + +```text +workspace/ +├── repo/ # isolated Frontier-Engineering clone +├── env/ # task-specific Python environment +├── cache/ # package/download cache +└── runs/ # disposable experiment output +``` + +Run all setup, evaluation, and test processes with CPU affinity `64-73`. +Set `OMP_NUM_THREADS`, `OPENBLAS_NUM_THREADS`, `MKL_NUM_THREADS`, and +`NUMEXPR_NUM_THREADS` to `1`; use process-level parallelism only where tests +show it is safe. Do not read, write, move, or delete sibling agent directories. + +## Benchmark Package + +```text +repo/ +├── docs/ +│ └── superpowers/ +│ └── specs/ +│ └── 2026-07-13-wntr-pump-scheduling-design.md +└── benchmarks/ + └── WaterDistribution/ + ├── README.md + ├── README_zh-CN.md + └── PumpScheduling/ + ├── README.md + ├── README_zh-CN.md + ├── Task.md + ├── Task_zh-CN.md + ├── THIRD_PARTY_NOTICES.md + ├── scripts/ + │ └── init.py + ├── references/ + │ ├── scenario_schema.json + │ ├── scenarios_public.json + │ └── provenance.md + ├── verification/ + │ ├── evaluator.py + │ ├── rollout.py + │ ├── scoring.py + │ ├── scenario_loader.py + │ ├── requirements.txt + │ └── tests/ + │ ├── test_candidate_contract.py + │ ├── test_evaluator_failures.py + │ ├── test_reproducibility.py + │ └── test_scoring.py + └── frontier_eval/ + ├── initial_program.txt + ├── candidate_destination.txt + ├── eval_command.txt + ├── eval_cwd.txt + ├── agent_files.txt + ├── copy_files.txt + ├── readonly_files.txt + ├── artifact_files.txt + └── constraints.txt +``` + +Generated files such as `metrics.json`, `artifacts.json`, evaluator logs, caches, +and virtual environments are excluded from version control. + +## Candidate Contract + +The editable file is `scripts/init.py`. Keep exactly one evolution region using +`EVOLVE-BLOCK-START` and `EVOLVE-BLOCK-END`. + +The candidate exports: + +```python +def control(observation: dict) -> dict[str, float]: + """Return one normalized speed in [0, 1] for every configured pump.""" +``` + +The observation contains only causal information: + +- current hour and tariff; +- current tank levels; +- recent minimum and mean pressure; +- recent pump state and energy; +- demand forecast for the next control interval; +- configured pump identifiers and bounds. + +The evaluator rejects missing pumps, extra pumps, non-finite values, out-of-range +speeds, mutation of the observation, nondeterministic repeated calls, exceptions, +and calls exceeding the per-step timeout. Candidate code cannot access scenario +files, WNTR internals, the network model, future realized demand, or verifier +state. + +## Hydraulic Rollout + +Pin WNTR and its EPANET engine in `verification/requirements.txt`. Load the +package-provided Net3 model instead of copying an external network file into the +benchmark. Record the exact WNTR version, source URL, Revised BSD notice, and +EPANET MIT notice in `THIRD_PARTY_NOTICES.md`. + +Use 24 one-hour control intervals. At each interval: + +1. Build the observation from carried tank levels and previous results. +2. Call the candidate once under a timeout. +3. Apply constant pump speeds for the next hour. +4. Run a one-hour EPANET hydraulic simulation. +5. Carry terminal tank levels into the next interval. +6. Accumulate pump energy, pressure, tank, and switching metrics. + +Use pressure-dependent demand where supported. Fix all random seeds and reset the +network between scenarios. Water quality is outside scope. + +## Scenarios + +`scenarios_public.json` contains three documented development scenarios: + +1. nominal weekday demand with flat tariff; +2. morning/evening peaks with time-of-use tariff; +3. forecast error with shifted peak demand. + +The frozen evaluator defines three additional evaluation scenarios that are not +listed in `agent_files.txt`: + +1. high-demand stress day; +2. a bounded leak at a fixed junction and interval; +3. combined tariff spike, forecast error, and leak. + +All parameters are deterministic and versioned. The task documentation explains +the scenario families without exposing evaluation values through Agent context or +artifacts. + +## Feasibility + +A scenario is feasible only when: + +- every hydraulic solve converges; +- every candidate output satisfies its contract; +- critical-node pressure remains at or above `20 m`; +- every tank stays within the physical Net3 limits; +- terminal tank levels are no more than `0.25 m` below their initial levels; +- pump speeds remain in `[0, 1]`; +- the complete rollout finishes within the evaluator timeout. + +Any infeasible scenario makes the whole candidate invalid. This prevents good +performance on easy scenarios from compensating for loss of service. + +## Scoring + +For each feasible scenario, compute four higher-is-better components normalized +against the shipped feasible baseline: + +- energy-cost improvement: 55%; +- peak-power improvement: 20%; +- switching/speed-smoothness improvement: 10%; +- terminal tank recovery: 15%. + +Clip each component to a documented bounded range before aggregation. Convert the +weighted result to `[0, 100]`. Aggregate scenario scores as: + +```text +combined_score = 0.70 * mean(scenario_scores) + + 0.30 * min(scenario_scores) +``` + +This rewards average efficiency while preserving robustness. The evaluator emits +numeric `valid` and `combined_score` in `metrics.json`. `artifacts.json` includes +bounded per-scenario summaries and failure diagnostics, but not hidden scenario +parameters or reference solutions. + +## Baseline + +Ship a deterministic rule-based controller that: + +- starts pumps when aggregate tank storage is low; +- reduces pumping during expensive tariff intervals when storage permits; +- increases pumping when pressure or storage approaches a safety threshold; +- applies rate limits to avoid excessive speed changes. + +The baseline must be feasible on every scenario while leaving measurable cost and +peak-power headroom. Calibrate thresholds using the public scenarios, then verify +feasibility on all frozen scenarios without adding scenario-specific branches. + +## Unified Metadata + +Use the following contract: + +```text +initial_program.txt -> scripts/init.py +candidate_destination.txt -> scripts/init.py +eval_command.txt -> {python} verification/evaluator.py {candidate} +eval_cwd.txt -> . +copy_files.txt -> . +``` + +Expose only task documentation, the candidate, public scenario schema, public +scenarios, and constraints through `agent_files.txt`. Protect `verification/`, +`references/`, task contracts, notices, and evaluation metadata through +`readonly_files.txt`. Collect only bounded `metrics.json`, `artifacts.json`, and +the evaluator log. + +## Error Handling and Security + +- Execute candidates in a subprocess with a fixed timeout and sanitized + environment. +- Disable network access when an available isolation mechanism supports it. +- Restrict candidate imports to the Python standard library plus NumPy if needed. +- Run each scenario from a fresh copied model. +- Treat non-convergence, timeout, invalid JSON, NaN/Inf, and readonly violations + as invalid results with actionable artifacts. +- Never execute task code outside the agent-owned server directory during + development. + +The server lacks Docker, so validation uses host processes. The documentation +must state that candidate code is untrusted and that production evaluation should +use stronger process or container isolation. + +## Validation + +Run in this order with CPU affinity `64-73`: + +1. Create the isolated Python 3.11 environment and install pinned dependencies. +2. Confirm Net3 loads and completes a nominal hydraulic simulation. +3. Run unit tests for candidate contract, scoring, reproducibility, and failures. +4. Run `python verification/evaluator.py scripts/init.py` twice and confirm + identical metrics. +5. Run deliberately invalid candidates covering exceptions, timeout, NaN, + missing pumps, and pressure failure. +6. Run the unified baseline-only command with `algorithm.iterations=0`. +7. Run the repository readonly metadata audit in strict mode. +8. Run a small real optimization only when model credentials are available and + the user authorizes API spending. + +## Acceptance Criteria + +- The shipped baseline is feasible on all six scenarios. +- Repeated evaluation produces identical metrics within exact JSON equality for + deterministic fields. +- At least one simple, non-scenario-specific controller improvement beats the + baseline, demonstrating optimization headroom. +- Candidate exceptions and contract violations produce `valid=0` without + crashing the batch runner. +- The unified zero-iteration run succeeds using only documented dependencies. +- The readonly audit passes. +- No file outside the isolated workspace is modified.