From d6cd5de179a3dd63e4f560f82d47c65572b0c95d Mon Sep 17 00:00:00 2001 From: yibashishuo Date: Wed, 29 Jul 2026 13:28:16 +0800 Subject: [PATCH] feat(Cybersecurity): add dependency-aware patch scheduling benchmark --- .../DependencyAwarePatchScheduling/README.md | 78 ++ .../README_zh-CN.md | 78 ++ .../DependencyAwarePatchScheduling/Task.md | 398 +++++++++ .../Task_zh-CN.md | 396 +++++++++ .../baseline/heuristic.py | 20 + .../baseline/weak.py | 20 + .../benchmark.yaml | 361 +++++++++ .../data/seeds.json | 11 + .../frontier_eval/agent_files.txt | 5 + .../frontier_eval/artifact_files.txt | 3 + .../frontier_eval/candidate_destination.txt | 1 + .../frontier_eval/constraints.txt | 350 ++++++++ .../frontier_eval/copy_files.txt | 1 + .../frontier_eval/eval_command.txt | 1 + .../frontier_eval/eval_cwd.txt | 1 + .../frontier_eval/initial_program.txt | 1 + .../frontier_eval/readonly_files.txt | 8 + .../reference/exact.py | 20 + .../citations/src-03966d08d77b4404.json | 12 + .../citations/src-106e8a33951b796b.json | 12 + .../references/provenance.json | 31 + .../scripts/init.py | 213 +++++ .../verification/docker/Dockerfile | 13 + .../verification/evaluator.py | 762 ++++++++++++++++++ .../verification/problem.py | 754 +++++++++++++++++ .../verification/process_runner.py | 90 +++ benchmarks/Cybersecurity/README.md | 17 + benchmarks/Cybersecurity/README_zh-CN.md | 14 + 28 files changed, 3671 insertions(+) create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README.md create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README_zh-CN.md create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task.md create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task_zh-CN.md create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/heuristic.py create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/weak.py create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/benchmark.yaml create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/data/seeds.json create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/agent_files.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/artifact_files.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/candidate_destination.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/constraints.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/copy_files.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_command.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_cwd.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/initial_program.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/readonly_files.txt create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/reference/exact.py create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-03966d08d77b4404.json create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-106e8a33951b796b.json create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/provenance.json create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/scripts/init.py create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/docker/Dockerfile create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/evaluator.py create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/problem.py create mode 100644 benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/process_runner.py create mode 100644 benchmarks/Cybersecurity/README.md create mode 100644 benchmarks/Cybersecurity/README_zh-CN.md diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README.md b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README.md new file mode 100644 index 00000000..62b818ee --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README.md @@ -0,0 +1,78 @@ +# Dependency-Aware Cybersecurity Patch Scheduling + +This benchmark models offline patch selection and integer-slot scheduling under prerequisites, maintenance calendars, resource limits, asset exclusivity, service downtime limits, and rollback risk. + +Each deterministic instance contains assets, services, renewable resources, vulnerabilities, and patches. A candidate returns a schedule containing selected patch identifiers and start slots. Omitted patches remain unapplied. + +## Files + +- `Task.md`: English candidate specification. +- `Task_zh-CN.md`: Simplified Chinese candidate specification. +- `scripts/init.py`: self-contained starter solver matching the baseline behavior. +- `verification/problem.py`: deterministic instance generation, baseline solvers, validation, and raw objective evaluation. + +## Required API + +`verification.problem` exposes: + +- `generate_instances(seed) -> list[dict]` +- `solve_random(instance) -> dict` +- `solve_baseline(instance) -> dict` +- `solve_reference(instance) -> dict` +- `validate_solution(instance, solution) -> tuple[bool, str]` +- `evaluate_solution(instance, solution) -> int | float` + +Every seed produces one small, one medium, and one large instance. Horizons, entity counts, dependency density, identifiers, and array order vary deterministically with the seed. Generation uses a local deterministic integer generator and does not invoke a solver. + +## Objective + +The raw metric is total expected monetary loss in microunits and is minimized. It combines: + +1. Criticality-adjusted expected security loss accumulated before remediation. +2. Slot-specific business downtime loss for scheduled patches. +3. Expected rollback loss. + +Probability products use integer parts-per-million arithmetic with round-half-up after each multiplication. The evaluator returns only the positive raw loss. Score normalization is performed by the benchmark framework using `log2(baseline / candidate)`. + +## Baselines + +The deterministic control solver returns the feasible empty schedule. The starter and baseline use dependency-aware benefit-density ordering, schedule complete prerequisite bundles at their earliest feasible slots, and retain only objective-improving bundles. + +The reference solver exhaustively enumerates every dependency-closed patch subset and feasible start-time assignment on the five-patch small tier. Medium and large tiers use additional deterministic priority orders and forced-first lookahead schedules. The exact small-tier result audits solution quality, while the larger tiers preserve practical algorithmic headroom. + +## Evidence + +NIST SP 800-40 Rev. 4 motivates risk-based enterprise patch planning under operational constraints. CISA BOD 22-01 motivates prioritizing vulnerabilities with evidence of active exploitation and explicit remediation deadlines. The benchmark's monetary values and generated dependency graphs are synthetic calibration data, not empirical values attributed to either source. + +## Scope + +The model is reproducible and independently checkable, but it remains a planning abstraction. It does not model attacker adaptation, emergency approvals, uncertain durations, correlated rollback failures, partial deployments, undiscovered dependencies, incident response, or human coordination delays. + + +## Evaluation Contract + +The verifier recomputes `verifier_recomputed_total_expected_monetary_loss_microunits` and candidates must minimize it. +Each valid case is scored by `log2` improvement over the baseline and the final score is the +mean across cases. Invalid solutions receive `-1e18`. + +## Evaluation Design + +Setting: `offline_batch`. Arrival model: Each instance is a static planning batch: the complete vulnerability set, patch catalog, dependency graph, asset and service data, exploit-probability curves, capacities, maintenance windows, durations, and cost parameters are revealed before a schedule is submitted. No vulnerabilities, windows, or capacity changes arrive during execution. Benchmark instances are regenerated deterministically from fixed seeds, and the seed does not change in response to candidate behavior. + +Objective rationale: Expected monetary loss is appropriate because patching is not valuable merely for maximizing patch count or minimizing completion time. It prices the security exposure retained by delaying or omitting patches while also charging for the business disruption and rollback exposure caused by applying them. Expressing all three components in a common monetary unit produces a continuous, auditable tradeoff and keeps feasibility rules separate from preferences. The raw objective is strictly lower-is-better and can be kept positive through instance construction, making it suitable for framework-owned log2_baseline_ratio normalization. + +Literature alignment: NIST SP 800-40 Rev. 4 frames enterprise patching as risk-based preventive maintenance that must be planned alongside operational constraints. CISA BOD 22-01 provides a concrete basis for prioritizing vulnerabilities with evidence of active exploitation and for modeling remediation deadlines. This benchmark turns those planning principles into a deterministic offline optimization problem; its synthetic monetary loss, rollback, resource, and downtime parameters are benchmark abstractions rather than values claimed by either source. It differs from classical makespan scheduling by allowing economically rational patch omission and by jointly enforcing dependency closure, maintenance calendars, renewable resources, and service downtime limits. + +Local execution is only for reviewed code: + +```bash +python verification/evaluator.py scripts/init.py --local +``` + +Publish evaluation requires Docker and the pinned runtime image: + +```bash +docker pull python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 +python verification/evaluator.py scripts/init.py +``` + diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README_zh-CN.md b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README_zh-CN.md new file mode 100644 index 00000000..5ae00821 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/README_zh-CN.md @@ -0,0 +1,78 @@ +# 依赖感知的网络安全补丁调度 + +本基准研究离线补丁选择与整数时隙调度。调度必须同时满足补丁前置依赖、维护窗口、可再生资源容量、资产互斥、服务停机容量和累计停机预算等约束，并考虑回滚风险。 + +每个确定性实例包含资产、服务、可再生资源、漏洞和补丁。候选程序返回所选补丁的标识符与开始时隙；未出现在日程中的补丁视为未应用。 + +## 文件 + +- `Task.md`：英文候选任务说明。 +- `Task_zh-CN.md`：简体中文候选任务说明。 +- `scripts/init.py`：与基线行为一致的自包含起始求解器。 +- `verification/problem.py`：确定性实例生成、基线求解器、验证器和原始目标计算。 + +## 必需 API + +`verification.problem` 导出： + +- `generate_instances(seed) -> list[dict]` +- `solve_random(instance) -> dict` +- `solve_baseline(instance) -> dict` +- `solve_reference(instance) -> dict` +- `validate_solution(instance, solution) -> tuple[bool, str]` +- `evaluate_solution(instance, solution) -> int | float` + +每个种子生成一个小型、一个中型和一个大型实例。规划时域、实体数量、依赖密度、标识符和数组顺序均随种子确定性变化。生成器仅使用局部的确定性整数随机生成过程，并且不会调用任何求解器。 + +## 目标函数 + +需要最小化的原始指标是以微货币单位表示的总期望损失，包括： + +1. 修复完成前累计的、经资产关键度调整的期望安全损失。 +2. 已调度补丁造成的逐时隙业务停机损失。 +3. 期望回滚损失。 + +概率采用百万分之一整数表示。每次概率乘法后均按四舍五入规则取整。评估器只返回严格为正的原始损失；`log2(baseline / candidate)` 归一化由基准框架负责。 + +## 基线 + +确定性对照求解器返回可行的空日程。起始求解器和基线按照依赖感知的收益密度排序，在最早可行时隙安排完整的前置补丁包，并且只保留能降低目标值的补丁包。 + +参考求解器会在五个补丁的小型实例上穷举全部依赖闭合的补丁子集和可行开始时间组合。中型和大型实例使用多种确定性优先级顺序及指定首个补丁的前瞻搜索。小型实例的精确结果用于审计解质量，中大型实例用于保留实际算法优化空间。 + +## 证据 + +NIST SP 800-40 Rev. 4 为在运营约束下开展基于风险的企业补丁规划提供依据。CISA BOD 22-01 为优先处理已有在野利用证据的漏洞和设置修复期限提供依据。benchmark 中的金额和依赖图是用于校准的合成数据，并非两份资料提供的实测值。 + +## 适用范围 + +该模型可复现且可独立验证，但仍是规划抽象。它不表示攻击者适应、紧急审批、工期不确定性、相关回滚故障、部分部署、未知依赖、实时事件响应或人员协调延迟。 + + +## 评测契约 + +验证器会独立重算 `verifier_recomputed_total_expected_monetary_loss_microunits`，候选方案需要将其最小化。 +每个有效实例按照相对基线的 `log2` 改进计分，最终取所有实例分数的平均； +无效方案得分为 `-1e18`。 + +## 评测设计 + +问题设定：`离线批量`。到达模型：Each instance is a static planning batch: the complete vulnerability set, patch catalog, dependency graph, asset and service data, exploit-probability curves, capacities, maintenance windows, durations, and cost parameters are revealed before a schedule is submitted. No vulnerabilities, windows, or capacity changes arrive during execution. Benchmark instances are regenerated deterministically from fixed seeds, and the seed does not change in response to candidate behavior. + +目标理由：Expected monetary loss is appropriate because patching is not valuable merely for maximizing patch count or minimizing completion time. It prices the security exposure retained by delaying or omitting patches while also charging for the business disruption and rollback exposure caused by applying them. Expressing all three components in a common monetary unit produces a continuous, auditable tradeoff and keeps feasibility rules separate from preferences. The raw objective is strictly lower-is-better and can be kept positive through instance construction, making it suitable for framework-owned log2_baseline_ratio normalization. + +文献对齐：NIST SP 800-40 Rev. 4 frames enterprise patching as risk-based preventive maintenance that must be planned alongside operational constraints. CISA BOD 22-01 provides a concrete basis for prioritizing vulnerabilities with evidence of active exploitation and for modeling remediation deadlines. This benchmark turns those planning principles into a deterministic offline optimization problem; its synthetic monetary loss, rollback, resource, and downtime parameters are benchmark abstractions rather than values claimed by either source. It differs from classical makespan scheduling by allowing economically rational patch omission and by jointly enforcing dependency closure, maintenance calendars, renewable resources, and service downtime limits. + +本地执行只适用于已经审核的代码： + +```bash +python verification/evaluator.py scripts/init.py --local +``` + +正式发布评测需要 Docker 和固定摘要的运行镜像： + +```bash +docker pull python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 +python verification/evaluator.py scripts/init.py +``` + diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task.md b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task.md new file mode 100644 index 00000000..06fefe42 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task.md @@ -0,0 +1,398 @@ +# Task: Dependency-Aware Cybersecurity Patch Scheduling + +Implement `solve(instance)` in `scripts/init.py`. The function must return a JSON-compatible object with exactly one field, `schedule`. + +A schedule has the form `{'schedule': [{'patch_id': 'P000', 'start_slot': 0}, ...]}`. Each entry must contain exactly `patch_id` and `start_slot`. Patch identifiers must exist in the instance, start slots must be non-negative integers, and a patch may appear at most once. Omitting a patch means it is not applied. + +## Scheduling semantics + +A selected patch occupies the half-open interval `[start_slot, start_slot + duration)`. Execution is non-preemptive, and the completion slot is derived from the instance. + +Every submitted schedule must satisfy all of the following: + +- Every patch completes no later than `horizon`. +- Selecting a patch selects every direct and transitive prerequisite. +- Every direct prerequisite completes no later than the dependent patch starts. +- A patch interval is wholly contained in at least one maintenance window of every affected asset and every service listed in its service demands. +- Patches affecting the same asset cannot overlap when that asset has `exclusive_change` set to true. +- In every slot, aggregate demand for each renewable resource is no greater than its slot capacity. +- In every slot, aggregate downtime demand for each service is no greater than its slot capacity. +- Total downtime units for each service are no greater than its maintenance budget. + +All durations, relationships, demands, and costs are authoritative instance data. Candidate-supplied completion times, costs, or feasibility claims are not accepted. + +## Objective + +Minimize total expected monetary loss in microunits. + +For each vulnerability, remediation occurs at the earliest completion of any selected covering patch. If no covering patch completes, remediation time is the horizon. Security loss uses all exploit probabilities in slots before remediation. The vulnerability impact is first adjusted by `criticality_ppm`, and the cumulative exploit probability is computed by repeatedly multiplying the probability of no exploit using the instance scale of 1,000,000. + +Business loss is the sum of `downtime_units * loss_microunits_by_slot[t]` over every selected patch, demanded service, and occupied slot. Rollback loss is the rounded value of `rollback_probability_ppm * rollback_impact_microunits / 1,000,000` for each selected patch. + +The verifier recomputes feasibility and the raw objective exclusively from the instance and schedule. Lower raw values are better. Framework-owned normalization is `log2(baseline_raw / candidate_raw)`; it is not part of `solve` or `evaluate_solution`. + +## Determinism and isolation + +`solve` must be deterministic for a given instance. It must not read files, use the network, inspect environment variables or secrets, use evaluator output, or depend on hidden seed values. Keep every import required by `scripts/init.py` inside `solve`. + +The supplied starter is a feasible self-contained baseline. You may replace its internal heuristic while preserving the required function signature and output schema. + + +## Input Schema + +```json +{ + "type": "object", + "additionalProperties": false, + "required": [ + "instance_id", + "tier", + "horizon", + "probability_scale", + "assets", + "services", + "resources", + "vulnerabilities", + "patches" + ], + "properties": { + "instance_id": { + "type": "string", + "minLength": 1 + }, + "tier": { + "type": "string", + "enum": [ + "small", + "medium", + "large" + ] + }, + "horizon": { + "type": "integer", + "minimum": 1 + }, + "probability_scale": { + "type": "integer", + "enum": [ + 1000000 + ] + }, + "assets": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "asset_id", + "exclusive_change", + "maintenance_windows" + ], + "properties": { + "asset_id": { + "type": "string", + "minLength": 1 + }, + "exclusive_change": { + "type": "boolean" + }, + "maintenance_windows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "start_slot", + "end_slot" + ], + "properties": { + "start_slot": { + "type": "integer", + "minimum": 0 + }, + "end_slot": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + } + }, + "services": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "service_id", + "maintenance_windows", + "downtime_capacity_by_slot", + "downtime_budget", + "loss_microunits_by_slot" + ], + "properties": { + "service_id": { + "type": "string", + "minLength": 1 + }, + "maintenance_windows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "start_slot", + "end_slot" + ], + "properties": { + "start_slot": { + "type": "integer", + "minimum": 0 + }, + "end_slot": { + "type": "integer", + "minimum": 1 + } + } + } + }, + "downtime_capacity_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + }, + "downtime_budget": { + "type": "integer", + "minimum": 0 + }, + "loss_microunits_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + } + } + } + }, + "resources": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "resource_id", + "capacity_by_slot" + ], + "properties": { + "resource_id": { + "type": "string", + "minLength": 1 + }, + "capacity_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + } + } + } + }, + "vulnerabilities": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "vulnerability_id", + "asset_id", + "impact_microunits", + "criticality_ppm", + "exploit_probability_ppm_by_slot" + ], + "properties": { + "vulnerability_id": { + "type": "string", + "minLength": 1 + }, + "asset_id": { + "type": "string", + "minLength": 1 + }, + "impact_microunits": { + "type": "integer", + "minimum": 1 + }, + "criticality_ppm": { + "type": "integer", + "minimum": 1, + "maximum": 1000000 + }, + "exploit_probability_ppm_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 1000000 + } + } + } + } + }, + "patches": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "patch_id", + "duration", + "prerequisite_patch_ids", + "affected_asset_ids", + "covered_vulnerability_ids", + "resource_demands", + "service_demands", + "rollback_probability_ppm", + "rollback_impact_microunits" + ], + "properties": { + "patch_id": { + "type": "string", + "minLength": 1 + }, + "duration": { + "type": "integer", + "minimum": 1 + }, + "prerequisite_patch_ids": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "affected_asset_ids": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "covered_vulnerability_ids": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "resource_demands": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "resource_id", + "units" + ], + "properties": { + "resource_id": { + "type": "string", + "minLength": 1 + }, + "units": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "service_demands": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "service_id", + "downtime_units" + ], + "properties": { + "service_id": { + "type": "string", + "minLength": 1 + }, + "downtime_units": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "rollback_probability_ppm": { + "type": "integer", + "minimum": 0, + "maximum": 1000000 + }, + "rollback_impact_microunits": { + "type": "integer", + "minimum": 0 + } + } + } + } + } +} +``` + +## Output Schema + +```json +{ + "type": "object", + "additionalProperties": false, + "required": [ + "schedule" + ], + "properties": { + "schedule": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "patch_id", + "start_slot" + ], + "properties": { + "patch_id": { + "type": "string", + "minLength": 1 + }, + "start_slot": { + "type": "integer", + "minimum": 0 + } + } + } + } + } +} +``` + +## Constraints and Objective + +The output must satisfy every hard constraint described above. The frozen verifier independently +checks feasibility and recomputes `verifier_recomputed_total_expected_monetary_loss_microunits`. The objective is to minimize +that strictly positive raw metric. Valid cases use `log2` baseline improvement and are aggregated +with the mean; invalid solutions receive `-1e18`. + +The problem setting is `offline_batch`. Objective rationale: Expected monetary loss is appropriate because patching is not valuable merely for maximizing patch count or minimizing completion time. It prices the security exposure retained by delaying or omitting patches while also charging for the business disruption and rollback exposure caused by applying them. Expressing all three components in a common monetary unit produces a continuous, auditable tradeoff and keeps feasibility rules separate from preferences. The raw objective is strictly lower-is-better and can be kept positive through instance construction, making it suitable for framework-owned log2_baseline_ratio normalization. +Literature alignment: NIST SP 800-40 Rev. 4 frames enterprise patching as risk-based preventive maintenance that must be planned alongside operational constraints. CISA BOD 22-01 provides a concrete basis for prioritizing vulnerabilities with evidence of active exploitation and for modeling remediation deadlines. This benchmark turns those planning principles into a deterministic offline optimization problem; its synthetic monetary loss, rollback, resource, and downtime parameters are benchmark abstractions rather than values claimed by either source. It differs from classical makespan scheduling by allowing economically rational patch omission and by jointly enforcing dependency closure, maintenance calendars, renewable resources, and service downtime limits. + diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task_zh-CN.md b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task_zh-CN.md new file mode 100644 index 00000000..7682e358 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/Task_zh-CN.md @@ -0,0 +1,396 @@ +# 任务：依赖感知的网络安全补丁调度 + +请在 `scripts/init.py` 中实现 `solve(instance)`。函数必须返回可 JSON 序列化的对象，并且对象只能包含 `schedule` 字段。 + +日程格式为 `{'schedule': [{'patch_id': 'P000', 'start_slot': 0}, ...]}`。每个条目必须且只能包含 `patch_id` 与 `start_slot`。补丁标识符必须存在于实例中，开始时隙必须是非负整数，同一补丁最多出现一次。未列出的补丁视为未应用。 + +## 调度语义 + +所选补丁占用半开区间 `[start_slot, start_slot + duration)`。执行不可抢占，完成时隙由实例中的工期推导。 + +提交的日程必须满足以下全部条件： + +- 每个补丁不晚于 `horizon` 完成。 +- 选择补丁时，必须选择其全部直接和传递前置补丁。 +- 每个直接前置补丁必须不晚于依赖补丁开始时完成。 +- 补丁的完整执行区间必须包含在每个受影响资产以及每个服务需求所列服务的至少一个维护窗口内。 +- 当资产的 `exclusive_change` 为 true 时，影响该资产的补丁不能重叠。 +- 每个时隙内，每类可再生资源的总需求不能超过该时隙容量。 +- 每个时隙内，每个服务的总停机需求不能超过该时隙容量。 +- 每个服务在整个规划期内的总停机单位不能超过其维护预算。 + +所有工期、关系、需求和成本均以实例数据为准。候选程序提供的完成时间、成本或可行性声明不会被接受。 + +## 目标函数 + +最小化以微货币单位表示的总期望损失。 + +每个漏洞的修复时间是任一覆盖该漏洞的已选补丁的最早完成时隙。若没有覆盖补丁完成，则修复时间为规划期末。安全损失使用修复前所有时隙的利用概率。漏洞影响先乘以 `criticality_ppm` 调整；累计利用概率通过按 1,000,000 比例反复乘以“未被利用”的概率得到。 + +业务损失是在每个已选补丁、服务需求及占用时隙上累加 `downtime_units * loss_microunits_by_slot[t]`。每个已选补丁的回滚损失是 `rollback_probability_ppm * rollback_impact_microunits / 1,000,000` 的四舍五入整数值。 + +验证器仅根据实例与日程独立重算可行性和原始目标值。原始值越低越好。框架使用 `log2(baseline_raw / candidate_raw)` 进行归一化；该计算不属于 `solve` 或 `evaluate_solution`。 + +## 确定性与隔离 + +对于同一实例，`solve` 必须具有确定性。它不得读取文件、访问网络、检查环境变量或秘密、使用评估器输出，也不得依赖隐藏种子值。`scripts/init.py` 所需的所有导入必须放在 `solve` 函数内部。 + +所提供的起始实现是一个可行、自包含的基线。你可以替换其内部启发式算法，但必须保留函数签名和输出结构。 + + +## 输入 Schema + +```json +{ + "type": "object", + "additionalProperties": false, + "required": [ + "instance_id", + "tier", + "horizon", + "probability_scale", + "assets", + "services", + "resources", + "vulnerabilities", + "patches" + ], + "properties": { + "instance_id": { + "type": "string", + "minLength": 1 + }, + "tier": { + "type": "string", + "enum": [ + "small", + "medium", + "large" + ] + }, + "horizon": { + "type": "integer", + "minimum": 1 + }, + "probability_scale": { + "type": "integer", + "enum": [ + 1000000 + ] + }, + "assets": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "asset_id", + "exclusive_change", + "maintenance_windows" + ], + "properties": { + "asset_id": { + "type": "string", + "minLength": 1 + }, + "exclusive_change": { + "type": "boolean" + }, + "maintenance_windows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "start_slot", + "end_slot" + ], + "properties": { + "start_slot": { + "type": "integer", + "minimum": 0 + }, + "end_slot": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + } + }, + "services": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "service_id", + "maintenance_windows", + "downtime_capacity_by_slot", + "downtime_budget", + "loss_microunits_by_slot" + ], + "properties": { + "service_id": { + "type": "string", + "minLength": 1 + }, + "maintenance_windows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "start_slot", + "end_slot" + ], + "properties": { + "start_slot": { + "type": "integer", + "minimum": 0 + }, + "end_slot": { + "type": "integer", + "minimum": 1 + } + } + } + }, + "downtime_capacity_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + }, + "downtime_budget": { + "type": "integer", + "minimum": 0 + }, + "loss_microunits_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + } + } + } + }, + "resources": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "resource_id", + "capacity_by_slot" + ], + "properties": { + "resource_id": { + "type": "string", + "minLength": 1 + }, + "capacity_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + } + } + } + }, + "vulnerabilities": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "vulnerability_id", + "asset_id", + "impact_microunits", + "criticality_ppm", + "exploit_probability_ppm_by_slot" + ], + "properties": { + "vulnerability_id": { + "type": "string", + "minLength": 1 + }, + "asset_id": { + "type": "string", + "minLength": 1 + }, + "impact_microunits": { + "type": "integer", + "minimum": 1 + }, + "criticality_ppm": { + "type": "integer", + "minimum": 1, + "maximum": 1000000 + }, + "exploit_probability_ppm_by_slot": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 1000000 + } + } + } + } + }, + "patches": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "patch_id", + "duration", + "prerequisite_patch_ids", + "affected_asset_ids", + "covered_vulnerability_ids", + "resource_demands", + "service_demands", + "rollback_probability_ppm", + "rollback_impact_microunits" + ], + "properties": { + "patch_id": { + "type": "string", + "minLength": 1 + }, + "duration": { + "type": "integer", + "minimum": 1 + }, + "prerequisite_patch_ids": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "affected_asset_ids": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "covered_vulnerability_ids": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "resource_demands": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "resource_id", + "units" + ], + "properties": { + "resource_id": { + "type": "string", + "minLength": 1 + }, + "units": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "service_demands": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "service_id", + "downtime_units" + ], + "properties": { + "service_id": { + "type": "string", + "minLength": 1 + }, + "downtime_units": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "rollback_probability_ppm": { + "type": "integer", + "minimum": 0, + "maximum": 1000000 + }, + "rollback_impact_microunits": { + "type": "integer", + "minimum": 0 + } + } + } + } + } +} +``` + +## 输出 Schema + +```json +{ + "type": "object", + "additionalProperties": false, + "required": [ + "schedule" + ], + "properties": { + "schedule": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "patch_id", + "start_slot" + ], + "properties": { + "patch_id": { + "type": "string", + "minLength": 1 + }, + "start_slot": { + "type": "integer", + "minimum": 0 + } + } + } + } + } +} +``` + +## 约束与目标 + +输出必须满足上文全部硬约束。冻结验证器会独立检查可行性并重算 +`verifier_recomputed_total_expected_monetary_loss_microunits`；优化目标是将这个严格为正的原始指标最小化。 +有效实例使用相对基线的 `log2` 改进值，并取平均；无效方案得分为 `-1e18`。 +问题设定为 `离线批量`。目标设计理由：Expected monetary loss is appropriate because patching is not valuable merely for maximizing patch count or minimizing completion time. It prices the security exposure retained by delaying or omitting patches while also charging for the business disruption and rollback exposure caused by applying them. Expressing all three components in a common monetary unit produces a continuous, auditable tradeoff and keeps feasibility rules separate from preferences. The raw objective is strictly lower-is-better and can be kept positive through instance construction, making it suitable for framework-owned log2_baseline_ratio normalization. +文献对齐：NIST SP 800-40 Rev. 4 frames enterprise patching as risk-based preventive maintenance that must be planned alongside operational constraints. CISA BOD 22-01 provides a concrete basis for prioritizing vulnerabilities with evidence of active exploitation and for modeling remediation deadlines. This benchmark turns those planning principles into a deterministic offline optimization problem; its synthetic monetary loss, rollback, resource, and downtime parameters are benchmark abstractions rather than values claimed by either source. It differs from classical makespan scheduling by allowing economically rational patch omission and by jointly enforcing dependency closure, maintenance calendars, renewable resources, and service downtime limits. + diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/heuristic.py b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/heuristic.py new file mode 100644 index 00000000..d6641f55 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/heuristic.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from verification.problem import solve_baseline # noqa: E402 + + +def solve(instance: dict[str, Any]) -> dict[str, Any]: + return solve_baseline(instance) + + +if __name__ == "__main__": + json.dump(solve(json.load(sys.stdin)), sys.stdout, allow_nan=False) + sys.stdout.write("\n") diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/weak.py b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/weak.py new file mode 100644 index 00000000..87d24d4d --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/baseline/weak.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from verification.problem import solve_random # noqa: E402 + + +def solve(instance: dict[str, Any]) -> dict[str, Any]: + return solve_random(instance) + + +if __name__ == "__main__": + json.dump(solve(json.load(sys.stdin)), sys.stdout, allow_nan=False) + sys.stdout.write("\n") diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/benchmark.yaml b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/benchmark.yaml new file mode 100644 index 00000000..809036a5 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/benchmark.yaml @@ -0,0 +1,361 @@ +api_version: benchgen/v1 +benchmark_id: Cybersecurity/DependencyAwarePatchScheduling +title: Dependency-Aware Cybersecurity Patch Scheduling +summary: An offline structured-solution benchmark in which a solver selects patches + and assigns integer start slots over a finite planning horizon. Each deterministic + instance specifies vulnerabilities, affected assets, patch coverage, patch prerequisites, + asset and service relationships, time-varying exploit probabilities, monetary impact + and criticality values, maintenance windows, patch durations, renewable resource + demands, service downtime limits, and rollback probabilities and costs. The candidate + output is a JSON patch schedule; omitted patches remain unapplied. Instances are + generated reproducibly from a published fixed seed set, with small, medium, and + large tiers spanning sparse and dense dependency graphs, constrained maintenance + calendars, resource bottlenecks, and competing security and operational costs. +archetype: structured-solution +candidate: + path: scripts/init.py + function: solve + input_schema: + type: object + additionalProperties: false + required: + - instance_id + - tier + - horizon + - probability_scale + - assets + - services + - resources + - vulnerabilities + - patches + properties: + instance_id: + type: string + minLength: 1 + tier: + type: string + enum: + - small + - medium + - large + horizon: + type: integer + minimum: 1 + probability_scale: + type: integer + enum: + - 1000000 + assets: + type: array + items: + type: object + additionalProperties: false + required: + - asset_id + - exclusive_change + - maintenance_windows + properties: + asset_id: + type: string + minLength: 1 + exclusive_change: + type: boolean + maintenance_windows: + type: array + items: + type: object + additionalProperties: false + required: + - start_slot + - end_slot + properties: + start_slot: + type: integer + minimum: 0 + end_slot: + type: integer + minimum: 1 + services: + type: array + items: + type: object + additionalProperties: false + required: + - service_id + - maintenance_windows + - downtime_capacity_by_slot + - downtime_budget + - loss_microunits_by_slot + properties: + service_id: + type: string + minLength: 1 + maintenance_windows: + type: array + items: + type: object + additionalProperties: false + required: + - start_slot + - end_slot + properties: + start_slot: + type: integer + minimum: 0 + end_slot: + type: integer + minimum: 1 + downtime_capacity_by_slot: + type: array + items: + type: integer + minimum: 0 + downtime_budget: + type: integer + minimum: 0 + loss_microunits_by_slot: + type: array + items: + type: integer + minimum: 0 + resources: + type: array + items: + type: object + additionalProperties: false + required: + - resource_id + - capacity_by_slot + properties: + resource_id: + type: string + minLength: 1 + capacity_by_slot: + type: array + items: + type: integer + minimum: 0 + vulnerabilities: + type: array + items: + type: object + additionalProperties: false + required: + - vulnerability_id + - asset_id + - impact_microunits + - criticality_ppm + - exploit_probability_ppm_by_slot + properties: + vulnerability_id: + type: string + minLength: 1 + asset_id: + type: string + minLength: 1 + impact_microunits: + type: integer + minimum: 1 + criticality_ppm: + type: integer + minimum: 1 + maximum: 1000000 + exploit_probability_ppm_by_slot: + type: array + items: + type: integer + minimum: 0 + maximum: 1000000 + patches: + type: array + items: + type: object + additionalProperties: false + required: + - patch_id + - duration + - prerequisite_patch_ids + - affected_asset_ids + - covered_vulnerability_ids + - resource_demands + - service_demands + - rollback_probability_ppm + - rollback_impact_microunits + properties: + patch_id: + type: string + minLength: 1 + duration: + type: integer + minimum: 1 + prerequisite_patch_ids: + type: array + uniqueItems: true + items: + type: string + minLength: 1 + affected_asset_ids: + type: array + uniqueItems: true + items: + type: string + minLength: 1 + covered_vulnerability_ids: + type: array + uniqueItems: true + items: + type: string + minLength: 1 + resource_demands: + type: array + items: + type: object + additionalProperties: false + required: + - resource_id + - units + properties: + resource_id: + type: string + minLength: 1 + units: + type: integer + minimum: 0 + service_demands: + type: array + items: + type: object + additionalProperties: false + required: + - service_id + - downtime_units + properties: + service_id: + type: string + minLength: 1 + downtime_units: + type: integer + minimum: 0 + rollback_probability_ppm: + type: integer + minimum: 0 + maximum: 1000000 + rollback_impact_microunits: + type: integer + minimum: 0 + output_schema: + type: object + additionalProperties: false + required: + - schedule + properties: + schedule: + type: array + items: + type: object + additionalProperties: false + required: + - patch_id + - start_slot + properties: + patch_id: + type: string + minLength: 1 + start_slot: + type: integer + minimum: 0 + timeout_s: 30.0 + max_output_bytes: 1000000 +instances: + generator_module: verification.problem + public_seeds: + - 1729 + - 2718 + evaluation_seeds: + - 141421 + - 161803 + - 173205 + - 223606 + - 314159 + instances_per_seed: 3 +design: + problem_setting: offline_batch + arrival_model: 'Each instance is a static planning batch: the complete vulnerability + set, patch catalog, dependency graph, asset and service data, exploit-probability + curves, capacities, maintenance windows, durations, and cost parameters are revealed + before a schedule is submitted. No vulnerabilities, windows, or capacity changes + arrive during execution. Benchmark instances are regenerated deterministically + from fixed seeds, and the seed does not change in response to candidate behavior.' + objective_rationale: Expected monetary loss is appropriate because patching is not + valuable merely for maximizing patch count or minimizing completion time. It prices + the security exposure retained by delaying or omitting patches while also charging + for the business disruption and rollback exposure caused by applying them. Expressing + all three components in a common monetary unit produces a continuous, auditable + tradeoff and keeps feasibility rules separate from preferences. The raw objective + is strictly lower-is-better and can be kept positive through instance construction, + making it suitable for framework-owned log2_baseline_ratio normalization. + literature_alignment: NIST SP 800-40 Rev. 4 frames enterprise patching as risk-based + preventive maintenance that must be planned alongside operational constraints. + CISA BOD 22-01 provides a concrete basis for prioritizing vulnerabilities with + evidence of active exploitation and for modeling remediation deadlines. This benchmark + turns those planning principles into a deterministic offline optimization problem; + its synthetic monetary loss, rollback, resource, and downtime parameters are benchmark + abstractions rather than values claimed by either source. It differs from classical + makespan scheduling by allowing economically rational patch omission and by jointly + enforcing dependency closure, maintenance calendars, renewable resources, and + service downtime limits. +objective: + raw_metric: verifier_recomputed_total_expected_monetary_loss_microunits + direction: minimize + aggregation: mean + normalization: log2_baseline_ratio + invalid_score: -1.0e+18 +baseline: + module: baseline.heuristic + callable: solve +reference: + module: reference.exact + callable: solve +runtime: + isolation: docker + docker_image: python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 + timeout_s: 480.0 + cpus: 1.0 + memory_mb: 512 + pids_limit: 128 + network_disabled: true +provenance: + sources: + - id: src-03966d08d77b4404 + title: 'NIST SP 800-40 Rev. 4: Guide to Enterprise Patch Management Planning' + location: references/citations/src-03966d08d77b4404.json + sha256: 03966d08d77b4404e360aef7e96f902f2abbd3f9c6736bdce085d55ba2eda8b0 + license: citation-only + retrieved_at: '2026-07-29T03:54:39.026947+00:00' + origin: https://doi.org/10.6028/NIST.SP.800-40r4 + final_url: https://doi.org/10.6028/NIST.SP.800-40r4 + citation_only: true + publishable: true + - id: src-106e8a33951b796b + title: CISA Binding Operational Directive 22-01 + location: references/citations/src-106e8a33951b796b.json + sha256: 106e8a33951b796b337077a1016723812b4fde6eee09056df0a5104fb8771744 + license: citation-only + retrieved_at: '2026-07-29T03:54:39.285584+00:00' + origin: https://www.cisa.gov/news-events/directives/bod-22-01-reducing-significant-risk-known-exploited-vulnerabilities + final_url: https://www.cisa.gov/news-events/directives/bod-22-01-reducing-significant-risk-known-exploited-vulnerabilities + citation_only: true + publishable: true + data_license: CC0-1.0 + generated_from_seeds: true +frontier: + enabled: true + domain: Cybersecurity + task: DependencyAwarePatchScheduling + languages: + - en + - zh-CN +calibration: + minimum_reference_score: 0.05 + deterministic_tolerance: 1.0e-12 + unified_score_tolerance: 1.0e-09 diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/data/seeds.json b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/data/seeds.json new file mode 100644 index 00000000..2dce59e2 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/data/seeds.json @@ -0,0 +1,11 @@ +{ + "evaluation": { + "count": 5, + "visibility": "frozen-verifier-only" + }, + "instances_per_seed": 3, + "public": [ + 1729, + 2718 + ] +} diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/agent_files.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/agent_files.txt new file mode 100644 index 00000000..5288d154 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/agent_files.txt @@ -0,0 +1,5 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +scripts/init.py diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/artifact_files.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..fb5eabb7 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/artifact_files.txt @@ -0,0 +1,3 @@ +artifacts.json +metrics.json +outputs/*.json diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/candidate_destination.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/candidate_destination.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/candidate_destination.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/constraints.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/constraints.txt new file mode 100644 index 00000000..db185311 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/constraints.txt @@ -0,0 +1,350 @@ +Dependency-Aware Cybersecurity Patch Scheduling constraints: +1) Candidate file is `scripts/init.py` and must expose `solve(instance)`. +2) Candidate input must match this JSON Schema: +{ + "additionalProperties": false, + "properties": { + "assets": { + "items": { + "additionalProperties": false, + "properties": { + "asset_id": { + "minLength": 1, + "type": "string" + }, + "exclusive_change": { + "type": "boolean" + }, + "maintenance_windows": { + "items": { + "additionalProperties": false, + "properties": { + "end_slot": { + "minimum": 1, + "type": "integer" + }, + "start_slot": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "start_slot", + "end_slot" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "asset_id", + "exclusive_change", + "maintenance_windows" + ], + "type": "object" + }, + "type": "array" + }, + "horizon": { + "minimum": 1, + "type": "integer" + }, + "instance_id": { + "minLength": 1, + "type": "string" + }, + "patches": { + "items": { + "additionalProperties": false, + "properties": { + "affected_asset_ids": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + "covered_vulnerability_ids": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + "duration": { + "minimum": 1, + "type": "integer" + }, + "patch_id": { + "minLength": 1, + "type": "string" + }, + "prerequisite_patch_ids": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + "resource_demands": { + "items": { + "additionalProperties": false, + "properties": { + "resource_id": { + "minLength": 1, + "type": "string" + }, + "units": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "resource_id", + "units" + ], + "type": "object" + }, + "type": "array" + }, + "rollback_impact_microunits": { + "minimum": 0, + "type": "integer" + }, + "rollback_probability_ppm": { + "maximum": 1000000, + "minimum": 0, + "type": "integer" + }, + "service_demands": { + "items": { + "additionalProperties": false, + "properties": { + "downtime_units": { + "minimum": 0, + "type": "integer" + }, + "service_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "service_id", + "downtime_units" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "patch_id", + "duration", + "prerequisite_patch_ids", + "affected_asset_ids", + "covered_vulnerability_ids", + "resource_demands", + "service_demands", + "rollback_probability_ppm", + "rollback_impact_microunits" + ], + "type": "object" + }, + "type": "array" + }, + "probability_scale": { + "enum": [ + 1000000 + ], + "type": "integer" + }, + "resources": { + "items": { + "additionalProperties": false, + "properties": { + "capacity_by_slot": { + "items": { + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "resource_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "resource_id", + "capacity_by_slot" + ], + "type": "object" + }, + "type": "array" + }, + "services": { + "items": { + "additionalProperties": false, + "properties": { + "downtime_budget": { + "minimum": 0, + "type": "integer" + }, + "downtime_capacity_by_slot": { + "items": { + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "loss_microunits_by_slot": { + "items": { + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "maintenance_windows": { + "items": { + "additionalProperties": false, + "properties": { + "end_slot": { + "minimum": 1, + "type": "integer" + }, + "start_slot": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "start_slot", + "end_slot" + ], + "type": "object" + }, + "type": "array" + }, + "service_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "service_id", + "maintenance_windows", + "downtime_capacity_by_slot", + "downtime_budget", + "loss_microunits_by_slot" + ], + "type": "object" + }, + "type": "array" + }, + "tier": { + "enum": [ + "small", + "medium", + "large" + ], + "type": "string" + }, + "vulnerabilities": { + "items": { + "additionalProperties": false, + "properties": { + "asset_id": { + "minLength": 1, + "type": "string" + }, + "criticality_ppm": { + "maximum": 1000000, + "minimum": 1, + "type": "integer" + }, + "exploit_probability_ppm_by_slot": { + "items": { + "maximum": 1000000, + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "impact_microunits": { + "minimum": 1, + "type": "integer" + }, + "vulnerability_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "vulnerability_id", + "asset_id", + "impact_microunits", + "criticality_ppm", + "exploit_probability_ppm_by_slot" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "instance_id", + "tier", + "horizon", + "probability_scale", + "assets", + "services", + "resources", + "vulnerabilities", + "patches" + ], + "type": "object" +} +3) Candidate output must match this JSON Schema: +{ + "additionalProperties": false, + "properties": { + "schedule": { + "items": { + "additionalProperties": false, + "properties": { + "patch_id": { + "minLength": 1, + "type": "string" + }, + "start_slot": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "patch_id", + "start_slot" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "schedule" + ], + "type": "object" +} +4) Every hard constraint is recomputed by the frozen verifier. +5) Candidate timeout is 30 seconds per instance. +6) Candidate output is limited to 1000000 bytes. +7) Read-only benchmark assets include verification/, baseline/, reference/, data/, and references/. +8) Publish evaluation requires Docker image `python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7`; candidate and verifier + run in separate network-disabled, non-root containers. +9) Frontier unified must use its process isolation mode because the benchmark evaluator owns the + inner candidate/verifier containers; do not wrap this evaluator in another Docker runtime. diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/copy_files.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/copy_files.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/copy_files.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_command.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_command.txt new file mode 100644 index 00000000..613443e7 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +{python} {benchmark}/verification/evaluator.py {candidate} diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_cwd.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/initial_program.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/initial_program.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/readonly_files.txt b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..ab2c6d75 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/frontier_eval/readonly_files.txt @@ -0,0 +1,8 @@ +baseline/ +data/ +reference/ +references/ +verification/docker/ +verification/evaluator.py +verification/problem.py +verification/process_runner.py diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/reference/exact.py b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/reference/exact.py new file mode 100644 index 00000000..191b5705 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/reference/exact.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from verification.problem import solve_reference # noqa: E402 + + +def solve(instance: dict[str, Any]) -> dict[str, Any]: + return solve_reference(instance) + + +if __name__ == "__main__": + json.dump(solve(json.load(sys.stdin)), sys.stdout, allow_nan=False) + sys.stdout.write("\n") diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-03966d08d77b4404.json b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-03966d08d77b4404.json new file mode 100644 index 00000000..cd576070 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-03966d08d77b4404.json @@ -0,0 +1,12 @@ +{ + "citation_only": true, + "final_url": "https://doi.org/10.6028/NIST.SP.800-40r4", + "id": "src-03966d08d77b4404", + "license": "citation-only", + "location": "sources/src-03966d08d77b4404.raw", + "origin": "https://doi.org/10.6028/NIST.SP.800-40r4", + "publishable": true, + "retrieved_at": "2026-07-29T03:54:39.026947+00:00", + "sha256": "03966d08d77b4404e360aef7e96f902f2abbd3f9c6736bdce085d55ba2eda8b0", + "title": "NIST SP 800-40 Rev. 4: Guide to Enterprise Patch Management Planning" +} diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-106e8a33951b796b.json b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-106e8a33951b796b.json new file mode 100644 index 00000000..82d4d4b6 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/citations/src-106e8a33951b796b.json @@ -0,0 +1,12 @@ +{ + "citation_only": true, + "final_url": "https://www.cisa.gov/news-events/directives/bod-22-01-reducing-significant-risk-known-exploited-vulnerabilities", + "id": "src-106e8a33951b796b", + "license": "citation-only", + "location": "sources/src-106e8a33951b796b.raw", + "origin": "https://www.cisa.gov/news-events/directives/bod-22-01-reducing-significant-risk-known-exploited-vulnerabilities", + "publishable": true, + "retrieved_at": "2026-07-29T03:54:39.285584+00:00", + "sha256": "106e8a33951b796b337077a1016723812b4fde6eee09056df0a5104fb8771744", + "title": "CISA Binding Operational Directive 22-01" +} diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/provenance.json b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/provenance.json new file mode 100644 index 00000000..cbc1edbd --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/references/provenance.json @@ -0,0 +1,31 @@ +{ + "benchmark_id": "Cybersecurity/DependencyAwarePatchScheduling", + "data_license": "CC0-1.0", + "generated_from_seeds": true, + "sources": [ + { + "citation_only": true, + "final_url": "https://doi.org/10.6028/NIST.SP.800-40r4", + "id": "src-03966d08d77b4404", + "license": "citation-only", + "location": "references/citations/src-03966d08d77b4404.json", + "origin": "https://doi.org/10.6028/NIST.SP.800-40r4", + "publishable": true, + "retrieved_at": "2026-07-29T03:54:39.026947+00:00", + "sha256": "03966d08d77b4404e360aef7e96f902f2abbd3f9c6736bdce085d55ba2eda8b0", + "title": "NIST SP 800-40 Rev. 4: Guide to Enterprise Patch Management Planning" + }, + { + "citation_only": true, + "final_url": "https://www.cisa.gov/news-events/directives/bod-22-01-reducing-significant-risk-known-exploited-vulnerabilities", + "id": "src-106e8a33951b796b", + "license": "citation-only", + "location": "references/citations/src-106e8a33951b796b.json", + "origin": "https://www.cisa.gov/news-events/directives/bod-22-01-reducing-significant-risk-known-exploited-vulnerabilities", + "publishable": true, + "retrieved_at": "2026-07-29T03:54:39.285584+00:00", + "sha256": "106e8a33951b796b337077a1016723812b4fde6eee09056df0a5104fb8771744", + "title": "CISA Binding Operational Directive 22-01" + } + ] +} diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/scripts/init.py b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/scripts/init.py new file mode 100644 index 00000000..0a6b6856 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/scripts/init.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import json +import sys +from typing import Any + + +# EVOLVE-BLOCK-START +def solve(instance): + scale = 1000000 + horizon = instance['horizon'] + patches = {p['patch_id']: p for p in instance['patches']} + vulnerabilities = {v['vulnerability_id']: v for v in instance['vulnerabilities']} + assets = {a['asset_id']: a for a in instance['assets']} + services = {s['service_id']: s for s in instance['services']} + resources = {r['resource_id']: r for r in instance['resources']} + + def rounded_div(numerator, denominator): + return (numerator + denominator // 2) // denominator + + def security_loss(vulnerability, remediation_slot): + remaining_ppm = scale + limit = min(horizon, remediation_slot) + probabilities = vulnerability['exploit_probability_ppm_by_slot'] + for slot in range(limit): + remaining_ppm = rounded_div(remaining_ppm * (scale - probabilities[slot]), scale) + breach_ppm = scale - remaining_ppm + adjusted_impact = rounded_div(vulnerability['impact_microunits'] * vulnerability['criticality_ppm'], scale) + return rounded_div(adjusted_impact * breach_ppm, scale) + + def objective(schedule): + remediation = {vid: horizon for vid in vulnerabilities} + for entry in schedule: + patch = patches[entry['patch_id']] + completion = entry['start_slot'] + patch['duration'] + for vid in patch['covered_vulnerability_ids']: + if completion < remediation[vid]: + remediation[vid] = completion + security_total = sum((security_loss(vulnerability, remediation[vid]) for vid, vulnerability in vulnerabilities.items())) + downtime_total = 0 + rollback_total = 0 + for entry in schedule: + patch = patches[entry['patch_id']] + start = entry['start_slot'] + end = start + patch['duration'] + for demand in patch['service_demands']: + service = services[demand['service_id']] + units = demand['downtime_units'] + for slot in range(start, end): + downtime_total += units * service['loss_microunits_by_slot'][slot] + rollback_total += rounded_div(patch['rollback_probability_ppm'] * patch['rollback_impact_microunits'], scale) + return max(1, security_total + downtime_total + rollback_total) + + def contained(windows, start, end): + return any((window['start_slot'] <= start and end <= window['end_slot'] for window in windows)) + + def feasible(schedule): + selected = {entry['patch_id']: entry['start_slot'] for entry in schedule} + if len(selected) != len(schedule): + return False + records = [] + for entry in schedule: + pid = entry['patch_id'] + start = entry['start_slot'] + patch = patches[pid] + end = start + patch['duration'] + if start < 0 or end > horizon: + return False + for prerequisite in patch['prerequisite_patch_ids']: + if prerequisite not in selected: + return False + prerequisite_end = selected[prerequisite] + patches[prerequisite]['duration'] + if prerequisite_end > start: + return False + for aid in patch['affected_asset_ids']: + if not contained(assets[aid]['maintenance_windows'], start, end): + return False + for demand in patch['service_demands']: + service = services[demand['service_id']] + if not contained(service['maintenance_windows'], start, end): + return False + records.append((pid, start, end, patch)) + occupied_assets = set() + for pid, start, end, patch in records: + for aid in patch['affected_asset_ids']: + if not assets[aid]['exclusive_change']: + continue + for slot in range(start, end): + key = (aid, slot) + if key in occupied_assets: + return False + occupied_assets.add(key) + resource_usage = {rid: [0] * horizon for rid in resources} + service_usage = {sid: [0] * horizon for sid in services} + for pid, start, end, patch in records: + for demand in patch['resource_demands']: + usage = resource_usage[demand['resource_id']] + for slot in range(start, end): + usage[slot] += demand['units'] + for demand in patch['service_demands']: + usage = service_usage[demand['service_id']] + for slot in range(start, end): + usage[slot] += demand['downtime_units'] + for rid, usage in resource_usage.items(): + capacity = resources[rid]['capacity_by_slot'] + if any((usage[slot] > capacity[slot] for slot in range(horizon))): + return False + for sid, usage in service_usage.items(): + service = services[sid] + capacity = service['downtime_capacity_by_slot'] + if any((usage[slot] > capacity[slot] for slot in range(horizon))): + return False + if sum(usage) > service['downtime_budget']: + return False + return True + + def closure(root): + ordered = [] + seen = set() + + def visit(pid): + if pid in seen: + return + seen.add(pid) + for prerequisite in sorted(patches[pid]['prerequisite_patch_ids']): + visit(prerequisite) + ordered.append(pid) + visit(root) + return ordered + + def priority_order(): + full_losses = {vid: security_loss(vulnerability, horizon) for vid, vulnerability in vulnerabilities.items()} + scored = [] + for root in sorted(patches): + bundle = closure(root) + covered = set() + penalty = 0 + work = 0 + for pid in bundle: + patch = patches[pid] + covered.update(patch['covered_vulnerability_ids']) + work += patch['duration'] * (1 + sum((d['units'] for d in patch['resource_demands'])) + sum((d['downtime_units'] for d in patch['service_demands']))) + penalty += rounded_div(patch['rollback_probability_ppm'] * patch['rollback_impact_microunits'], scale) + for demand in patch['service_demands']: + service = services[demand['service_id']] + average_rate = sum(service['loss_microunits_by_slot']) // horizon + penalty += patch['duration'] * demand['downtime_units'] * average_rate + benefit = sum((full_losses[vid] for vid in covered)) + net = max(0, benefit - penalty) + density = net * scale // max(1, work) + scored.append((-density, root)) + scored.sort() + return [root for _, root in scored] + + def earliest_start(pid, schedule): + patch = patches[pid] + selected = {entry['patch_id']: entry['start_slot'] for entry in schedule} + lower_bound = 0 + for prerequisite in patch['prerequisite_patch_ids']: + if prerequisite not in selected: + return None + lower_bound = max(lower_bound, selected[prerequisite] + patches[prerequisite]['duration']) + last_start = horizon - patch['duration'] + for start in range(lower_bound, last_start + 1): + trial = schedule + [{'patch_id': pid, 'start_slot': start}] + if feasible(trial): + return start + return None + schedule = [] + current_value = objective(schedule) + selected = set() + for root in priority_order(): + if root in selected: + continue + needed = [pid for pid in closure(root) if pid not in selected] + trial = [dict(entry) for entry in schedule] + failed = False + for pid in needed: + start = earliest_start(pid, trial) + if start is None: + failed = True + break + trial.append({'patch_id': pid, 'start_slot': start}) + if failed: + continue + trial_value = objective(trial) + if trial_value < current_value: + schedule = trial + current_value = trial_value + selected = {entry['patch_id'] for entry in schedule} + schedule.sort(key=lambda entry: (entry['start_slot'], entry['patch_id'])) + return {'schedule': schedule} +# EVOLVE-BLOCK-END + + +def main() -> int: + try: + instance = json.load(sys.stdin) + if not isinstance(instance, dict): + raise TypeError("input must be a JSON object") + solution = solve(instance) + if not isinstance(solution, dict): + raise TypeError("solve() must return a JSON object") + json.dump(solution, sys.stdout, allow_nan=False, separators=(",", ":")) + sys.stdout.write("\n") + return 0 + except Exception as exc: + print(f"candidate error: {type(exc).__name__}: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/docker/Dockerfile b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/docker/Dockerfile new file mode 100644 index 00000000..9512d042 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=0 + +RUN groupadd --gid 65532 benchgen \ + && useradd --uid 65532 --gid 65532 --no-create-home --shell /usr/sbin/nologin benchgen + +WORKDIR /workspace +USER 65532:65532 + +CMD ["python", "--version"] diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/evaluator.py b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/evaluator.py new file mode 100644 index 00000000..72bcf620 --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/evaluator.py @@ -0,0 +1,762 @@ +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import math +import os +import statistics +import sys +import time +import uuid +from pathlib import Path +from types import ModuleType +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from verification.process_runner import BoundedResult, run_bounded # noqa: E402 + +SEEDS = [141421, 161803, 173205, 223606, 314159] +SMOKE_SEED = 134176928 +INSTANCES_PER_SEED = 3 +INPUT_SCHEMA = json.loads( + "{\"additionalProperties\": false, \"propert" + "ies\": {\"assets\": {\"items\": {\"additionalP" + "roperties\": false, \"properties\": {\"asset" + "_id\": {\"minLength\": 1, \"type\": \"string\"}" + ", \"exclusive_change\": {\"type\": \"boolean\"" + "}, \"maintenance_windows\": {\"items\": {\"ad" + "ditionalProperties\": false, \"properties\"" + ": {\"end_slot\": {\"minimum\": 1, \"type\": \"i" + "nteger\"}, \"start_slot\": {\"minimum\": 0, \"" + "type\": \"integer\"}}, \"required\": [\"start_" + "slot\", \"end_slot\"], \"type\": \"object\"}, \"" + "type\": \"array\"}}, \"required\": [\"asset_id" + "\", \"exclusive_change\", \"maintenance_wind" + "ows\"], \"type\": \"object\"}, \"type\": \"array" + "\"}, \"horizon\": {\"minimum\": 1, \"type\": \"i" + "nteger\"}, \"instance_id\": {\"minLength\": 1" + ", \"type\": \"string\"}, \"patches\": {\"items\"" + ": {\"additionalProperties\": false, \"prope" + "rties\": {\"affected_asset_ids\": {\"items\":" + " {\"minLength\": 1, \"type\": \"string\"}, \"ty" + "pe\": \"array\", \"uniqueItems\": true}, \"cov" + "ered_vulnerability_ids\": {\"items\": {\"min" + "Length\": 1, \"type\": \"string\"}, \"type\": \"" + "array\", \"uniqueItems\": true}, \"duration\"" + ": {\"minimum\": 1, \"type\": \"integer\"}, \"pa" + "tch_id\": {\"minLength\": 1, \"type\": \"strin" + "g\"}, \"prerequisite_patch_ids\": {\"items\":" + " {\"minLength\": 1, \"type\": \"string\"}, \"ty" + "pe\": \"array\", \"uniqueItems\": true}, \"res" + "ource_demands\": {\"items\": {\"additionalPr" + "operties\": false, \"properties\": {\"resour" + "ce_id\": {\"minLength\": 1, \"type\": \"string" + "\"}, \"units\": {\"minimum\": 0, \"type\": \"int" + "eger\"}}, \"required\": [\"resource_id\", \"un" + "its\"], \"type\": \"object\"}, \"type\": \"array" + "\"}, \"rollback_impact_microunits\": {\"mini" + "mum\": 0, \"type\": \"integer\"}, \"rollback_p" + "robability_ppm\": {\"maximum\": 1000000, \"m" + "inimum\": 0, \"type\": \"integer\"}, \"service" + "_demands\": {\"items\": {\"additionalPropert" + "ies\": false, \"properties\": {\"downtime_un" + "its\": {\"minimum\": 0, \"type\": \"integer\"}," + " \"service_id\": {\"minLength\": 1, \"type\": " + "\"string\"}}, \"required\": [\"service_id\", \"" + "downtime_units\"], \"type\": \"object\"}, \"ty" + "pe\": \"array\"}}, \"required\": [\"patch_id\"," + " \"duration\", \"prerequisite_patch_ids\", \"" + "affected_asset_ids\", \"covered_vulnerabil" + "ity_ids\", \"resource_demands\", \"service_d" + "emands\", \"rollback_probability_ppm\", \"ro" + "llback_impact_microunits\"], \"type\": \"obj" + "ect\"}, \"type\": \"array\"}, \"probability_sc" + "ale\": {\"enum\": [1000000], \"type\": \"integ" + "er\"}, \"resources\": {\"items\": {\"additiona" + "lProperties\": false, \"properties\": {\"cap" + "acity_by_slot\": {\"items\": {\"minimum\": 0," + " \"type\": \"integer\"}, \"type\": \"array\"}, \"" + "resource_id\": {\"minLength\": 1, \"type\": \"" + "string\"}}, \"required\": [\"resource_id\", \"" + "capacity_by_slot\"], \"type\": \"object\"}, \"" + "type\": \"array\"}, \"services\": {\"items\": {" + "\"additionalProperties\": false, \"properti" + "es\": {\"downtime_budget\": {\"minimum\": 0, " + "\"type\": \"integer\"}, \"downtime_capacity_b" + "y_slot\": {\"items\": {\"minimum\": 0, \"type\"" + ": \"integer\"}, \"type\": \"array\"}, \"loss_mi" + "crounits_by_slot\": {\"items\": {\"minimum\":" + " 0, \"type\": \"integer\"}, \"type\": \"array\"}" + ", \"maintenance_windows\": {\"items\": {\"add" + "itionalProperties\": false, \"properties\":" + " {\"end_slot\": {\"minimum\": 1, \"type\": \"in" + "teger\"}, \"start_slot\": {\"minimum\": 0, \"t" + "ype\": \"integer\"}}, \"required\": [\"start_s" + "lot\", \"end_slot\"], \"type\": \"object\"}, \"t" + "ype\": \"array\"}, \"service_id\": {\"minLengt" + "h\": 1, \"type\": \"string\"}}, \"required\": [" + "\"service_id\", \"maintenance_windows\", \"do" + "wntime_capacity_by_slot\", \"downtime_budg" + "et\", \"loss_microunits_by_slot\"], \"type\":" + " \"object\"}, \"type\": \"array\"}, \"tier\": {\"" + "enum\": [\"small\", \"medium\", \"large\"], \"ty" + "pe\": \"string\"}, \"vulnerabilities\": {\"ite" + "ms\": {\"additionalProperties\": false, \"pr" + "operties\": {\"asset_id\": {\"minLength\": 1," + " \"type\": \"string\"}, \"criticality_ppm\": {" + "\"maximum\": 1000000, \"minimum\": 1, \"type\"" + ": \"integer\"}, \"exploit_probability_ppm_b" + "y_slot\": {\"items\": {\"maximum\": 1000000, " + "\"minimum\": 0, \"type\": \"integer\"}, \"type\"" + ": \"array\"}, \"impact_microunits\": {\"minim" + "um\": 1, \"type\": \"integer\"}, \"vulnerabili" + "ty_id\": {\"minLength\": 1, \"type\": \"string" + "\"}}, \"required\": [\"vulnerability_id\", \"a" + "sset_id\", \"impact_microunits\", \"critical" + "ity_ppm\", \"exploit_probability_ppm_by_sl" + "ot\"], \"type\": \"object\"}, \"type\": \"array\"" + "}}, \"required\": [\"instance_id\", \"tier\", " + "\"horizon\", \"probability_scale\", \"assets\"" + ", \"services\", \"resources\", \"vulnerabilit" + "ies\", \"patches\"], \"type\": \"object\"}" +) +OUTPUT_SCHEMA = json.loads( + "{\"additionalProperties\": false, \"propert" + "ies\": {\"schedule\": {\"items\": {\"additiona" + "lProperties\": false, \"properties\": {\"pat" + "ch_id\": {\"minLength\": 1, \"type\": \"string" + "\"}, \"start_slot\": {\"minimum\": 0, \"type\":" + " \"integer\"}}, \"required\": [\"patch_id\", \"" + "start_slot\"], \"type\": \"object\"}, \"type\":" + " \"array\"}}, \"required\": [\"schedule\"], \"t" + "ype\": \"object\"}" +) +DIRECTION = "minimize" +AGGREGATION = "mean" +INVALID_SCORE = -1e+18 +TIMEOUT_S = 30.0 +MAX_OUTPUT_BYTES = 1000000 +RUNTIME_IMAGE = ( + "python:3.12.11-slim-bookworm@sha256:519591d6871b7bc437060736b9f7456b87" + "31f1499a57e22e6c285135ae657bf7" +) +MEMORY_MB = 512 +CPUS = 1.0 +PIDS_LIMIT = 128 + + +def _load_module(relative_module: str) -> ModuleType: + path = ROOT / (relative_module.replace(".", "/") + ".py") + module_spec = importlib.util.spec_from_file_location("benchgen_task_module", path) + if module_spec is None or module_spec.loader is None: + raise RuntimeError(f"cannot load task module: {path}") + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + return module + + +def _valid( + problem: ModuleType, instance: dict[str, Any], solution: dict[str, Any] +) -> tuple[bool, str]: + result = problem.validate_solution(instance, solution) + if result is None: + return True, "" + if isinstance(result, bool): + return result, "" if result else "solution rejected by verifier" + if isinstance(result, tuple) and len(result) == 2: + return bool(result[0]), str(result[1]) + if hasattr(result, "valid"): + return bool(result.valid), str(getattr(result, "reason", "")) + raise TypeError("invalid validate_solution return value") + + +def _finite(value: Any) -> bool: + if isinstance(value, float): + return math.isfinite(value) + if isinstance(value, dict): + return all(_finite(key) and _finite(item) for key, item in value.items()) + if isinstance(value, list): + return all(_finite(item) for item in value) + return True + + +def _validate_json_schema(value: Any, schema: dict[str, Any], path: str = "$") -> list[str]: + """Validate the dependency-free JSON Schema subset supported by benchgen v1.""" + + errors: list[str] = [] + expected_type = schema.get("type") + if expected_type and not _matches_type(value, expected_type): + return [f"{path}: expected {expected_type}, got {type(value).__name__}"] + if "enum" in schema and not any(_json_equal(value, item) for item in schema["enum"]): + errors.append(f"{path}: value is not in enum") + + if isinstance(value, dict): + properties = schema.get("properties", {}) + required = schema.get("required", []) + for key in required: + if key not in value: + errors.append(f"{path}: missing required property {key!r}") + for key, item in value.items(): + if key in properties: + errors.extend(_validate_json_schema(item, properties[key], f"{path}.{key}")) + elif schema.get("additionalProperties") is False: + errors.append(f"{path}: unexpected property {key!r}") + elif isinstance(value, list): + if "minItems" in schema and len(value) < schema["minItems"]: + errors.append(f"{path}: fewer than minItems") + if "maxItems" in schema and len(value) > schema["maxItems"]: + errors.append(f"{path}: more than maxItems") + if schema.get("uniqueItems"): + canonical = [_json_key(item) for item in value] + if len(canonical) != len(set(canonical)): + errors.append(f"{path}: items are not unique") + if isinstance(schema.get("items"), dict): + for index, item in enumerate(value): + errors.extend(_validate_json_schema(item, schema["items"], f"{path}[{index}]")) + elif isinstance(value, str): + if "minLength" in schema and len(value) < schema["minLength"]: + errors.append(f"{path}: shorter than minLength") + if "maxLength" in schema and len(value) > schema["maxLength"]: + errors.append(f"{path}: longer than maxLength") + elif isinstance(value, (int, float)) and not isinstance(value, bool): + if "minimum" in schema and value < schema["minimum"]: + errors.append(f"{path}: below minimum") + if "maximum" in schema and value > schema["maximum"]: + errors.append(f"{path}: above maximum") + return errors + + +def _json_equal(left: Any, right: Any) -> bool: + return _json_key(left) == _json_key(right) + + +def _json_key(value: Any) -> Any: + if value is None: + return ("null",) + if isinstance(value, bool): + return ("boolean", value) + if isinstance(value, (int, float)): + return ("number", value) + if isinstance(value, str): + return ("string", value) + if isinstance(value, list): + return ("array", tuple(_json_key(item) for item in value)) + if isinstance(value, dict): + return ( + "object", + tuple(sorted((str(key), _json_key(item)) for key, item in value.items())), + ) + return (type(value).__name__, repr(value)) + + +def _matches_type(value: Any, expected: str | list[str]) -> bool: + if isinstance(expected, list): + return any(_matches_type(value, item) for item in expected) + checks = { + "null": lambda item: item is None, + "boolean": lambda item: isinstance(item, bool), + "object": lambda item: isinstance(item, dict), + "array": lambda item: isinstance(item, list), + "string": lambda item: isinstance(item, str), + "integer": lambda item: isinstance(item, int) and not isinstance(item, bool), + "number": lambda item: isinstance(item, (int, float)) and not isinstance(item, bool), + } + return expected in checks and checks[expected](value) + + +def _require_schema(value: Any, schema: dict[str, Any], label: str) -> None: + errors = _validate_json_schema(value, schema) + if errors: + raise ValueError(f"{label} violates JSON Schema: {'; '.join(errors[:5])}") + + +def _metric_value(value: Any, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{label} must be an int or float, got {type(value).__name__}") + metric = float(value) + if not math.isfinite(metric): + raise ValueError(f"{label} must be finite") + if metric <= 0: + raise ValueError(f"{label} must be strictly positive") + return metric + + +def _raw_metric( + problem: ModuleType, + instance: dict[str, Any], + solution: dict[str, Any], + label: str, +) -> float: + value = problem.evaluate_solution(instance, solution) + return _metric_value(value, f"{label} evaluate_solution result") + + +def _canonical_json(value: Any, label: str) -> str: + try: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + except (TypeError, ValueError) as exc: + raise TypeError(f"{label} must be JSON serializable: {exc}") from exc + + +def _smoke_instances(problem: ModuleType) -> list[dict[str, Any]]: + generated = problem.generate_instances(SMOKE_SEED) + instances = [generated] if isinstance(generated, dict) else list(generated) + if len(instances) != INSTANCES_PER_SEED: + raise RuntimeError( + f"smoke generator returned {len(instances)} instances; " + f"expected exactly {INSTANCES_PER_SEED}" + ) + checked: list[dict[str, Any]] = [] + for instance in instances: + if not isinstance(instance, dict) or not _finite(instance): + raise TypeError("smoke instance must be a finite JSON object") + _require_schema(instance, INPUT_SCHEMA, "smoke instance") + _canonical_json(instance, "smoke instance") + checked.append(instance) + return checked + + +def _smoke_solver( + problem: ModuleType, + solver: Any, + instance: dict[str, Any], + label: str, +) -> tuple[dict[str, Any], float]: + solution = solver(copy.deepcopy(instance)) + if not isinstance(solution, dict) or not _finite(solution): + raise TypeError(f"{label} output must be a finite JSON object") + _require_schema(solution, OUTPUT_SCHEMA, f"{label} output") + _canonical_json(solution, f"{label} output") + valid, reason = _valid(problem, copy.deepcopy(instance), copy.deepcopy(solution)) + if not valid: + raise ValueError(f"{label} output is invalid: {reason or 'solution rejected'}") + metric = _raw_metric( + problem, + copy.deepcopy(instance), + copy.deepcopy(solution), + label, + ) + return solution, metric + + +def _smoke_payload() -> dict[str, Any]: + problem = _load_module("verification.problem") + first_instances = _smoke_instances(problem) + second_instances = _smoke_instances(problem) + if _canonical_json(first_instances, "generated instances") != _canonical_json( + second_instances, "repeated generated instances" + ): + raise ValueError("generate_instances is not deterministic") + + solvers = ( + ("random solver", problem.solve_random), + ("baseline solver", problem.solve_baseline), + ("reference solver", problem.solve_reference), + ) + solver_runs = 0 + for label, solver in solvers: + for instance in first_instances: + first_solution, first_metric = _smoke_solver(problem, solver, instance, label) + second_solution, second_metric = _smoke_solver(problem, solver, instance, label) + solver_runs += 2 + if _canonical_json(first_solution, f"{label} output") != _canonical_json( + second_solution, f"repeated {label} output" + ): + raise ValueError(f"{label} is not deterministic") + if first_metric != second_metric: + raise ValueError(f"{label} metric is not deterministic") + return { + "ok": True, + "instances_checked": len(first_instances), + "solver_runs": solver_runs, + "solvers_checked": [label for label, _solver in solvers], + } + + +def _score(baseline: float, candidate: float) -> float: + if not math.isfinite(baseline) or not math.isfinite(candidate): + raise ValueError("objective metrics must be finite") + if baseline <= 0 or candidate <= 0: + raise ValueError("log2 normalization requires positive metrics") + ratio = baseline / candidate if DIRECTION == "minimize" else candidate / baseline + return math.log2(ratio) + + +def _prepare_payload() -> dict[str, Any]: + problem = _load_module("verification.problem") + baseline_module = _load_module("baseline.heuristic") + baseline_solver = getattr(baseline_module, "solve") # noqa: B009 + cases: list[dict[str, Any]] = [] + for seed in SEEDS: + generated = problem.generate_instances(seed) + instances = [generated] if isinstance(generated, dict) else list(generated) + if len(instances) != INSTANCES_PER_SEED: + raise RuntimeError( + f"seed {seed} generated {len(instances)} instances; " + f"expected exactly {INSTANCES_PER_SEED}" + ) + for index, instance in enumerate(instances): + if not isinstance(instance, dict) or not _finite(instance): + raise TypeError("generated instance must be a finite JSON object") + _require_schema(instance, INPUT_SCHEMA, "generated instance") + baseline_solution = baseline_solver(instance) + if not isinstance(baseline_solution, dict) or not _finite(baseline_solution): + raise TypeError("baseline output must be a finite JSON object") + _require_schema(baseline_solution, OUTPUT_SCHEMA, "baseline output") + valid, reason = _valid(problem, instance, baseline_solution) + if not valid: + raise RuntimeError(f"invalid benchmark baseline: {reason}") + baseline_metric = _raw_metric( + problem, instance, baseline_solution, "baseline" + ) + cases.append( + { + "case_id": f"{seed}:{index}", + "seed": seed, + "index": index, + "instance": instance, + "baseline_metric": baseline_metric, + } + ) + return {"cases": cases} + + +def _score_payload(payload: dict[str, Any]) -> dict[str, Any]: + problem = _load_module("verification.problem") + prepared = payload.get("prepared", {}) + cases = prepared.get("cases", []) if isinstance(prepared, dict) else [] + submissions = payload.get("submissions", []) + if not isinstance(cases, list) or not isinstance(submissions, list): + raise TypeError("score payload must contain cases and submissions lists") + if len(cases) != len(submissions): + raise ValueError("submission count does not match prepared case count") + records: list[dict[str, Any]] = [] + for case, submission in zip(cases, submissions, strict=True): + started = time.monotonic() + record = {"case_index": len(records)} + try: + if submission.get("case_id") != case.get("case_id"): + raise ValueError("submission case id mismatch") + if submission.get("runner_error"): + raise RuntimeError(str(submission["runner_error"])) + solution = submission.get("solution") + if not isinstance(solution, dict) or not _finite(solution): + raise TypeError("candidate output must be a finite JSON object") + _require_schema(solution, OUTPUT_SCHEMA, "candidate output") + instance = case["instance"] + if not isinstance(instance, dict) or not _finite(instance): + raise TypeError("prepared instance must be a finite JSON object") + _require_schema(instance, INPUT_SCHEMA, "prepared instance") + valid, reason = _valid(problem, instance, solution) + if not valid: + raise ValueError(reason or "invalid candidate solution") + baseline_metric = _metric_value(case["baseline_metric"], "prepared baseline metric") + candidate_metric = _raw_metric(problem, instance, solution, "candidate") + record.update( + valid=True, + score=_score(baseline_metric, candidate_metric), + baseline_metric=baseline_metric, + candidate_metric=candidate_metric, + ) + except Exception as exc: + record.update(valid=False, score=INVALID_SCORE, reason=f"{type(exc).__name__}: {exc}") + record["runtime_s"] = time.monotonic() - started + records.append(record) + all_valid = bool(records) and all(record["valid"] for record in records) + scores = [float(record["score"]) for record in records] + combined = ( + (statistics.fmean(scores) if AGGREGATION == "mean" else statistics.median(scores)) + if all_valid + else INVALID_SCORE + ) + metrics = { + "valid": 1.0 if all_valid else 0.0, + "combined_score": combined, + "instances_total": len(records), + "instances_valid": sum(bool(record["valid"]) for record in records), + "runtime_s": sum(float(record["runtime_s"]) for record in records), + } + return {"metrics": metrics, "artifacts": {"instance_results": records}} + + +def _parse_json_result(result: BoundedResult, label: str) -> dict[str, Any]: + if result.timed_out: + raise TimeoutError(f"{label} timed out") + if result.output_truncated: + raise ValueError(f"{label} exceeded output limit") + if result.returncode != 0: + raise RuntimeError(f"{label} exited with {result.returncode}: {result.stderr[-2000:]}") + value = json.loads(result.stdout) + if not isinstance(value, dict) or not _finite(value): + raise TypeError(f"{label} output must be a finite JSON object") + return value + + +def _candidate_local( + candidate: Path, + instance: dict[str, Any], + *, + timeout_s: float = TIMEOUT_S, + max_output_bytes: int = MAX_OUTPUT_BYTES, +) -> dict[str, Any]: + result = run_bounded( + [sys.executable, str(candidate)], + json.dumps(instance, allow_nan=False), + timeout_s=timeout_s, + max_output_bytes=max_output_bytes, + ) + return _parse_json_result(result, "candidate") + + +def _docker_base(name: str) -> list[str]: + return [ + "docker", + "run", + "--rm", + "--interactive", + "--name", + name, + "--network", + "none", + "--read-only", + "--user", + "65532:65532", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + str(PIDS_LIMIT), + "--memory", + f"{MEMORY_MB}m", + "--cpus", + str(CPUS), + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=64m", + ] + + +def _docker_json( + command: list[str], + input_payload: dict[str, Any] | None, + *, + timeout_s: float, + label: str, + max_output_bytes: int = 2_000_000, +) -> dict[str, Any]: + name = f"benchgen-{label}-{uuid.uuid4().hex[:12]}" + full_command = _docker_base(name) + command + result = run_bounded( + full_command, + json.dumps(input_payload, allow_nan=False) if input_payload is not None else "", + timeout_s=timeout_s, + max_output_bytes=max_output_bytes, + ) + if result.timed_out: + run_bounded( + ["docker", "rm", "--force", name], + "", + timeout_s=10, + max_output_bytes=10_000, + ) + return _parse_json_result(result, label) + + +def _verifier_container(mode: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + if "," in str(ROOT) or "\n" in str(ROOT): + raise ValueError("benchmark path contains unsupported Docker mount characters") + command: list[str] = [] + for relative in ("verification", "baseline", "reference", "data", "references"): + source = ROOT / relative + if source.exists(): + command.extend( + ( + "--mount", + f"type=bind,src={source},dst=/workspace/benchmark/{relative},readonly", + ) + ) + command.extend( + ( + "--workdir", + "/workspace/benchmark", + RUNTIME_IMAGE, + "python", + "/workspace/benchmark/verification/evaluator.py", + mode, + ) + ) + return _docker_json(command, payload, timeout_s=480.0, label="verifier") + + +def _candidate_container( + candidate: Path, + case: dict[str, Any], + *, + timeout_s: float = TIMEOUT_S, + max_output_bytes: int = MAX_OUTPUT_BYTES, +) -> dict[str, Any]: + if "," in str(candidate) or "\n" in str(candidate): + raise ValueError("candidate path contains unsupported Docker mount characters") + trusted_solver = False + try: + relative = candidate.relative_to(ROOT) + trusted_solver = bool(relative.parts and relative.parts[0] in {"baseline", "reference"}) + except ValueError: + relative = Path(candidate.name) + if trusted_solver: + command = [ + "--mount", + f"type=bind,src={ROOT},dst=/workspace/benchmark,readonly", + "--workdir", + "/workspace/benchmark", + RUNTIME_IMAGE, + "python", + f"/workspace/benchmark/{relative.as_posix()}", + ] + else: + command = [ + "--mount", + f"type=bind,src={candidate},dst=/workspace/candidate.py,readonly", + "--workdir", + "/tmp", + RUNTIME_IMAGE, + "python", + "/workspace/candidate.py", + ] + return _docker_json( + command, + case["instance"], + timeout_s=timeout_s, + label="candidate", + max_output_bytes=max_output_bytes, + ) + + +def evaluate( + candidate: Path, + *, + local: bool = False, + candidate_timeout_s: float = TIMEOUT_S, + candidate_max_output_bytes: int = MAX_OUTPUT_BYTES, +) -> tuple[dict[str, Any], dict[str, Any]]: + candidate = candidate.resolve() + if not candidate.is_file(): + raise FileNotFoundError(f"candidate not found: {candidate}") + if local: + prepared = _prepare_payload() + submissions = [] + for case in prepared["cases"]: + try: + solution = _candidate_local( + candidate, + case["instance"], + timeout_s=candidate_timeout_s, + max_output_bytes=candidate_max_output_bytes, + ) + submissions.append({"case_id": case["case_id"], "solution": solution}) + except Exception as exc: + submissions.append( + {"case_id": case["case_id"], "runner_error": f"{type(exc).__name__}: {exc}"} + ) + result = _score_payload({"prepared": prepared, "submissions": submissions}) + else: + prepared = _verifier_container("--prepare") + submissions = [] + for case in prepared["cases"]: + try: + solution = _candidate_container( + candidate, + case, + timeout_s=candidate_timeout_s, + max_output_bytes=candidate_max_output_bytes, + ) + submissions.append({"case_id": case["case_id"], "solution": solution}) + except Exception as exc: + submissions.append( + {"case_id": case["case_id"], "runner_error": f"{type(exc).__name__}: {exc}"} + ) + result = _verifier_container( + "--score", {"prepared": prepared, "submissions": submissions} + ) + return result["metrics"], result["artifacts"] + + +def _atomic_json(path: Path, payload: dict[str, Any]) -> None: + from tempfile import NamedTemporaryFile + + path.parent.mkdir(parents=True, exist_ok=True) + with NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: + json.dump(payload, handle, indent=2, sort_keys=True, allow_nan=False) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + temporary = Path(handle.name) + temporary.replace(path) + + +def main() -> int: + if len(sys.argv) == 2 and sys.argv[1] == "--smoke": + try: + payload = _verifier_container("--smoke-local") + except Exception as exc: + message = f"{type(exc).__name__}: {exc}" + print(json.dumps({"ok": False, "error": message}, sort_keys=True)) + print(message, file=sys.stderr) + return 1 + print(json.dumps(payload, sort_keys=True, allow_nan=False)) + return 0 + if len(sys.argv) == 2 and sys.argv[1] == "--smoke-local": + try: + payload = _smoke_payload() + except Exception as exc: + message = f"{type(exc).__name__}: {exc}" + print(json.dumps({"ok": False, "error": message}, sort_keys=True)) + print(message, file=sys.stderr) + return 1 + print(json.dumps(payload, sort_keys=True, allow_nan=False)) + return 0 + if len(sys.argv) == 2 and sys.argv[1] == "--prepare": + print(json.dumps(_prepare_payload(), sort_keys=True, allow_nan=False)) + return 0 + if len(sys.argv) == 2 and sys.argv[1] == "--score": + payload = json.load(sys.stdin) + print(json.dumps(_score_payload(payload), sort_keys=True, allow_nan=False)) + return 0 + parser = argparse.ArgumentParser(description="Evaluate one structured-solution candidate.") + parser.add_argument("candidate", type=Path) + parser.add_argument("--local", action="store_true") + parser.add_argument( + "--_benchgen-probe-timeout-s", + type=float, + default=TIMEOUT_S, + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--_benchgen-probe-max-output-bytes", + type=int, + default=MAX_OUTPUT_BYTES, + help=argparse.SUPPRESS, + ) + arguments = parser.parse_args() + if not 0 < arguments._benchgen_probe_timeout_s <= TIMEOUT_S: + parser.error("probe timeout must be positive and cannot relax the benchmark limit") + if not 0 < arguments._benchgen_probe_max_output_bytes <= MAX_OUTPUT_BYTES: + parser.error("probe output limit must be positive and cannot relax the benchmark limit") + metrics, artifacts = evaluate( + arguments.candidate, + local=arguments.local, + candidate_timeout_s=arguments._benchgen_probe_timeout_s, + candidate_max_output_bytes=arguments._benchgen_probe_max_output_bytes, + ) + _atomic_json(Path.cwd() / "metrics.json", metrics) + _atomic_json(Path.cwd() / "artifacts.json", artifacts) + print(json.dumps(metrics, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/problem.py b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/problem.py new file mode 100644 index 00000000..a50864cc --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/problem.py @@ -0,0 +1,754 @@ +_SCALE = 1000000 +_MASK64 = (1 << 64) - 1 + + +class _DeterministicRng: + def __init__(self, seed): + self.state = (int(seed) ^ 0x9E3779B97F4A7C15) & _MASK64 + if self.state == 0: + self.state = 0xD1B54A32D192ED03 + + def next_u64(self): + value = self.state + value ^= value >> 12 + value ^= (value << 25) & _MASK64 + value ^= value >> 27 + self.state = value & _MASK64 + return (self.state * 2685821657736338717) & _MASK64 + + def randbelow(self, bound): + return self.next_u64() % bound + + def randint(self, lower, upper): + return lower + self.randbelow(upper - lower + 1) + + +def _shuffle(rng, values): + for index in range(len(values) - 1, 0, -1): + other = rng.randbelow(index + 1) + values[index], values[other] = values[other], values[index] + + +def _identifier_map(rng, prefix, count, width): + original = [('%s%0' + str(width) + 'd') % (prefix, index) for index in range(count)] + replacement = list(original) + _shuffle(rng, replacement) + return dict(zip(original, replacement)) + + +def _relabel_and_shuffle(rng, assets, services, resources, vulnerabilities, patches): + asset_ids = _identifier_map(rng, 'A', len(assets), 2) + service_ids = _identifier_map(rng, 'S', len(services), 2) + resource_ids = _identifier_map(rng, 'R', len(resources), 2) + vulnerability_ids = _identifier_map(rng, 'V', len(vulnerabilities), 3) + patch_ids = _identifier_map(rng, 'P', len(patches), 3) + + for asset in assets: + asset['asset_id'] = asset_ids[asset['asset_id']] + for service in services: + service['service_id'] = service_ids[service['service_id']] + for resource in resources: + resource['resource_id'] = resource_ids[resource['resource_id']] + for vulnerability in vulnerabilities: + vulnerability['vulnerability_id'] = vulnerability_ids[ + vulnerability['vulnerability_id'] + ] + vulnerability['asset_id'] = asset_ids[vulnerability['asset_id']] + for patch in patches: + patch['patch_id'] = patch_ids[patch['patch_id']] + patch['prerequisite_patch_ids'] = sorted( + patch_ids[pid] for pid in patch['prerequisite_patch_ids'] + ) + patch['affected_asset_ids'] = sorted( + asset_ids[aid] for aid in patch['affected_asset_ids'] + ) + patch['covered_vulnerability_ids'] = sorted( + vulnerability_ids[vid] for vid in patch['covered_vulnerability_ids'] + ) + for demand in patch['resource_demands']: + demand['resource_id'] = resource_ids[demand['resource_id']] + patch['resource_demands'].sort(key=lambda item: item['resource_id']) + for demand in patch['service_demands']: + demand['service_id'] = service_ids[demand['service_id']] + patch['service_demands'].sort(key=lambda item: item['service_id']) + + for collection in (assets, services, resources, vulnerabilities, patches): + _shuffle(rng, collection) + + +def _probability_curve(rng, horizon, base, spread): + result = [] + for slot in range(horizon): + cycle = ((slot % 7) - 3) * spread // 10 + value = base + cycle + rng.randint(-spread, spread) + result.append(max(1000, min(250000, value))) + return result + + +def _split_windows(horizon, rng): + split = horizon // 2 + rng.randint(-1, 1) + return [ + {'start_slot': 0, 'end_slot': split}, + {'start_slot': split + 2, 'end_slot': horizon}, + ] + + +def _make_instance(seed, tier_index): + configurations = [ + ('small', 16, 4, 2, 2, 5), + ('medium', 22, 5, 3, 2, 9), + ('large', 28, 7, 4, 3, 13), + ] + tier, base_horizon, base_assets, base_services, base_resources, base_patches = ( + configurations[tier_index] + ) + mixed_seed = ( + (int(seed) & _MASK64) + ^ ((tier_index + 1) * 0xA24BAED4963EE407) + ) & _MASK64 + rng = _DeterministicRng(mixed_seed) + horizon = base_horizon + rng.randbelow((3, 6, 9)[tier_index]) + asset_count = base_assets + rng.randbelow((2, 3, 4)[tier_index]) + service_count = base_services + rng.randbelow((2, 2, 3)[tier_index]) + resource_count = base_resources + rng.randbelow((1, 2, 2)[tier_index]) + patch_count = base_patches + rng.randbelow((1, 4, 5)[tier_index]) + + assets = [] + for index in range(asset_count): + windows = ( + [{'start_slot': 0, 'end_slot': horizon}] + if index < 2 + else _split_windows(horizon, rng) + ) + assets.append( + { + 'asset_id': 'A%02d' % index, + 'exclusive_change': index % 2 == 0, + 'maintenance_windows': windows, + } + ) + + services = [] + for index in range(service_count): + windows = ( + [{'start_slot': 0, 'end_slot': horizon}] + if index < 2 + else _split_windows(horizon, rng) + ) + if index == 0: + capacities = [1] * horizon + budget = 3 + else: + capacities = [ + 1 + (1 if (slot + index) % 5 == 0 else 0) + for slot in range(horizon) + ] + budget = max(6, horizon // 2 + index * 2) + base_loss = 16000 + index * 5000 + rng.randint(0, 3000) + losses = [ + base_loss + ((slot * 977 + rng.randbelow(4001)) % 7000) + for slot in range(horizon) + ] + services.append( + { + 'service_id': 'S%02d' % index, + 'maintenance_windows': windows, + 'downtime_capacity_by_slot': capacities, + 'downtime_budget': budget, + 'loss_microunits_by_slot': losses, + } + ) + + resources = [] + for index in range(resource_count): + capacities = [] + for slot in range(horizon): + if index == 0: + capacity = 1 if slot % 6 == 4 else 2 + elif index == 1: + capacity = 2 if slot % 7 == 3 else 1 + else: + capacity = 1 if slot % 5 in (1, 2) else 2 + capacities.append(capacity) + resources.append( + { + 'resource_id': 'R%02d' % index, + 'capacity_by_slot': capacities, + } + ) + + vulnerabilities = [ + { + 'vulnerability_id': 'V000', + 'asset_id': 'A00', + 'impact_microunits': 5200000 + rng.randint(0, 250000), + 'criticality_ppm': 950000, + 'exploit_probability_ppm_by_slot': _probability_curve( + rng, horizon, 62000, 9000 + ), + }, + { + 'vulnerability_id': 'V001', + 'asset_id': 'A00', + 'impact_microunits': 14000000 + rng.randint(0, 500000), + 'criticality_ppm': 1000000, + 'exploit_probability_ppm_by_slot': _probability_curve( + rng, horizon, 68000, 10000 + ), + }, + { + 'vulnerability_id': 'V002', + 'asset_id': 'A01', + 'impact_microunits': 1100000 + rng.randint(0, 150000), + 'criticality_ppm': 800000, + 'exploit_probability_ppm_by_slot': _probability_curve( + rng, horizon, 32000, 6000 + ), + }, + ] + + for index in range(3, patch_count): + asset_index = 2 + ((index - 3) % (asset_count - 2)) + vulnerabilities.append( + { + 'vulnerability_id': 'V%03d' % index, + 'asset_id': 'A%02d' % asset_index, + 'impact_microunits': 500000 + rng.randint(0, 1000000), + 'criticality_ppm': 550000 + rng.randint(0, 350000), + 'exploit_probability_ppm_by_slot': _probability_curve( + rng, + horizon, + 18000 + rng.randint(0, 22000), + 5000, + ), + } + ) + + patches = [ + { + 'patch_id': 'P000', + 'duration': 1, + 'prerequisite_patch_ids': [], + 'affected_asset_ids': ['A00'], + 'covered_vulnerability_ids': ['V000'], + 'resource_demands': [{'resource_id': 'R00', 'units': 1}], + 'service_demands': [ + {'service_id': 'S00', 'downtime_units': 1} + ], + 'rollback_probability_ppm': 20000, + 'rollback_impact_microunits': 400000, + }, + { + 'patch_id': 'P001', + 'duration': 1, + 'prerequisite_patch_ids': [], + 'affected_asset_ids': ['A01'], + 'covered_vulnerability_ids': ['V002'], + 'resource_demands': [{'resource_id': 'R01', 'units': 1}], + 'service_demands': [ + {'service_id': 'S01', 'downtime_units': 1} + ], + 'rollback_probability_ppm': 15000, + 'rollback_impact_microunits': 300000, + }, + { + 'patch_id': 'P002', + 'duration': 3, + 'prerequisite_patch_ids': ['P001'], + 'affected_asset_ids': ['A00'], + 'covered_vulnerability_ids': ['V001'], + 'resource_demands': [{'resource_id': 'R00', 'units': 1}], + 'service_demands': [ + {'service_id': 'S00', 'downtime_units': 1} + ], + 'rollback_probability_ppm': 35000, + 'rollback_impact_microunits': 800000, + }, + ] + + for index in range(3, patch_count): + pool = ['P001'] + ['P%03d' % prior for prior in range(3, index)] + max_dependencies = (1, 2, 3)[tier_index] + dependency_count = 0 + if pool and (tier_index > 0 or index % 2 == 0): + dependency_count = 1 + rng.randbelow( + min(max_dependencies, len(pool)) + ) + available = list(pool) + dependencies = [] + for _ in range(dependency_count): + chosen_index = rng.randbelow(len(available)) + dependencies.append(available.pop(chosen_index)) + dependencies.sort() + + asset_index = 2 + ((index - 3) % (asset_count - 2)) + service_index = 1 + ((index - 3) % (service_count - 1)) + resource_index = (index - 3) % resource_count + resource_demands = [ + {'resource_id': 'R%02d' % resource_index, 'units': 1} + ] + if tier_index == 2 and resource_count > 2 and index % 5 == 0: + second_resource = (resource_index + 1) % resource_count + resource_demands.append( + {'resource_id': 'R%02d' % second_resource, 'units': 1} + ) + patches.append( + { + 'patch_id': 'P%03d' % index, + 'duration': 1 + rng.randbelow(3), + 'prerequisite_patch_ids': dependencies, + 'affected_asset_ids': ['A%02d' % asset_index], + 'covered_vulnerability_ids': ['V%03d' % index], + 'resource_demands': resource_demands, + 'service_demands': [ + { + 'service_id': 'S%02d' % service_index, + 'downtime_units': 1, + } + ], + 'rollback_probability_ppm': 10000 + rng.randint(0, 50000), + 'rollback_impact_microunits': 150000 + rng.randint(0, 650000), + } + ) + + _relabel_and_shuffle( + rng, + assets, + services, + resources, + vulnerabilities, + patches, + ) + token = rng.next_u64() & 0xFFFFFFFF + return { + 'instance_id': 'daps-%s-%08x' % (tier, token), + 'tier': tier, + 'horizon': horizon, + 'probability_scale': _SCALE, + 'assets': assets, + 'services': services, + 'resources': resources, + 'vulnerabilities': vulnerabilities, + 'patches': patches, + } + + +def generate_instances(seed): + return [_make_instance(seed, tier_index) for tier_index in range(3)] + + +def _rounded_div(numerator, denominator): + return (numerator + denominator // 2) // denominator + + +def _security_loss(vulnerability, remediation_slot, horizon): + remaining_ppm = _SCALE + probabilities = vulnerability['exploit_probability_ppm_by_slot'] + for slot in range(min(horizon, remediation_slot)): + remaining_ppm = _rounded_div( + remaining_ppm * (_SCALE - probabilities[slot]), _SCALE + ) + breach_ppm = _SCALE - remaining_ppm + adjusted_impact = _rounded_div( + vulnerability['impact_microunits'] * vulnerability['criticality_ppm'], + _SCALE, + ) + return _rounded_div(adjusted_impact * breach_ppm, _SCALE) + + +def _objective_from_schedule(instance, schedule): + horizon = instance['horizon'] + patches = {p['patch_id']: p for p in instance['patches']} + vulnerabilities = { + v['vulnerability_id']: v for v in instance['vulnerabilities'] + } + services = {s['service_id']: s for s in instance['services']} + + remediation = {vid: horizon for vid in vulnerabilities} + for entry in schedule: + patch = patches[entry['patch_id']] + completion = entry['start_slot'] + patch['duration'] + for vid in patch['covered_vulnerability_ids']: + remediation[vid] = min(remediation[vid], completion) + + security_total = sum( + _security_loss(vulnerability, remediation[vid], horizon) + for vid, vulnerability in vulnerabilities.items() + ) + downtime_total = 0 + rollback_total = 0 + for entry in schedule: + patch = patches[entry['patch_id']] + start = entry['start_slot'] + end = start + patch['duration'] + for demand in patch['service_demands']: + service = services[demand['service_id']] + units = demand['downtime_units'] + for slot in range(start, end): + downtime_total += units * service['loss_microunits_by_slot'][slot] + rollback_total += _rounded_div( + patch['rollback_probability_ppm'] + * patch['rollback_impact_microunits'], + _SCALE, + ) + return max(1, security_total + downtime_total + rollback_total) + + +def validate_solution(instance, solution): + if type(solution) is not dict or set(solution) != {'schedule'}: + return False, 'solution must be an object containing only schedule' + schedule = solution['schedule'] + if type(schedule) is not list: + return False, 'schedule must be an array' + + horizon = instance['horizon'] + patches = {p['patch_id']: p for p in instance['patches']} + assets = {a['asset_id']: a for a in instance['assets']} + services = {s['service_id']: s for s in instance['services']} + resources = {r['resource_id']: r for r in instance['resources']} + selected = {} + + for index, entry in enumerate(schedule): + if type(entry) is not dict or set(entry) != {'patch_id', 'start_slot'}: + return False, 'schedule entry %d has invalid fields' % index + pid = entry['patch_id'] + start = entry['start_slot'] + if type(pid) is not str or not pid: + return False, 'schedule entry %d has an invalid patch_id' % index + if pid not in patches: + return False, 'unknown patch_id: %s' % pid + if pid in selected: + return False, 'duplicate patch_id: %s' % pid + if type(start) is not int or type(start) is bool: + return False, 'start_slot for %s must be an integer' % pid + if start < 0: + return False, 'start_slot for %s must be non-negative' % pid + selected[pid] = start + + def contained(windows, start, end): + return any( + window['start_slot'] <= start and end <= window['end_slot'] + for window in windows + ) + + records = [] + for pid, start in selected.items(): + patch = patches[pid] + end = start + patch['duration'] + if end > horizon: + return False, 'patch %s completes after the horizon' % pid + for prerequisite in patch['prerequisite_patch_ids']: + if prerequisite not in selected: + return False, 'patch %s is missing prerequisite %s' % ( + pid, + prerequisite, + ) + prerequisite_end = ( + selected[prerequisite] + patches[prerequisite]['duration'] + ) + if prerequisite_end > start: + return False, 'prerequisite %s does not precede %s' % ( + prerequisite, + pid, + ) + for aid in patch['affected_asset_ids']: + if aid not in assets: + return False, 'patch %s references unknown asset %s' % (pid, aid) + if not contained(assets[aid]['maintenance_windows'], start, end): + return False, 'patch %s is outside asset %s maintenance windows' % ( + pid, + aid, + ) + for demand in patch['service_demands']: + sid = demand['service_id'] + if sid not in services: + return False, 'patch %s references unknown service %s' % (pid, sid) + if not contained(services[sid]['maintenance_windows'], start, end): + return False, 'patch %s is outside service %s maintenance windows' % ( + pid, + sid, + ) + records.append((pid, start, end, patch)) + + occupied_assets = {} + for pid, start, end, patch in records: + for aid in patch['affected_asset_ids']: + if not assets[aid]['exclusive_change']: + continue + for slot in range(start, end): + key = (aid, slot) + if key in occupied_assets: + return False, 'patches %s and %s overlap on asset %s' % ( + occupied_assets[key], + pid, + aid, + ) + occupied_assets[key] = pid + + resource_usage = {rid: [0] * horizon for rid in resources} + service_usage = {sid: [0] * horizon for sid in services} + for pid, start, end, patch in records: + for demand in patch['resource_demands']: + rid = demand['resource_id'] + if rid not in resources: + return False, 'patch %s references unknown resource %s' % (pid, rid) + for slot in range(start, end): + resource_usage[rid][slot] += demand['units'] + for demand in patch['service_demands']: + sid = demand['service_id'] + for slot in range(start, end): + service_usage[sid][slot] += demand['downtime_units'] + + for rid, usage in resource_usage.items(): + capacity = resources[rid]['capacity_by_slot'] + for slot in range(horizon): + if usage[slot] > capacity[slot]: + return False, 'resource %s exceeds capacity at slot %d' % ( + rid, + slot, + ) + for sid, usage in service_usage.items(): + service = services[sid] + capacity = service['downtime_capacity_by_slot'] + for slot in range(horizon): + if usage[slot] > capacity[slot]: + return False, 'service %s exceeds downtime capacity at slot %d' % ( + sid, + slot, + ) + if sum(usage) > service['downtime_budget']: + return False, 'service %s exceeds its cumulative downtime budget' % sid + + return True, 'ok' + + +def evaluate_solution(instance, solution): + return _objective_from_schedule(instance, solution['schedule']) + + +def _closure_ids(root, patches): + ordered = [] + seen = set() + + def visit(pid): + if pid in seen: + return + seen.add(pid) + for prerequisite in sorted(patches[pid]['prerequisite_patch_ids']): + visit(prerequisite) + ordered.append(pid) + + visit(root) + return ordered + + +def _patch_order(instance, by_density): + horizon = instance['horizon'] + patches = {p['patch_id']: p for p in instance['patches']} + vulnerabilities = { + v['vulnerability_id']: v for v in instance['vulnerabilities'] + } + services = {s['service_id']: s for s in instance['services']} + full_losses = { + vid: _security_loss(vulnerability, horizon, horizon) + for vid, vulnerability in vulnerabilities.items() + } + scored = [] + for root in sorted(patches): + bundle = _closure_ids(root, patches) + covered = set() + penalty = 0 + work = 0 + for pid in bundle: + patch = patches[pid] + covered.update(patch['covered_vulnerability_ids']) + work += patch['duration'] * ( + 1 + + sum(d['units'] for d in patch['resource_demands']) + + sum(d['downtime_units'] for d in patch['service_demands']) + ) + penalty += _rounded_div( + patch['rollback_probability_ppm'] + * patch['rollback_impact_microunits'], + _SCALE, + ) + for demand in patch['service_demands']: + service = services[demand['service_id']] + average_rate = sum(service['loss_microunits_by_slot']) // horizon + penalty += ( + patch['duration'] + * demand['downtime_units'] + * average_rate + ) + benefit = sum(full_losses[vid] for vid in covered) + net = max(0, benefit - penalty) + score = (net * _SCALE) // max(1, work) if by_density else net + scored.append((-score, root)) + scored.sort() + return [root for _, root in scored] + + +def _earliest_start(instance, pid, schedule, patches): + horizon = instance['horizon'] + patch = patches[pid] + selected = {entry['patch_id']: entry['start_slot'] for entry in schedule} + lower_bound = 0 + for prerequisite in patch['prerequisite_patch_ids']: + if prerequisite not in selected: + return None + lower_bound = max( + lower_bound, + selected[prerequisite] + patches[prerequisite]['duration'], + ) + last_start = horizon - patch['duration'] + for start in range(lower_bound, last_start + 1): + trial = {'schedule': schedule + [{'patch_id': pid, 'start_slot': start}]} + valid, _ = validate_solution(instance, trial) + if valid: + return start + return None + + +def _construct(instance, order): + patches = {p['patch_id']: p for p in instance['patches']} + schedule = [] + selected = set() + current_value = _objective_from_schedule(instance, schedule) + for root in order: + if root in selected: + continue + needed = [ + pid for pid in _closure_ids(root, patches) if pid not in selected + ] + trial = [dict(entry) for entry in schedule] + failed = False + for pid in needed: + start = _earliest_start(instance, pid, trial, patches) + if start is None: + failed = True + break + trial.append({'patch_id': pid, 'start_slot': start}) + if failed: + continue + trial_value = _objective_from_schedule(instance, trial) + if trial_value < current_value: + schedule = trial + current_value = trial_value + selected = {entry['patch_id'] for entry in schedule} + schedule.sort(key=lambda entry: (entry['start_slot'], entry['patch_id'])) + return {'schedule': schedule} + + +def _exact_small(instance, incumbent): + patches = {p['patch_id']: p for p in instance['patches']} + patch_ids = sorted(patches) + if len(patch_ids) > 6: + raise ValueError('exact reference is limited to at most six patches') + + best_schedule = [dict(entry) for entry in incumbent['schedule']] + best_key = ( + _objective_from_schedule(instance, best_schedule), + tuple((entry['start_slot'], entry['patch_id']) for entry in best_schedule), + ) + + for mask in range(1 << len(patch_ids)): + selected = { + patch_ids[index] + for index in range(len(patch_ids)) + if mask & (1 << index) + } + if any( + prerequisite not in selected + for pid in selected + for prerequisite in patches[pid]['prerequisite_patch_ids'] + ): + continue + + remaining = set(selected) + order = [] + while remaining: + ready = sorted( + pid + for pid in remaining + if set(patches[pid]['prerequisite_patch_ids']).issubset(order) + ) + if not ready: + raise ValueError('patch dependency graph contains a cycle') + chosen = ready[0] + order.append(chosen) + remaining.remove(chosen) + + def search(index, schedule): + nonlocal best_key, best_schedule + if index == len(order): + canonical = sorted( + schedule, + key=lambda entry: (entry['start_slot'], entry['patch_id']), + ) + key = ( + _objective_from_schedule(instance, canonical), + tuple( + (entry['start_slot'], entry['patch_id']) + for entry in canonical + ), + ) + if key < best_key: + best_key = key + best_schedule = [dict(entry) for entry in canonical] + return + + pid = order[index] + patch = patches[pid] + lower_bound = 0 + placed = {entry['patch_id']: entry for entry in schedule} + for prerequisite in patch['prerequisite_patch_ids']: + entry = placed[prerequisite] + lower_bound = max( + lower_bound, + entry['start_slot'] + patches[prerequisite]['duration'], + ) + for start in range( + lower_bound, + instance['horizon'] - patch['duration'] + 1, + ): + trial = schedule + [{'patch_id': pid, 'start_slot': start}] + valid, _ = validate_solution(instance, {'schedule': trial}) + if valid: + search(index + 1, trial) + + search(0, []) + + return {'schedule': best_schedule} + + +def solve_random(instance): + return {'schedule': []} + + +def solve_baseline(instance): + return _construct(instance, _patch_order(instance, True)) + + +def solve_reference(instance): + density_order = _patch_order(instance, True) + value_order = _patch_order(instance, False) + orders = [density_order, value_order] + for first in value_order: + orders.append([first] + [pid for pid in density_order if pid != first]) + + best_solution = None + best_key = None + for order in orders: + solution = _construct(instance, order) + metric = _objective_from_schedule(instance, solution['schedule']) + signature = tuple( + (entry['start_slot'], entry['patch_id']) + for entry in solution['schedule'] + ) + key = (metric, signature) + if best_key is None or key < best_key: + best_key = key + best_solution = solution + if instance['tier'] == 'small': + return _exact_small(instance, best_solution) + return best_solution diff --git a/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/process_runner.py b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/process_runner.py new file mode 100644 index 00000000..3498497b --- /dev/null +++ b/benchmarks/Cybersecurity/DependencyAwarePatchScheduling/verification/process_runner.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import os +import signal +import subprocess +import threading +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BoundedResult: + returncode: int + stdout: str + stderr: str + timed_out: bool + output_truncated: bool + + +def run_bounded( + command: list[str], input_text: str, *, timeout_s: float, max_output_bytes: int +) -> BoundedResult: + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + stdout = bytearray() + stderr = bytearray() + truncated = [False, False] + readers = [ + threading.Thread( + target=_drain, + args=(process.stdout, stdout, max_output_bytes, truncated, 0), + daemon=True, + ), + threading.Thread( + target=_drain, + args=(process.stderr, stderr, max_output_bytes, truncated, 1), + daemon=True, + ), + ] + for reader in readers: + reader.start() + writer = threading.Thread(target=_write, args=(process, input_text.encode("utf-8")), daemon=True) + writer.start() + timed_out = False + try: + process.wait(timeout=timeout_s) + except subprocess.TimeoutExpired: + timed_out = True + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + process.wait() + for reader in readers: + reader.join() + return BoundedResult( + returncode=process.returncode, + stdout=stdout.decode("utf-8", errors="replace"), + stderr=stderr.decode("utf-8", errors="replace"), + timed_out=timed_out, + output_truncated=any(truncated), + ) + + +def _drain(stream, target: bytearray, limit: int, truncated: list[bool], index: int) -> None: + if stream is None: + return + while True: + chunk = stream.read(64 * 1024) + if not chunk: + return + remaining = limit - len(target) + if remaining > 0: + target.extend(chunk[:remaining]) + if len(chunk) > remaining: + truncated[index] = True + + +def _write(process: subprocess.Popen[bytes], content: bytes) -> None: + if process.stdin is None: + return + try: + process.stdin.write(content) + process.stdin.close() + except (BrokenPipeError, OSError): + pass diff --git a/benchmarks/Cybersecurity/README.md b/benchmarks/Cybersecurity/README.md new file mode 100644 index 00000000..416c0508 --- /dev/null +++ b/benchmarks/Cybersecurity/README.md @@ -0,0 +1,17 @@ +# Cybersecurity + +This domain collects executable cybersecurity engineering optimization tasks with +explicit operational constraints, risk-sensitive objectives, and independently +recomputed verification. + +## Tasks + +- `DependencyAwarePatchScheduling` + - Unified benchmark: `task=unified task.benchmark=Cybersecurity/DependencyAwarePatchScheduling` + - Quick run: `python -m frontier_eval task=unified task.benchmark=Cybersecurity/DependencyAwarePatchScheduling task.runtime.isolation_mode=process algorithm=openevolve algorithm.iterations=0` + - Description: risk-based patch selection and integer-slot scheduling under vulnerability dependencies, maintenance windows, service downtime limits, renewable resource capacities, and rollback costs. + +The task models offline enterprise patch planning. NIST SP 800-40 Rev. 4 and CISA +BOD 22-01 provide the risk-based planning and remediation-priority context; the +benchmark's monetary parameters and generated dependency graphs are synthetic and +are recomputed by the frozen verifier. diff --git a/benchmarks/Cybersecurity/README_zh-CN.md b/benchmarks/Cybersecurity/README_zh-CN.md new file mode 100644 index 00000000..d0441b18 --- /dev/null +++ b/benchmarks/Cybersecurity/README_zh-CN.md @@ -0,0 +1,14 @@ +# Cybersecurity + +本领域收集可执行的网络安全工程优化任务，强调明确的运行约束、风险敏感目标和由验证器独立重算的结果。 + +## 任务列表 + +- `DependencyAwarePatchScheduling` + - `frontier_eval` 任务：`task=unified task.benchmark=Cybersecurity/DependencyAwarePatchScheduling` + - 快速运行：`python -m frontier_eval task=unified task.benchmark=Cybersecurity/DependencyAwarePatchScheduling task.runtime.isolation_mode=process algorithm=openevolve algorithm.iterations=0` + - 简介：在漏洞依赖、维护窗口、服务停机上限、可再生资源容量和回滚成本约束下进行风险感知的补丁选择与整数时隙调度。 + +该任务研究离线企业补丁规划。NIST SP 800-40 Rev. 4 和 CISA BOD 22-01 +提供基于风险的规划和修复优先级背景；benchmark 中的金额参数和依赖图是合成数据， +并由冻结验证器独立重算。