diff --git a/SKILL_TEMPLATE.md b/SKILL_TEMPLATE.md index f647b5fa..dd38d83e 100644 --- a/SKILL_TEMPLATE.md +++ b/SKILL_TEMPLATE.md @@ -65,9 +65,10 @@ Hard rules only — falsifiable and enforceable. No "consider" / "may" language. What the agent emits or changes when this fires. Keep complex logic in a reference/script file (§7), not inline. Every fix recommendation must include -remediation guidance, confidence, blast radius, and behavior-change risk. If this -skill can modify code or configuration, classify each remediation path using the -repo-level `docs/fixer-policy.md` before applying changes. +remediation guidance, confidence, blast radius, behavior-change risk, and a +test strategy that names what proves the issue is fixed. If this skill can +modify code or configuration, classify each remediation path using the repo-level +`docs/fixer-policy.md` before applying changes. **Before (vulnerable):** ``` @@ -86,7 +87,20 @@ remediations: confidence: high # low | medium | high blast_radius: "" behavior_change_risk: medium # low | medium | high - test_strategy: "" + test_strategy: + summary: "" + recommended_tests: + - name: "" + type: regression # static | unit | integration | e2e | regression | manual + purpose: "" + command: "" + expected_result: "" + generated_tests: + - path: "" + type: regression # static | unit | integration | e2e | regression + purpose: "" + command: "" + expected_result: "" ``` ## 5. Verification (falsifiable) @@ -144,7 +158,8 @@ skills/// - [ ] At least one machine-matchable detection signal (regex / structural) - [ ] Rules are hard constraints (no "consider"/"may") - [ ] Before/after remediation example present -- [ ] Every fix recommendation includes `guidance`, `confidence`, `blast_radius`, and `behavior_change_risk` +- [ ] Every fix recommendation includes `guidance`, `confidence`, `blast_radius`, `behavior_change_risk`, and `test_strategy` +- [ ] Every `test_strategy` includes a summary plus recommended tests, generated tests, or both - [ ] Falsifiable verification test defined (binary pass/fail) - [ ] Gotchas: ≥2 false positives + ≥1 precision trap - [ ] `SKILL.md` stays lean; long detail moved to reference files diff --git a/docs/remediation-output.md b/docs/remediation-output.md index aa3a4ce7..577feccc 100644 --- a/docs/remediation-output.md +++ b/docs/remediation-output.md @@ -7,10 +7,42 @@ items under each finding. Every remediation item must include: - `confidence`: `low`, `medium`, or `high` confidence that the recommendation is correct for the observed evidence. - `blast_radius`: the expected scope of systems, files, users, integrations, data, or workflows affected by the change. - `behavior_change_risk`: `low`, `medium`, or `high` risk that the fix changes intended behavior. +- `test_strategy`: a structured validation plan that explains what proves the issue is fixed. -`test_strategy` is optional in the normalized schema. Include it when the skill -can name a concrete validation approach, but keep detailed test strategy policy -in the dedicated test-strategy workstream. +`test_strategy` must include a `summary` and at least one of: + +- `recommended_tests`: tests the user or agent should run when the remediation + recommends a fix but does not generate test files. +- `generated_tests`: test files or cases produced as part of the remediation. + +Each test entry must state the test type, purpose, command or manual check, and +expected result. Generated tests must also identify the test file path. The goal +is to make every fix recommendation falsifiable: the output should say which +test proves the issue is fixed, not only how to change the code. + +Example: + +```yaml +remediations: + - guidance: "Reject untrusted redirect targets unless they resolve to an allowed relative path." + confidence: high + blast_radius: "Redirect handling in the login callback only." + behavior_change_risk: medium + test_strategy: + summary: "Proves external redirect targets are blocked while relative paths still work." + recommended_tests: + - name: "Reject external next URL" + type: integration + purpose: "Confirms the vulnerable open-redirect input no longer succeeds." + command: "bundle exec rspec spec/requests/login_redirect_spec.rb" + expected_result: "External next=https://evil.example is rejected or replaced with a safe default." + generated_tests: + - path: "spec/requests/login_redirect_spec.rb" + type: regression + purpose: "Covers the blocked external redirect and allowed internal redirect cases." + command: "bundle exec rspec spec/requests/login_redirect_spec.rb" + expected_result: "All login redirect regression examples pass." +``` The machine-readable contract lives in [`schemas/finding.schema.json`](../schemas/finding.schema.json). diff --git a/schemas/finding.schema.json b/schemas/finding.schema.json index 5822ac6c..d9109a05 100644 --- a/schemas/finding.schema.json +++ b/schemas/finding.schema.json @@ -71,7 +71,7 @@ "items": { "$ref": "#/$defs/remediation" }, - "description": "Every fix recommendation for this finding. Each item must include remediation guidance, confidence, blast radius, and behavior-change risk." + "description": "Every fix recommendation for this finding. Each item must include remediation guidance, confidence, blast radius, behavior-change risk, and test strategy." } }, "anyOf": [ @@ -86,7 +86,7 @@ "remediation": { "type": "object", "additionalProperties": false, - "required": ["guidance", "confidence", "blast_radius", "behavior_change_risk"], + "required": ["guidance", "confidence", "blast_radius", "behavior_change_risk", "test_strategy"], "properties": { "guidance": { "$ref": "#/$defs/nonEmptyString", @@ -105,8 +105,94 @@ "description": "Risk that the remediation changes intended application or operational behavior." }, "test_strategy": { + "$ref": "#/$defs/testStrategy", + "description": "Required validation strategy proving the remediation fixes the finding. Include recommended tests, generated tests, or both." + } + } + }, + "testStrategy": { + "type": "object", + "additionalProperties": false, + "required": ["summary"], + "properties": { + "summary": { + "$ref": "#/$defs/nonEmptyString", + "description": "Concise explanation of what proves the remediation fixed the issue." + }, + "recommended_tests": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/recommendedTest" + }, + "description": "Tests the user or agent should run when tests are not generated directly." + }, + "generated_tests": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/generatedTest" + }, + "description": "Test files or cases produced as part of the remediation." + } + }, + "anyOf": [ + { + "required": ["recommended_tests"] + }, + { + "required": ["generated_tests"] + } + ] + }, + "recommendedTest": { + "type": "object", + "additionalProperties": false, + "required": ["name", "type", "purpose", "command", "expected_result"], + "properties": { + "name": { + "$ref": "#/$defs/nonEmptyString" + }, + "type": { + "type": "string", + "enum": ["static", "unit", "integration", "e2e", "regression", "manual"] + }, + "purpose": { + "$ref": "#/$defs/nonEmptyString", + "description": "Behavior, vulnerability condition, or regression this test proves." + }, + "command": { + "$ref": "#/$defs/nonEmptyString", + "description": "Command or manual check to execute for this test." + }, + "expected_result": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "generatedTest": { + "type": "object", + "additionalProperties": false, + "required": ["path", "type", "purpose", "command", "expected_result"], + "properties": { + "path": { "$ref": "#/$defs/nonEmptyString", - "description": "Optional validation strategy for the remediation. Kept optional because test strategy requirements are defined separately." + "description": "Path to the generated or modified test file." + }, + "type": { + "type": "string", + "enum": ["static", "unit", "integration", "e2e", "regression"] + }, + "purpose": { + "$ref": "#/$defs/nonEmptyString", + "description": "Behavior, vulnerability condition, or regression this generated test proves." + }, + "command": { + "$ref": "#/$defs/nonEmptyString", + "description": "Command that runs the generated test." + }, + "expected_result": { + "$ref": "#/$defs/nonEmptyString" } } }