feat(remediation): require test strategy for fixes

SKILL_TEMPLATE.md

@@ -65,9 +65,10 @@ Hard rules only — falsifiable and enforceable. No "consider" / "may" language.
 
 What the agent emits or changes when this fires. Keep complex logic in a
 reference/script file (§7), not inline. Every fix recommendation must include
-remediation guidance, confidence, blast radius, and behavior-change risk. If this
-skill can modify code or configuration, classify each remediation path using the
-repo-level `docs/fixer-policy.md` before applying changes.
+remediation guidance, confidence, blast radius, behavior-change risk, and a
+test strategy that names what proves the issue is fixed. If this skill can
+modify code or configuration, classify each remediation path using the repo-level
+`docs/fixer-policy.md` before applying changes.
 
 **Before (vulnerable):**
 ```
@@ -86,7 +87,20 @@ remediations:
     confidence: high                    # low | medium | high
     blast_radius: "<affected files, users, systems, integrations, or workflows>"
     behavior_change_risk: medium        # low | medium | high
-    test_strategy: "<optional validation approach>"
+    test_strategy:
+      summary: "<what proves this remediation fixed the finding>"
+      recommended_tests:
+        - name: "<test name>"
+          type: regression              # static | unit | integration | e2e | regression | manual
+          purpose: "<vulnerable behavior or regression this test proves>"
+          command: "<command or manual check to run>"
+          expected_result: "<binary passing result>"
+      generated_tests:
+        - path: "<path/to/generated_test_file>"
+          type: regression              # static | unit | integration | e2e | regression
+          purpose: "<vulnerable behavior or regression this generated test proves>"
+          command: "<command that runs the generated test>"
+          expected_result: "<binary passing result>"
 ```
 
 ## 5. Verification (falsifiable)
@@ -144,7 +158,8 @@ skills/<domain>/<skill-name>/
 - [ ] At least one machine-matchable detection signal (regex / structural)
 - [ ] Rules are hard constraints (no "consider"/"may")
 - [ ] Before/after remediation example present
-- [ ] Every fix recommendation includes `guidance`, `confidence`, `blast_radius`, and `behavior_change_risk`
+- [ ] Every fix recommendation includes `guidance`, `confidence`, `blast_radius`, `behavior_change_risk`, and `test_strategy`
+- [ ] Every `test_strategy` includes a summary plus recommended tests, generated tests, or both
 - [ ] Falsifiable verification test defined (binary pass/fail)
 - [ ] Gotchas: ≥2 false positives + ≥1 precision trap
 - [ ] `SKILL.md` stays lean; long detail moved to reference files

docs/remediation-output.md

@@ -7,10 +7,42 @@ items under each finding. Every remediation item must include:
 - `confidence`: `low`, `medium`, or `high` confidence that the recommendation is correct for the observed evidence.
 - `blast_radius`: the expected scope of systems, files, users, integrations, data, or workflows affected by the change.
 - `behavior_change_risk`: `low`, `medium`, or `high` risk that the fix changes intended behavior.
+- `test_strategy`: a structured validation plan that explains what proves the issue is fixed.
 
-`test_strategy` is optional in the normalized schema. Include it when the skill
-can name a concrete validation approach, but keep detailed test strategy policy
-in the dedicated test-strategy workstream.
+`test_strategy` must include a `summary` and at least one of:
+
+- `recommended_tests`: tests the user or agent should run when the remediation
+  recommends a fix but does not generate test files.
+- `generated_tests`: test files or cases produced as part of the remediation.
+
+Each test entry must state the test type, purpose, command or manual check, and
+expected result. Generated tests must also identify the test file path. The goal
+is to make every fix recommendation falsifiable: the output should say which
+test proves the issue is fixed, not only how to change the code.
+
+Example:
+
+```yaml
+remediations:
+  - guidance: "Reject untrusted redirect targets unless they resolve to an allowed relative path."
+    confidence: high
+    blast_radius: "Redirect handling in the login callback only."
+    behavior_change_risk: medium
+    test_strategy:
+      summary: "Proves external redirect targets are blocked while relative paths still work."
+      recommended_tests:
+        - name: "Reject external next URL"
+          type: integration
+          purpose: "Confirms the vulnerable open-redirect input no longer succeeds."
+          command: "bundle exec rspec spec/requests/login_redirect_spec.rb"
+          expected_result: "External next=https://evil.example is rejected or replaced with a safe default."
+      generated_tests:
+        - path: "spec/requests/login_redirect_spec.rb"
+          type: regression
+          purpose: "Covers the blocked external redirect and allowed internal redirect cases."
+          command: "bundle exec rspec spec/requests/login_redirect_spec.rb"
+          expected_result: "All login redirect regression examples pass."
+```
 
 The machine-readable contract lives in
 [`schemas/finding.schema.json`](../schemas/finding.schema.json).

schemas/finding.schema.json

@@ -71,7 +71,7 @@
           "items": {
             "$ref": "#/$defs/remediation"
           },
-          "description": "Every fix recommendation for this finding. Each item must include remediation guidance, confidence, blast radius, and behavior-change risk."
+          "description": "Every fix recommendation for this finding. Each item must include remediation guidance, confidence, blast radius, behavior-change risk, and test strategy."
         }
       },
       "anyOf": [
@@ -86,7 +86,7 @@
     "remediation": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["guidance", "confidence", "blast_radius", "behavior_change_risk"],
+      "required": ["guidance", "confidence", "blast_radius", "behavior_change_risk", "test_strategy"],
       "properties": {
         "guidance": {
           "$ref": "#/$defs/nonEmptyString",
@@ -105,8 +105,94 @@
           "description": "Risk that the remediation changes intended application or operational behavior."
         },
         "test_strategy": {
+          "$ref": "#/$defs/testStrategy",
+          "description": "Required validation strategy proving the remediation fixes the finding. Include recommended tests, generated tests, or both."
+        }
+      }
+    },
+    "testStrategy": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["summary"],
+      "properties": {
+        "summary": {
+          "$ref": "#/$defs/nonEmptyString",
+          "description": "Concise explanation of what proves the remediation fixed the issue."
+        },
+        "recommended_tests": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "$ref": "#/$defs/recommendedTest"
+          },
+          "description": "Tests the user or agent should run when tests are not generated directly."
+        },
+        "generated_tests": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "$ref": "#/$defs/generatedTest"
+          },
+          "description": "Test files or cases produced as part of the remediation."
+        }
+      },
+      "anyOf": [
+        {
+          "required": ["recommended_tests"]
+        },
+        {
+          "required": ["generated_tests"]
+        }
+      ]
+    },
+    "recommendedTest": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "type", "purpose", "command", "expected_result"],
+      "properties": {
+        "name": {
+          "$ref": "#/$defs/nonEmptyString"
+        },
+        "type": {
+          "type": "string",
+          "enum": ["static", "unit", "integration", "e2e", "regression", "manual"]
+        },
+        "purpose": {
+          "$ref": "#/$defs/nonEmptyString",
+          "description": "Behavior, vulnerability condition, or regression this test proves."
+        },
+        "command": {
+          "$ref": "#/$defs/nonEmptyString",
+          "description": "Command or manual check to execute for this test."
+        },
+        "expected_result": {
+          "$ref": "#/$defs/nonEmptyString"
+        }
+      }
+    },
+    "generatedTest": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["path", "type", "purpose", "command", "expected_result"],
+      "properties": {
+        "path": {
           "$ref": "#/$defs/nonEmptyString",
-          "description": "Optional validation strategy for the remediation. Kept optional because test strategy requirements are defined separately."
+          "description": "Path to the generated or modified test file."
+        },
+        "type": {
+          "type": "string",
+          "enum": ["static", "unit", "integration", "e2e", "regression"]
+        },
+        "purpose": {
+          "$ref": "#/$defs/nonEmptyString",
+          "description": "Behavior, vulnerability condition, or regression this generated test proves."
+        },
+        "command": {
+          "$ref": "#/$defs/nonEmptyString",
+          "description": "Command that runs the generated test."
+        },
+        "expected_result": {
+          "$ref": "#/$defs/nonEmptyString"
         }
       }
     }