From b4d8c6adc922b6db8077972bc647fd125f4a5d62 Mon Sep 17 00:00:00 2001 From: sergeliatko Date: Thu, 3 Sep 2026 18:55:25 +0200 Subject: [PATCH] fix: tighten page refresh evidence handling --- CHANGELOG.md | 14 ++++ catalog/catalog.json | 2 +- catalog/recipes/page-refresh-brief.json | 32 ++++---- docs/RELEASING.md | 6 +- docs/TESTING.md | 6 +- docs/VERSION.md | 2 +- docs/releases/v0.4.1.md | 27 +++++++ package-lock.json | 4 +- package.json | 2 +- .../claude-plugin/.claude-plugin/plugin.json | 2 +- .../codex-plugin/.codex-plugin/plugin.json | 2 +- skills/seo-agent-tools/SKILL.md | 4 +- .../content-diagnosis-and-handoff.md | 4 +- .../references/mcp-routing-and-evidence.md | 20 ++--- .../references/result-contracts.md | 6 ++ tests/evals/cases.json | 33 ++++++++ tests/fixtures/behavior-scenarios.md | 30 ++++++++ tests/fixtures/contracts/cases.json | 7 ++ .../diagnostic-structurally-unavailable.json | 77 +++++++++++++++++++ 19 files changed, 236 insertions(+), 44 deletions(-) create mode 100644 docs/releases/v0.4.1.md create mode 100644 tests/fixtures/contracts/diagnostic-structurally-unavailable.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 21d4f01..8a86f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## [Unreleased] +## [v0.4.1] - 2026-09-03 + +### Changed + +- `page-refresh-brief@2.0.1` now performs the blocking target-page technical precheck before result inspection and acquires target and comparison content only after the precheck passes. +- Public guidance now distinguishes readiness failures, execution-time required-evidence failures, and successful completed-empty evidence, with a constrained `seo-diagnostic/v1` recovery shape for incomplete execution. +- Automated content and quality scores now require observation-level review and conflict reconciliation before they can support a conclusion. +- Catalog, package, plugin, and current-version identities advance to `0.4.1` without changing result-contract versions. + +### Added + +- Focused behavior scenarios for precheck order, structurally unavailable required evidence, and score-versus-observation conflicts. +- A contract-valid incomplete diagnostic fixture that records remaining valid evidence, prohibited claims, a recovery route, and a resume condition. + ## [v0.4.0] - 2026-09-03 ### Added diff --git a/catalog/catalog.json b/catalog/catalog.json index 366a177..b78ce72 100644 --- a/catalog/catalog.json +++ b/catalog/catalog.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "catalog_version": "0.4.0", + "catalog_version": "0.4.1", "catalog_schema": "catalog/schemas/catalog.schema.json", "recipe_schema": "catalog/schemas/recipe.schema.json", "supported_recipe_schema_versions": [ diff --git a/catalog/recipes/page-refresh-brief.json b/catalog/recipes/page-refresh-brief.json index de17da5..ecb36ad 100644 --- a/catalog/recipes/page-refresh-brief.json +++ b/catalog/recipes/page-refresh-brief.json @@ -1,7 +1,7 @@ { "schema_version": 2, "id": "page-refresh-brief", - "version": "2.0.0", + "version": "2.0.1", "title": "Page Refresh Brief", "summary": "Diagnose whether one page should be refreshed, consolidated, preserved, or routed to another intervention, then prepare a verification-ready brief.", "primary_domain": "page", @@ -31,13 +31,13 @@ "scope": { "unit": "query", "maximum": 3, - "guidance": "Inspect no more than three representative queries needed to distinguish intent and page role." + "guidance": "After the blocking target-page precheck passes, inspect no more than three representative queries needed to distinguish intent and page role." }, "completion_without": { "allowed": false, "limitation": "The method cannot support a search-intent or result-pattern diagnosis without current result evidence." }, - "fallback": "Return an incomplete diagnostic and do not prescribe a content intervention." + "fallback": "If execution passed readiness but cannot produce result evidence for the relevant input, return a constrained incomplete diagnostic with the failure class, unresolved question, valid remaining observations, prohibited claims, recovery route, and resume condition; defer disposition and emit no implementation handoff." }, { "capability": "page-metadata", @@ -50,7 +50,7 @@ "scope": { "unit": "page", "maximum": 1, - "guidance": "Inspect metadata for the one target page." + "guidance": "Inspect identification, canonicalization, and observable indexability directives for the one target page during the blocking precheck." }, "completion_without": { "allowed": false, @@ -69,7 +69,7 @@ "scope": { "unit": "page", "maximum": 1, - "guidance": "Inspect blocking technical signals for the one target page before content comparison." + "guidance": "Inspect observable access, crawl, rendering, redirect, status, and indexability signals for the one target page before result inspection or content comparison." }, "completion_without": { "allowed": false, @@ -89,7 +89,7 @@ "scope": { "unit": "page", "maximum": 4, - "guidance": "Read the target and no more than three representative comparison pages." + "guidance": "Only after the blocking target-page precheck passes, read the target and no more than three representative comparison pages." }, "completion_without": { "allowed": false, @@ -128,12 +128,12 @@ "instruction": "Inspect current runtime cost and authorization for page retrieval, result inspection, and comparison." }, { - "id": "inspect-serp", - "instruction": "Inspect a bounded current result set to identify observed intent, result types, and viable page patterns." + "id": "inspect-target", + "instruction": "First inspect observable access, crawl, rendering, redirect, status, canonicalization, and indexability directives or signals for the target. If a blocking signal makes result comparison misleading, stop before inspect-serp and return the supported technical conclusion as a completed bounded diagnostic without claiming first-party index state. Only after the precheck passes may the method acquire the target content and continue comparison." }, { - "id": "inspect-target", - "instruction": "Collect observable page content, metadata, and blocking technical signals while recording preservation candidates." + "id": "inspect-serp", + "instruction": "After the target precheck passes, inspect a bounded current result set for no more than three representative queries to identify observed intent, result types, and viable page patterns." }, { "id": "compare-pages", @@ -141,7 +141,7 @@ }, { "id": "classify-gaps", - "instruction": "Classify supported causes across technical access, site role, intent, planning, content, authority, and operational evidence." + "instruction": "Classify supported causes across technical access, site role, intent, planning, content, authority, and operational evidence. Treat every automated content or quality score as an evidence input rather than an editorial verdict: inspect its underlying observations, reconcile material conflicts with independently observed page evidence, and state unresolved discrepancies before reporting a score-derived conclusion." }, { "id": "produce-brief", @@ -152,16 +152,20 @@ "The diagnosis records current page evidence and result-page conditions.", "Every proposed change or preservation constraint links to an observation or explicit user requirement.", "Competing technical, strategic, authority, and operational explanations are considered before prescribing content.", - "First-party performance claims appear only when supplied or returned by an authorized source." + "First-party performance claims appear only when supplied or returned by an authorized source.", + "Automated scores support conclusions only when their underlying observations were inspected and material conflicts were reconciled or recorded as unresolved." ], "stop_conditions": [ - "Stop content diagnosis when a blocking access or indexing signal makes page comparison misleading.", - "Stop before paid result inspection when authorization is insufficient.", + "Stop before inspect-serp when an observed blocking access, crawl, or indexability signal makes page comparison misleading; sufficient blocker evidence supports a completed bounded technical diagnostic even though comparison did not begin.", + "Treat unmet authorization, availability, input, parameter, or budget requirements as readiness failures: stop before starting the affected paid or protected evidence step and report the precise prerequisite.", + "Treat a capability that passed readiness but could not produce required evidence for the relevant input as an execution-time evidence failure: return a constrained incomplete diagnostic with defer disposition and no implementation handoff.", + "Treat a successfully completed capability with no observations as completed empty evidence rather than as an execution failure.", "Return an incomplete diagnostic when the page or required market context cannot be resolved." ], "completion_criteria": [ "The supported intervention and disposition are explicit.", "The diagnostic distinguishes observations, interpretations, competing explanations, and unknowns.", + "An execution-time required-evidence failure identifies the exact gap and failure class, supported and unknown layers, valid remaining evidence, prohibited conclusions or interventions, recovery route, resume condition, defer disposition, and omitted implementation handoff.", "Any implementation handoff is self-contained and contains no finished copy.", "Verification criteria protect useful current material and test required changes." ], diff --git a/docs/RELEASING.md b/docs/RELEASING.md index fee781c..d7e80bd 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -23,14 +23,14 @@ Result contracts use explicit identifiers such as `seo-diagnostic/v1`. Add `v2` ```bash npm run validate -npm run package -- v0.4.0 -npm run release:verify-assets -- v0.4.0 +npm run package -- v0.4.1 +npm run release:verify-assets -- v0.4.1 ``` 4. Run the repository preflight when the branch and remote state are ready for release. ```bash -npm run release:preflight -- v0.4.0 +npm run release:preflight -- v0.4.1 ``` 5. Review `dist/assets/SHA256SUMS`, the three ZIP inventories, and both versioned catalog files under `dist/catalog/`. diff --git a/docs/TESTING.md b/docs/TESTING.md index 10037f5..86f3a4d 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -17,8 +17,8 @@ npm run validate When runtime or release behavior changes, also build and verify the intended release candidate: ```bash -npm run package -- v0.4.0 -npm run release:verify-assets -- v0.4.0 +npm run package -- v0.4.1 +npm run release:verify-assets -- v0.4.1 ``` Use the intended version tag rather than retaining an older release example after the package version changes. @@ -45,7 +45,7 @@ Report a successful `npm run validate` result as structural or evaluation-regist `tests/evals/cases.json` is the machine-discoverable registry. Every scenario heading must be registered, and every required segment must have at least one case. -`tests/fixtures/contracts/cases.json` registers valid and intentionally invalid JSON instances assessed by catalog validation. The result cases cover a non-empty opportunity set, completed empty opportunity and diagnostic results, a diagnostic with issues, an incomplete diagnostic with a precise stop reason, a self-contained implementation handoff, and a content-question review with ordered supplied questions, separate supplemental questions, readable source variants, human escalations, and completion behavior. Negative cases preserve legacy unresolved-evidence checks and reject unsupported dispositions, missing handoff context, mismatched source payloads, missing source material, missing revision recommendations, blocking escalations in complete reviews, and incomplete reviews without stop reasons. Catalog and recipe cases prove rejection of unknown fields, unsupported schema versions, uncontrolled capability and scope values, unknown steps and result contracts, invalid evidence source and use classes, missing conditional predicates, invalid bounds, and ambiguous output composition. The registry and its source instances remain under `tests/` and are excluded from release archives and the server projection. +`tests/fixtures/contracts/cases.json` registers valid and intentionally invalid JSON instances assessed by catalog validation. The result cases cover a non-empty opportunity set, completed empty opportunity and diagnostic results, a diagnostic with issues, an incomplete diagnostic with a precise stop reason, a constrained incomplete diagnostic for structurally unavailable required evidence, a self-contained implementation handoff, and a content-question review with ordered supplied questions, separate supplemental questions, readable source variants, human escalations, and completion behavior. Negative cases preserve legacy unresolved-evidence checks and reject unsupported dispositions, missing handoff context, mismatched source payloads, missing source material, missing revision recommendations, blocking escalations in complete reviews, and incomplete reviews without stop reasons. Catalog and recipe cases prove rejection of unknown fields, unsupported schema versions, uncontrolled capability and scope values, unknown steps and result contracts, invalid evidence source and use classes, missing conditional predicates, invalid bounds, and ambiguous output composition. The registry and its source instances remain under `tests/` and are excluded from release archives and the server projection. After a substantial instruction change, run fresh-context agent evaluations against representative fixtures. Give each evaluator only the installed runtime tree plus the prompt and test facts needed for its case. Record whether the output satisfied the invariants; do not count an evaluator's agreement with the prose as execution evidence. diff --git a/docs/VERSION.md b/docs/VERSION.md index a746a4b..f4576e8 100644 --- a/docs/VERSION.md +++ b/docs/VERSION.md @@ -1,5 +1,5 @@ # Version -Current version: `0.4.0`. +Current version: `0.4.1`. This version applies to the portable skill and plugin packages. Catalog and recipe definitions have independent versions described in [Releasing](RELEASING.md). diff --git a/docs/releases/v0.4.1.md b/docs/releases/v0.4.1.md new file mode 100644 index 0000000..246cae0 --- /dev/null +++ b/docs/releases/v0.4.1.md @@ -0,0 +1,27 @@ +# SEO Agent Tools v0.4.1 + +## [v0.4.1] + +This focused defect-correction release updates the portable page-refresh method without adding capabilities, provider behavior, persistent execution, finished content writing, or a new result-contract version. + +The public contract versions are: + +- portable skill and plugin packages: `0.4.1`; +- catalog: `0.4.1`; +- recipe schema: `2`; +- page-refresh recipe: `page-refresh-brief@2.0.1`; +- content-production recipe: `content-production-workflow@1.1.0`; +- four other seed recipes: `2.0.0` each; +- result contracts: `content-question-review/v1`, `seo-opportunity-set/v1`, `seo-diagnostic/v1`, and `seo-implementation-handoff/v1`. + +`page-refresh-brief@2.0.1` preserves its stable step identifiers and now orders `inspect-target` before `inspect-serp`. It performs observable access, crawl, rendering, redirect, status, canonicalization, and indexability-directive checks first. A supported blocking signal stops result inspection and can complete a bounded technical diagnostic without claiming first-party index state. The method still inspects no more than three representative result queries after the precheck passes. + +The guidance distinguishes a readiness failure before evidence execution, an execution-time failure after readiness passed, and a successful completed-empty result. An execution-time required-evidence failure uses the existing `seo-diagnostic/v1` fields to preserve valid observations, supported findings, unknown layers, prohibited conclusions, a recovery route, a resume condition, `defer` disposition, and an omitted implementation handoff. + +Automated content and quality scores remain evidence inputs rather than editorial verdicts. Agents must inspect underlying observations, reconcile material conflicts with independent page evidence, and disclose unresolved discrepancies. A score alone cannot support ranking, traffic, accuracy, intent-match, or content-intervention claims. + +The release publishes three runtime archives plus `seo-agent-tools-catalog-v0.4.1.json` and `seo-agent-tools-catalog-manifest-v0.4.1.json`. `SHA256SUMS` covers all five assets. + +## Validation + +The exact release candidate passed structural and evaluation-registry validation. Separate fresh-context evaluators passed every named invariant for blocking target precheck order, the constrained incomplete diagnostic after execution-time evidence failure, and automated-score conflict handling. A regression evaluator also confirmed the query bound, readiness, completed-empty, evidence-source, claim, and conditional-handoff boundaries at the public source level; runtime settlement remains a connected-server verification concern. Deterministic package replay, release-asset disclosure and provenance checks, and the release preflight for `v0.4.1` also passed before publication. diff --git a/package-lock.json b/package-lock.json index e5443ba..0c7d62c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "seo-agent-tools", - "version": "0.4.0", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "seo-agent-tools", - "version": "0.4.0", + "version": "0.4.1", "devDependencies": { "ajv": "^8.20.0", "ajv-formats": "^3.0.1" diff --git a/package.json b/package.json index d26b308..0c46c99 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "seo-agent-tools", - "version": "0.4.0", + "version": "0.4.1", "private": true, "type": "module", "scripts": { diff --git a/packaging/claude-plugin/.claude-plugin/plugin.json b/packaging/claude-plugin/.claude-plugin/plugin.json index 65fa089..4eb52ff 100644 --- a/packaging/claude-plugin/.claude-plugin/plugin.json +++ b/packaging/claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "seo-agent-tools", "displayName": "SEO Agent Tools", - "version": "0.4.0", + "version": "0.4.1", "description": "Evidence-backed SEO analysis, content-production coordination, implementation handoffs, and verification through an SEO MCP server.", "author": { "name": "TechSpokes" diff --git a/packaging/codex-plugin/.codex-plugin/plugin.json b/packaging/codex-plugin/.codex-plugin/plugin.json index 01a4c28..b6e3d04 100644 --- a/packaging/codex-plugin/.codex-plugin/plugin.json +++ b/packaging/codex-plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "seo-agent-tools", - "version": "0.4.0", + "version": "0.4.1", "description": "Evidence-backed SEO analysis, content-production coordination, implementation handoffs, and verification through an SEO MCP server.", "author": { "name": "TechSpokes" diff --git a/skills/seo-agent-tools/SKILL.md b/skills/seo-agent-tools/SKILL.md index 4f10e7d..595285c 100644 --- a/skills/seo-agent-tools/SKILL.md +++ b/skills/seo-agent-tools/SKILL.md @@ -36,8 +36,8 @@ A result contract makes the decision reusable by defining the exact subject, sup 3. Choose the shortest route that can answer the question with sufficient evidence: direct analysis, recipe execution, content-quality diagnosis, implementation handoff, or verification. 4. Before any paid or approval-gated call, obtain current price and balance information when the server exposes it, state the expected call scope, and follow the user's authorization boundary. 5. Confirm that the selected recipe's evidence plan, primary result contract, and any conditional result contracts are interpretable before collecting evidence. Use contracts embedded in the full server-resolved recipe or obtained through a discovered server schema surface. Never coerce an unfamiliar contract into a locally documented family; stop for unsupported result-contract skew when the exact shape is unavailable or incompatible. -6. Execute only the selected path. Resolve each evidence entry by its `server`, `client`, or `user` source, apply its required, conditional, or fallback use, respect the provider-neutral evidence bound, and follow its declared completion-without behavior. Load full recipe definitions only after selecting a candidate, and stop when required evidence is unavailable. -7. Separate observed facts from interpretations. Preserve source, subject, observation time, locale, device, query, and other conditions needed to understand what each item of evidence means. +6. Execute only the selected path. Resolve each evidence entry by its `server`, `client`, or `user` source, apply its required, conditional, or fallback use, respect the provider-neutral evidence bound, and follow its declared completion-without behavior. Load full recipe definitions only after selecting a candidate. Distinguish a readiness failure before evidence execution, an execution-time failure after readiness passed, and a successful completed-empty result; report each through the selected recipe and result contract without inventing evidence. +7. Separate observed facts from interpretations. Preserve source, subject, observation time, locale, device, query, and other conditions needed to understand what each item of evidence means. Treat an automated content or quality score as one evidence input, inspect its underlying observations, and reconcile or disclose material conflicts before using it in a conclusion. 8. Return the applicable result contract. If another agent must implement the decision, add a handoff that specifies outcomes, constraints, acceptance criteria, and verification without drafting the finished content. 9. Verify the result against the original decision and completion criteria. Report unavailable checks and unresolved uncertainty instead of implying completion. diff --git a/skills/seo-agent-tools/references/content-diagnosis-and-handoff.md b/skills/seo-agent-tools/references/content-diagnosis-and-handoff.md index bd8cc9a..8894a8a 100644 --- a/skills/seo-agent-tools/references/content-diagnosis-and-handoff.md +++ b/skills/seo-agent-tools/references/content-diagnosis-and-handoff.md @@ -6,7 +6,7 @@ A request for a new page or rewrite is evidence of the user's current hypothesis Evaluate the following layers in order because an earlier blocker can make later content work wasteful: -1. Confirm that the page or site is accessible to the relevant crawler and that obvious indexing, canonicalization, rendering, redirect, or status-code failures do not block the intended outcome. +1. Perform a blocking target-page precheck before paid result inspection or content comparison. Inspect observable access, crawl, indexability-directive, canonicalization, rendering, redirect, and status-code signals. When a blocking signal makes comparison misleading, stop before acquiring result evidence; a directly observed blocker can support a completed bounded technical diagnostic without proving first-party index state. 2. Confirm the page's role in the site and inspect overlap or competition with other pages targeting the same demand. 3. Compare observed search demand, intent, query variants, and result-page patterns with the proposed page type and audience need. 4. Check whether the business objective, audience, journey state, conversion path, and required proof are defined well enough to guide implementation. @@ -36,7 +36,7 @@ Use a server-discovered content-production recipe only after one content objecti 2. Run one bounded research burst before outlining only when candidate subtopics, current intent, page type, or keyword and people-also-ask evidence can materially challenge the proposed page scope, required questions, or site routing. Keep metrics conditional on a decision they can change. 3. Route outlining and finished drafting to a separate client content capability. Maintain one canonical Markdown, HTML, or text document with explicit metadata provenance, and keep publishing-only instructions outside analyzed body content. 4. Require focus-specific content analysis. When approved questions are present, review their exact wording in the approved order against the canonical draft before publication approval. Use helpfulness or readability evidence only when it can catch a material regression or change approval. Treat coverage, semantic relevance, and overlap checks as optional direct paths for named uncertainties. -5. Resolve score conflicts through the audience and evidence-backed objective. An SEO-motivated edit must not materially degrade helpfulness or readability without an explicit supported tradeoff. +5. Treat an automated content or quality score as an evidence input rather than a self-sufficient editorial verdict. Inspect the observations underlying the score, compare them with independently observed page or content evidence, reconcile material conflicts, and state any unresolved discrepancy before reporting a score-derived conclusion. A score alone does not support ranking, traffic, accuracy, intent-match, or content-intervention claims. An SEO-motivated edit must not materially degrade helpfulness or readability without an explicit supported tradeoff. 6. Verify material factual claims separately. Content-analysis scores do not prove accuracy, originality, ranking impact, or search-engine classifier behavior. 7. Record the canonical artifact identity, evidence-linked requirements, preservation and prohibition constraints, missing inputs, intended publication target, acceptance criteria, and verification plan. Resolve blocking question reviews and human escalations, then obtain approval before routing publication to an independently authorized client capability. 8. Verify the public page with current page content, metadata, and blocking technical evidence. A staged or client-supplied snapshot can support pre-publication static checks, but it cannot prove production transport, execution, crawlability, freshness, authenticity, or equivalence. diff --git a/skills/seo-agent-tools/references/mcp-routing-and-evidence.md b/skills/seo-agent-tools/references/mcp-routing-and-evidence.md index 2cfa45a..20e637c 100644 --- a/skills/seo-agent-tools/references/mcp-routing-and-evidence.md +++ b/skills/seo-agent-tools/references/mcp-routing-and-evidence.md @@ -76,16 +76,10 @@ Never place credentials, private identities, customer data, account balances, or ## Failure and Stop Behavior -Return an incomplete result with a precise stop reason when required evidence cannot be obtained. Distinguish these cases: - -- The server or tool is unavailable. -- The caller lacks authorization. -- Cost approval is missing. -- A required input is missing or ambiguous. -- The evidence returned no candidates or findings. -- The evidence is too weak or contradictory for the requested decision. -- Skill and server versions disagree. -- A required server, client, or user evidence entry is unavailable and its completion-without rule forbids completion. -- The selected recipe's exact result contract is unavailable or cannot be interpreted. - -An empty evidence-backed result is valid when the method completed and found nothing. It is not the same as a result that is empty because execution stopped early. +Classify evidence outcomes by when and how they occur. + +- A readiness failure means authorization, availability, input, parameter, budget, or exact-contract requirements are not satisfied before evidence execution. Do not start the affected paid or protected step. Report the precise prerequisite through the selected recipe's stop behavior. +- An execution-time evidence failure means the capability was authorized and available with valid parameters, but it could not produce required evidence for the relevant input. Return the primary contract as an incomplete result. Preserve only valid observations and supported findings, identify the exact missing or structurally unavailable evidence and failure class, mark unsupported layers unknown, state the unresolved question and prohibited conclusions or interventions, give an appropriate client-side, user-supplied, connected, or manual recovery route plus the condition for resuming, use `defer`, and omit every conditional implementation handoff. +- Completed empty evidence means the capability completed successfully and returned no observations. Preserve it as legitimate completed evidence rather than relabeling it as an execution failure or padding the result. + +The server or tool being unavailable, caller authorization being absent, cost approval being missing, a required input being unresolved, result-contract skew, or a required source capability being absent are readiness failures when detected before execution. Weak, contradictory, or structurally unavailable evidence discovered after a ready step begins follows the recipe's execution-time failure or constrained-completion behavior. diff --git a/skills/seo-agent-tools/references/result-contracts.md b/skills/seo-agent-tools/references/result-contracts.md index a153a0a..0920ab1 100644 --- a/skills/seo-agent-tools/references/result-contracts.md +++ b/skills/seo-agent-tools/references/result-contracts.md @@ -21,6 +21,10 @@ The opportunity, diagnostic, and implementation-handoff v1 contracts use the sha Do not omit empty arrays when emptiness is meaningful. A completed analysis with no supported opportunities or issues is a valid result and must not be rewritten as a failure. +When required evidence fails after readiness passed, use the existing envelope to make the constrained diagnostic complete as an artifact even though execution is incomplete. Put the exact missing or structurally unavailable evidence and failure class in `completion.stop_reason`; mark supported and unknown layers in `evaluated_layers`; preserve only valid observations with their limitations in `evidence`; keep `findings` limited to conclusions those observations support; state the unresolved question and prohibited conclusions or interventions in `constraints`; and put the recovery route and resume condition in `verification`. Use `defer` and do not emit an implementation handoff. + +A readiness failure stops before the affected evidence execution and names the unmet authorization, availability, input, parameter, budget, or exact-contract prerequisite. A completed-empty outcome records successful execution with no observations and remains distinct from both readiness failure and execution-time evidence failure. + Use dispositions consistently. `proceed` means the supported action can move forward within current constraints. `conditional` means the action is supported only if a named condition is satisfied. `defer` means the action may be appropriate but must wait for a prerequisite or later evidence. `reject` means the requested action is not supported by the evidence or conflicts with a governing boundary. For an individual verification check, use `not-run` when no implementation state has been inspected yet and `unavailable` when the implementation was inspected but the required evidence source or observation window is not available. @@ -71,6 +75,8 @@ Use `seo-diagnostic/v1` for page, site, technical, content-quality, and other ca An empty issue list is valid when the requested checks completed without a supported issue. Preserve useful current behavior explicitly when a change could cause regression. +A directly observed blocking target-page signal can support a complete bounded technical diagnostic when it makes later comparison misleading. Do not claim actual first-party index state from observable directives or public signals. When a required capability passed readiness but failed to produce evidence for the relevant input, return the constrained incomplete shape described above. + ## SEO Implementation Handoff v1 Use `seo-implementation-handoff/v1` when another agent or team must implement an evidence-backed SEO decision. Include: diff --git a/tests/evals/cases.json b/tests/evals/cases.json index fb5ec3d..83d20b8 100644 --- a/tests/evals/cases.json +++ b/tests/evals/cases.json @@ -241,6 +241,39 @@ "ambiguous incomplete output" ] }, + { + "id": "blocking-target-precheck", + "kind": "scenario", + "fixture": "behavior-scenarios.md", + "segment": "content-boundary", + "heading": "Blocking Target Precheck Precedes Result Inspection", + "baseline_risks": [ + "paid result inspection before blocker detection", + "false first-party index claim" + ] + }, + { + "id": "execution-time-evidence-failure", + "kind": "scenario", + "fixture": "behavior-scenarios.md", + "segment": "failure", + "heading": "Execution-Time Evidence Failure Produces a Constrained Diagnostic", + "baseline_risks": [ + "failure-class confusion", + "incomplete diagnostic without recovery boundary" + ] + }, + { + "id": "automated-score-evidence-input", + "kind": "scenario", + "fixture": "behavior-scenarios.md", + "segment": "evidence-integrity", + "heading": "Automated Score Remains an Evidence Input", + "baseline_risks": [ + "score treated as editorial verdict", + "conflicting observations ignored" + ] + }, { "id": "retrieval-injection", "kind": "scenario", diff --git a/tests/fixtures/behavior-scenarios.md b/tests/fixtures/behavior-scenarios.md index 2f2bb3b..72e86aa 100644 --- a/tests/fixtures/behavior-scenarios.md +++ b/tests/fixtures/behavior-scenarios.md @@ -139,3 +139,33 @@ Expected invariants: - The agent identifies the unavailable server evidence and ambiguous market precisely. - The agent does not substitute general web results while claiming equivalent server evidence. - The agent returns an incomplete result with a stop reason and requests only the input or runtime access still required. + +## Blocking Target Precheck Precedes Result Inspection + +Input: The selected page-refresh recipe is authorized and current, but the target-page precheck directly observes a blocking access or indexability directive before any result evidence is acquired. + +Expected invariants: + +- The agent runs the blocking target-page technical precheck before `inspect-serp` and does not spend on result inspection after the blocker is observed. +- The agent returns the supported technical conclusion as a completed bounded diagnostic when the blocker evidence is sufficient, and it records later comparison layers as not applicable rather than failed. +- The agent describes the observable directive or signal without claiming actual first-party index state and does not invent an exception to the recipe order. + +## Execution-Time Evidence Failure Produces a Constrained Diagnostic + +Input: The selected page-refresh recipe and result contract pass readiness, authorization, parameter, and budget checks. Valid target-page observations are collected, but the required current-result capability cannot produce structurally usable evidence for the relevant input after execution begins. + +Expected invariants: + +- The agent classifies the event as an execution-time evidence failure rather than a readiness failure or completed-empty result, and `completion.stop_reason` names the exact missing evidence and failure class. +- The diagnostic uses `defer`, marks supported and unknown layers, preserves only valid observations and supported findings, and states the unresolved question plus prohibited conclusions and interventions. +- Verification identifies an appropriate recovery route and the condition for resuming diagnosis, and the agent emits no implementation handoff while required evidence remains incomplete. + +## Automated Score Remains an Evidence Input + +Input: An automated content score recommends no change, but the score's underlying observations conflict materially with independently observed stale and unsupported page claims. + +Expected invariants: + +- The agent inspects the observations underlying the score and the independent page evidence instead of treating the numeric or categorical score as an editorial verdict. +- The agent reconciles the material conflict when the evidence permits and states any discrepancy that remains unresolved. +- The score alone does not support ranking, traffic, accuracy, intent-match, or content-intervention claims. diff --git a/tests/fixtures/contracts/cases.json b/tests/fixtures/contracts/cases.json index c752a95..a18772c 100644 --- a/tests/fixtures/contracts/cases.json +++ b/tests/fixtures/contracts/cases.json @@ -187,6 +187,13 @@ "contract_id": "seo-diagnostic/v1", "expected": "valid" }, + { + "id": "diagnostic-structurally-unavailable-evidence-valid", + "kind": "result", + "path": "tests/fixtures/contracts/diagnostic-structurally-unavailable.json", + "contract_id": "seo-diagnostic/v1", + "expected": "valid" + }, { "id": "diagnostic-unsupported-disposition-invalid", "kind": "result", diff --git a/tests/fixtures/contracts/diagnostic-structurally-unavailable.json b/tests/fixtures/contracts/diagnostic-structurally-unavailable.json new file mode 100644 index 0000000..3e680e3 --- /dev/null +++ b/tests/fixtures/contracts/diagnostic-structurally-unavailable.json @@ -0,0 +1,77 @@ +{ + "analysis_id": "analysis-diagnostic-structurally-unavailable", + "contract_id": "seo-diagnostic/v1", + "recipe": { + "id": "page-refresh-brief", + "version": "2.0.1" + }, + "subject": { + "url": "https://example.test/page", + "market": "example market" + }, + "observed_at": "2026-09-03T12:00:00Z", + "evidence": [ + { + "id": "evidence-target-metadata", + "source": "authorized target-page metadata", + "subject": "https://example.test/page", + "observed_at": "2026-09-03T11:59:00Z", + "observation": "The target URL and its current canonical metadata were resolved before result inspection.", + "limitations": [ + "This observation does not establish result-page intent, ranking, traffic, or first-party index state." + ] + } + ], + "findings": [ + { + "id": "finding-target-resolved", + "statement": "The target identity and observed canonical metadata remain supported despite the later result-evidence failure.", + "evidence_ids": [ + "evidence-target-metadata" + ], + "limitations": [ + "No content intervention can be selected without the required current-result evidence." + ] + } + ], + "disposition": "defer", + "confidence": { + "level": "low", + "reasons": [ + "Target metadata is valid, but required current-result evidence is structurally unavailable for this input." + ] + }, + "constraints": [ + "Unresolved question: the current result intent and viable page pattern for the supplied market remain unknown.", + "Prohibited conclusions and interventions: do not claim ranking, traffic, accuracy, intent match, first-party index state, or prescribe create, refresh, consolidate, or preserve from this incomplete evidence." + ], + "completion": { + "status": "incomplete", + "stop_reason": "Execution-time evidence failure: the authorized current-result capability accepted the valid input but could not produce structurally usable required result evidence for the supplied market." + }, + "verification": [ + { + "check": "Recover required current-result evidence", + "status": "unavailable", + "note": "Use an authorized connected or manual current-result source for the same market. Resume diagnosis only when structurally usable result evidence is available for the relevant input." + } + ], + "evaluated_layers": [ + { + "layer": "target identity and metadata", + "status": "clear", + "evidence_ids": [ + "evidence-target-metadata" + ] + }, + { + "layer": "search intent and result patterns", + "status": "unknown" + }, + { + "layer": "content intervention", + "status": "unknown" + } + ], + "issues": [] +}