Run SDK evolution across all vendor packages

docs/sdk-evolution-agent.md

@@ -4,6 +4,10 @@ The SDK evolution agent is a local dogfood workflow for keeping
 agent-runtime-kit aligned with Claude Agent SDK, OpenAI Codex SDK, and Google
 Antigravity SDK as those upstream packages evolve.
 
+For the intended architecture, evidence contract, behavior probe strategy,
+changelog strategy, caveats, and alternatives, see
+[`docs/sdk-evolution-agent-design.md`](sdk-evolution-agent-design.md).
+
 Run it from the repository:
 
 ```bash
@@ -32,6 +36,13 @@ directory is created with private permissions before the Codex runtime starts;
 authenticate that Codex home through supported Codex login/API-key/access-token
 flows before using it for real Codex-backed runs.
 
+Codex-backed SDK evolution runs explicitly choose `gpt-5.5` with
+`reasoning_effort=xhigh` for the AI stages that analyze direction, decide the
+update plan, implement allowed changes, and review the result. This model policy
+is applied only to `codex-agent-sdk`; Claude and Antigravity runs keep their
+provider-native model selection because `gpt-5.5` is not a valid model override
+for those adapters.
+
 For Antigravity, local auth can use `GEMINI_API_KEY` / `GOOGLE_API_KEY` or
 Google Application Default Credentials. ADC runs use Vertex AI config; provide a
 project through ADC, `GOOGLE_CLOUD_PROJECT`, or `GCLOUD_PROJECT`, and optionally
@@ -76,10 +87,20 @@ cutoff variables must not hide candidate releases.
 
 ## Candidate API Inspection
 
-By default, the command snapshots SDK APIs importable in the current
-environment. Use `--inspect-candidates` to install latest candidate SDK versions
-in temporary isolated virtualenvs for API snapshots and diffs. This avoids
-mutating the project lockfile or working environment.
+The command snapshots SDK APIs importable in the current environment. When a
+refresh preview is available, package update candidates come from the resolver's
+`uv lock --dry-run -P ...` output, not only from PyPI's `latest` metadata. For
+each resolver update candidate, the agent installs the target version in a
+temporary isolated virtualenv and writes an API snapshot plus `api_diffs.json`
+entry. This avoids false downgrade diffs for packages whose locked prerelease is
+newer than PyPI's stable latest field. Candidate inspection is always enabled
+for update candidates; `--inspect-candidates` remains accepted only for CLI
+compatibility.
+
+If `uv lock --dry-run -P ...` reports an SDK update but the run cannot produce a
+candidate-version API diff for that package, implementation is blocked and the
+architecture decision is marked `manual_design_required`. An empty added /
+removed / changed diff is valid; a missing diff object is not.
 
 ## Implementation Gates
 

examples/sdk_evolution_agent/cli.py

@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import argparse
+import re
+from dataclasses import replace
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
@@ -78,7 +80,11 @@ def parse_args(argv: list[str] | None = None) -> RunOptions:
     parser.add_argument(
         "--inspect-candidates",
         action="store_true",
-        help="Inspect latest candidate SDK versions in temporary virtualenvs.",
+        default=True,
+        help=(
+            "Inspect latest candidate SDK versions in temporary virtualenvs. "
+            "Always enabled for update candidates; accepted for compatibility."
+        ),
     )
     parser.add_argument("--create-branch", action="store_true", help="Create a local branch first.")
     parser.add_argument("--branch-name", help="Branch name for optional branch creation.")
@@ -114,6 +120,7 @@ async def run_agent(
 ) -> Path:
     """Run the full local SDK evolution workflow."""
 
+    options = replace(options, inspect_candidates=True)
     run_id = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ")
     report_root = (options.workspace / options.report_dir / run_id).resolve()
     event_log_path = report_root / "events.jsonl"
@@ -136,7 +143,7 @@ async def run_agent(
         pypi_client=pypi_client,
         command_runner=command_runner,
     )
-    snapshots = _collect_snapshots(evidence, inspect_candidates=options.inspect_candidates)
+    snapshots = _collect_snapshots(evidence)
     api_diffs = [to_jsonable(diff) for diff in diff_snapshot_groups(snapshots)]
     direction, architecture, review = await run_analysis_pipeline(
         selected_runtime,
@@ -222,15 +229,36 @@ async def run_agent(
     return report_path
 
 
-def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool) -> list[Any]:
+def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = True) -> list[Any]:
+    del inspect_candidates  # Candidate inspection is mandatory for update candidates.
     snapshots = []
+    update_versions = _refresh_update_versions(evidence)
+    refresh_preview_seen = evidence.get("refresh_preview") is not None
     for package in evidence.get("packages", []):
         if not isinstance(package, dict):
             continue
         name = str(package.get("name"))
         snapshots.append(snapshot_current_api(name, version=package.get("installed_version")))
-        latest = package.get("latest_version")
-        installed = package.get("installed_version") or package.get("locked_version")
-        if inspect_candidates and latest and latest != installed:
-            snapshots.append(snapshot_candidate_in_venv(name, str(latest)))
+        candidate = update_versions.get(name)
+        if candidate is None and not refresh_preview_seen:
+            latest = package.get("latest_version")
+            baseline = package.get("locked_version") or package.get("installed_version")
+            if latest and latest != baseline:
+                candidate = str(latest)
+        if candidate:
+            snapshots.append(snapshot_candidate_in_venv(name, candidate))
     return snapshots
+
+
+def _refresh_update_versions(evidence: dict[str, Any]) -> dict[str, str]:
+    preview = evidence.get("refresh_preview")
+    if not isinstance(preview, dict):
+        return {}
+    text = f"{preview.get('stdout') or ''}\n{preview.get('stderr') or ''}"
+    return {
+        package: version
+        for package, version in re.findall(
+            r"Update\s+([A-Za-z0-9_.-]+)\s+v\S+\s+->\s+v(\S+)",
+            text,
+        )
+    }

examples/sdk_evolution_agent/models.py

@@ -107,7 +107,7 @@ class RunOptions:
     report_dir: Path = Path("reports/sdk-evolution")
     implementation_enabled: bool = False
     refresh_preview: bool = False
-    inspect_candidates: bool = False
+    inspect_candidates: bool = True
     create_branch: bool = False
     branch_name: str | None = None
     draft_pr: bool = False

examples/sdk_evolution_agent/stages.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import re
 from collections.abc import Mapping, Sequence
 from pathlib import Path
 from typing import Any
@@ -44,6 +45,8 @@ class StageExecutionError(RuntimeError):
 
 
 SDK_EVOLUTION_CODEX_HOME = Path("~/.codex_agent_runtime_sdk").expanduser()
+SDK_EVOLUTION_CODEX_MODEL = "gpt-5.5"
+SDK_EVOLUTION_CODEX_REASONING_EFFORT = "xhigh"
 
 
 class FixtureEvolutionRuntime:
@@ -105,6 +108,7 @@ def _codex_evolution_runtime(**kwargs: Any) -> CodexAgentRuntime:
     SDK_EVOLUTION_CODEX_HOME.chmod(0o700)
     env = dict(kwargs.pop("env", {}) or {})
     env.setdefault("CODEX_HOME", str(SDK_EVOLUTION_CODEX_HOME))
+    kwargs.setdefault("default_model", SDK_EVOLUTION_CODEX_MODEL)
     return CodexAgentRuntime(env=env, **kwargs)
 
 
@@ -137,12 +141,12 @@ async def run_stage(
     permissions = _stage_permissions(runtime, write_enabled=write_enabled)
     task = AgentTask(
         goal=json.dumps(payload, sort_keys=True, default=str),
-        system=_stage_system_prompt(stage),
+        system=_stage_system_prompt(stage, schema),
         working_directory=context.workspace,
         permissions=permissions,
         event_sink=context.event_sink,
         output_schema=schema,
-        metadata={"stage": stage, "run_id": context.run_id},
+        metadata=_stage_metadata(runtime, stage=stage, context=context),
     )
     try:
         result = await runtime.run(task)
@@ -177,6 +181,7 @@ async def run_analysis_pipeline(
         schema=DIRECTION_ANALYSIS_SCHEMA,
         context=context,
     )
+    direction = _compact_stage_output(direction)
     architecture = await run_stage(
         runtime,
         stage="architecture-decision",
@@ -189,6 +194,9 @@ async def run_analysis_pipeline(
         context=context,
     )
     architecture = with_recursive_impact(architecture, api_diffs)
+    architecture = with_candidate_api_diff_guard(architecture, evidence, api_diffs)
+    architecture = with_manual_design_gate(architecture)
+    architecture = _compact_stage_output(architecture)
     review = await run_stage(
         runtime,
         stage="review",
@@ -313,13 +321,108 @@ def with_recursive_impact(
     return result
 
 
-def _stage_system_prompt(stage: str) -> str:
+def with_candidate_api_diff_guard(
+    architecture: Mapping[str, Any],
+    evidence: Mapping[str, Any],
+    api_diffs: Sequence[Mapping[str, Any] | ApiDiff],
+) -> dict[str, Any]:
+    """Block SDK update implementation when candidate API evidence is missing."""
+
+    update_packages = _refresh_update_packages(evidence)
+    if not update_packages:
+        return dict(architecture)
+    diff_packages = {
+        diff.package if isinstance(diff, ApiDiff) else str(diff.get("package") or "")
+        for diff in api_diffs
+    }
+    missing = tuple(sorted(package for package in update_packages if package not in diff_packages))
+    if not missing:
+        return dict(architecture)
+
+    result = dict(architecture)
+    result["safe_to_implement"] = False
+    result["manual_design_required"] = True
+    findings = list(result.get("findings") or [])
+    findings.append(
+        {
+            "classification": "manual-design-required",
+            "summary": (
+                "SDK update candidates require candidate-version API snapshot diffs "
+                "before implementation can be considered safe."
+            ),
+            "evidence": [f"missing api_diffs for {package}" for package in missing],
+        }
+    )
+    result["findings"] = findings
+    uncertainty = list(result.get("uncertainty") or [])
+    uncertainty.append(
+        "Candidate API diffs were not available for update candidate(s): "
+        + ", ".join(missing)
+    )
+    result["uncertainty"] = uncertainty
+    plan = list(result.get("self_adaptation_plan") or [])
+    plan.append(
+        "Rerun with candidate API inspection and review the generated api_diffs before "
+        "changing adapters or dependency locks."
+    )
+    result["self_adaptation_plan"] = plan
+    return result
+
+
+def with_manual_design_gate(architecture: Mapping[str, Any]) -> dict[str, Any]:
+    """Make manual design decisions block implementation unambiguously."""
+
+    result = dict(architecture)
+    if result.get("manual_design_required"):
+        result["safe_to_implement"] = False
+    return result
+
+
+def _refresh_update_packages(evidence: Mapping[str, Any]) -> tuple[str, ...]:
+    preview = evidence.get("refresh_preview")
+    if not isinstance(preview, Mapping):
+        return ()
+    text = f"{preview.get('stdout') or ''}\n{preview.get('stderr') or ''}"
+    return tuple(
+        sorted(set(re.findall(r"Update\s+([A-Za-z0-9_.-]+)\s+v\S+\s+->\s+v\S+", text)))
+    )
+
+
+def _compact_stage_output(value: Mapping[str, Any]) -> dict[str, Any]:
+    return {key: _compact_stage_value(item) for key, item in value.items()}
+
+
+def _compact_stage_value(value: Any, *, string_limit: int = 800, list_limit: int = 8) -> Any:
+    if isinstance(value, str):
+        if len(value) <= string_limit:
+            return value
+        return value[: string_limit - 16].rstrip() + " [truncated]"
+    if isinstance(value, list):
+        return [
+            _compact_stage_value(item, string_limit=string_limit, list_limit=list_limit)
+            for item in value[:list_limit]
+        ]
+    if isinstance(value, dict):
+        return {
+            key: _compact_stage_value(item, string_limit=string_limit, list_limit=list_limit)
+            for key, item in value.items()
+        }
+    return value
+
+
+def _stage_system_prompt(stage: str, schema: JsonSchema) -> str:
     return (
         "You are running inside the local SDK evolution agent. "
         "Use only the provided evidence. Preserve vendor-specific behavior, "
         "state uncertainty explicitly, and never claim implementation occurred "
         "unless it is reflected in the provided artifacts. "
-        f"Current stage: {stage}."
+        "Return only one JSON object that validates against the provided schema. "
+        "Do not include Markdown, code fences, file links, or prose outside JSON. "
+        "Do not call shell, command, file, or workspace tools; the deterministic "
+        "evidence bundle already contains the inspected data. "
+        "Keep each array to at most five high-signal items and each string concise. "
+        f"Current stage: {stage}. "
+        f"Output schema: {json.dumps(schema, sort_keys=True)}"
     )
 
 
@@ -339,6 +442,19 @@ def _stage_permissions(runtime: AgentRuntime, *, write_enabled: bool) -> Permiss
     )
 
 
+def _stage_metadata(
+    runtime: AgentRuntime,
+    *,
+    stage: str,
+    context: RunContext,
+) -> dict[str, Any]:
+    metadata: dict[str, Any] = {"stage": stage, "run_id": context.run_id}
+    if runtime.kind is AgentRuntimeKind.CODEX_AGENT_SDK:
+        metadata["model"] = SDK_EVOLUTION_CODEX_MODEL
+        metadata["reasoning_effort"] = SDK_EVOLUTION_CODEX_REASONING_EFFORT
+    return metadata
+
+
 def _fixture_payload(stage: str, task: AgentTask) -> dict[str, Any]:
     try:
         source = json.loads(task.goal)

reports/sdk-evolution-all-packages/20260622T091555Z/api_diffs.json

@@ -0,0 +1,51 @@
+[
+  {
+    "added": [
+      "TERMINAL_TASK_STATUSES",
+      "TaskUpdatedMessage",
+      "TaskUpdatedStatus"
+    ],
+    "changed": [],
+    "from_version": "0.2.96",
+    "package": "claude-agent-sdk",
+    "removed": [],
+    "to_version": "0.2.106"
+  },
+  {
+    "added": [
+      "Audio",
+      "BuiltinTools",
+      "Content",
+      "CustomSystemInstructions",
+      "Document",
+      "GeminiAPIEndpoint",
+      "GeminiModelOptions",
+      "Image",
+      "ModelEndpoint",
+      "ModelTarget",
+      "ModelType",
+      "SystemInstructionSection",
+      "SystemInstructions",
+      "TemplatedSystemInstructions",
+      "VertexEndpoint",
+      "Video",
+      "from_file",
+      "models"
+    ],
+    "changed": [
+      "CapabilitiesConfig",
+      "LocalAgentConfig",
+      "ToolContext"
+    ],
+    "from_version": "0.1.2",
+    "package": "google-antigravity",
+    "removed": [
+      "GeminiConfig",
+      "GenerationConfig",
+      "ModelConfig",
+      "ModelEntry",
+      "mcp"
+    ],
+    "to_version": "0.1.4"
+  }
+]