diff --git a/.github/workflows/inspect-r-api-update.yml b/.github/workflows/inspect-r-api-update.yml index 4aa6f0f3..b0e5da4c 100644 --- a/.github/workflows/inspect-r-api-update.yml +++ b/.github/workflows/inspect-r-api-update.yml @@ -131,7 +131,7 @@ jobs: if: steps.payload.outputs.fresh_cache != 'true' shell: bash env: - DISPATCH_TOKEN: ${{ secrets.PARITY_DISPATCH_TOKEN }} + DISPATCH_TOKEN: ${{ secrets.OVVO_SYNC_TOKEN }} run: | set -euo pipefail # The cache-based gates below only prove parity against the committed @@ -143,7 +143,7 @@ jobs: exit 0 fi if [ -z "${DISPATCH_TOKEN:-}" ]; then - echo "PARITY_DISPATCH_TOKEN not set; skipping auto-chain to parity-autofix." + echo "OVVO_SYNC_TOKEN not set; skipping auto-chain to parity-autofix." echo "Run parity-autofix manually with r_commit=${{ steps.payload.outputs.r_commit }} r_version=${{ steps.payload.outputs.r_version }}." exit 0 fi @@ -177,6 +177,7 @@ jobs: - name: Open R API inspection PR uses: peter-evans/create-pull-request@v6 with: + token: ${{ secrets.OVVO_SYNC_TOKEN || github.token }} branch: inspect-r-api-${{ steps.payload.outputs.r_commit }} title: Inspect R NNS API update ${{ steps.payload.outputs.r_commit }} body: | diff --git a/.github/workflows/parity-autofix.yml b/.github/workflows/parity-autofix.yml index 67a77cee..67bbd12a 100644 --- a/.github/workflows/parity-autofix.yml +++ b/.github/workflows/parity-autofix.yml @@ -1,19 +1,23 @@ name: Parity autofix -# Automated R-behavior parity fix loop for NNS-python. +# Live-R parity verification for NNS-python. # # This is the "fix if required" half of the fidelity chain. The detection half -# lives in inspect-r-api-update.yml. When live-R parity for a changed R API -# diverges, this workflow re-derives the divergence against live R and hands a -# structured report to an agent (anthropics/claude-code-action) that may draft a -# fix to src/nns/** ONLY, then opens a SEPARATE parity-correction PR. +# lives in inspect-r-api-update.yml. When a changed R API maps to parity tests, +# this workflow installs live R at the recorded R commit and re-runs the mapped +# parity to detect whether public Python behavior diverged. # -# Human-merge policy: this workflow never merges. A human reviews every fix PR. +# NOTE: the automated fixing agent (anthropics/claude-code-action) is deferred +# for a later date. For now this workflow verifies against live R and, on a +# divergence (or when live R cannot be verified), opens a human parity-review PR +# carrying the reports. A maintainer applies the reviewed fix. It never merges. +# +# All GitHub operations use the OVVO_SYNC_TOKEN secret (Contents + Pull requests +# write). No ANTHROPIC_API_KEY is required while the agent is deferred. # # Trigger model: # * repository_dispatch [nns-parity-divergence] - emitted by -# inspect-r-api-update.yml when live parity diverges (needs a PAT, see -# docs/parity_autofix.md), or by upstream automation. +# inspect-r-api-update.yml (uses OVVO_SYNC_TOKEN), or by upstream automation. # * workflow_dispatch - manual fallback. on: @@ -31,16 +35,10 @@ on: required: false default: "unknown" type: string - model: - description: Claude model id for the autofix agent - required: false - default: claude-sonnet-4-6 - type: string permissions: contents: write pull-requests: write - issues: write jobs: parity-autofix: @@ -60,13 +58,11 @@ jobs: echo "r_commit=${{ github.event.client_payload.r_commit }}" >> "$GITHUB_OUTPUT" echo "r_version=${{ github.event.client_payload.r_version }}" >> "$GITHUB_OUTPUT" echo "r_src_tree_hash=${{ github.event.client_payload.r_src_tree_hash }}" >> "$GITHUB_OUTPUT" - echo "model=claude-sonnet-4-6" >> "$GITHUB_OUTPUT" echo '${{ toJson(github.event.client_payload.changed_files) }}' > changed_files.json else echo "r_commit=${{ inputs.r_commit }}" >> "$GITHUB_OUTPUT" echo "r_version=${{ inputs.r_version }}" >> "$GITHUB_OUTPUT" echo "r_src_tree_hash=${{ inputs.r_src_tree_hash }}" >> "$GITHUB_OUTPUT" - echo "model=${{ inputs.model }}" >> "$GITHUB_OUTPUT" echo '[]' > changed_files.json fi @@ -130,7 +126,7 @@ jobs: --skip-install \ --out sync/last_live_r_parity_report.md - - name: Decide whether a fix is required + - name: Classify live parity outcome id: gate shell: bash run: | @@ -139,75 +135,43 @@ jobs: echo "live_parity_outcome=${outcome}" >> "$GITHUB_OUTPUT" echo "r_install_outcome=${r_ok}" >> "$GITHUB_OUTPUT" if [ "${outcome}" = "success" ]; then - echo "needs_fix=false" >> "$GITHUB_OUTPUT" - echo "Live R parity passed; no fix required." + echo "open_pr=false" >> "$GITHUB_OUTPUT" + echo "Live R parity passed; no parity-review PR needed." elif [ "${r_ok}" != "success" ]; then - # Cannot verify against live R -> do NOT let the agent guess a fix. - echo "needs_fix=false" >> "$GITHUB_OUTPUT" - echo "escalate=true" >> "$GITHUB_OUTPUT" - echo "Live R install failed; escalating instead of auto-fixing." + echo "open_pr=true" >> "$GITHUB_OUTPUT" + echo "cause=Live R could not be installed in CI, so parity could not be verified against live R." >> "$GITHUB_OUTPUT" + echo "Live R install failed; opening a parity-review PR." else - echo "needs_fix=true" >> "$GITHUB_OUTPUT" - echo "Live R parity diverged; dispatching autofix agent." + echo "open_pr=true" >> "$GITHUB_OUTPUT" + echo "cause=Live R parity diverged: public Python behavior differs from live R at the recorded commit." >> "$GITHUB_OUTPUT" + echo "Live R parity diverged; opening a parity-review PR." fi - - name: Run parity-fix agent - if: steps.gate.outputs.needs_fix == 'true' - uses: anthropics/claude-code-action@v1 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - # Use the GitHub App token if configured so the resulting PR triggers - # native-backend-ci (PRs opened with the default GITHUB_TOKEN do not). - github_token: ${{ secrets.PARITY_APP_TOKEN || github.token }} - prompt: | - You are fixing an R-behavior parity divergence in OVVO-Financial/NNS-python. - - Context: - - R behavioral truth: OVVO-Financial/NNS at commit ${{ steps.payload.outputs.r_commit }} (version ${{ steps.payload.outputs.r_version }}), checked out at upstream/NNS and installed as live R. - - The divergence report is at sync/last_live_r_parity_report.md. The parity plan is at sync/last_r_api_plan.json and the R->Python map is at sync/r_api_map.json. - - Hard rules (do not violate): - 1. Edit files under src/nns/** ONLY. NEVER edit extern/NNS-core/**, tools/NNS/**, tests/_r_cache.json, or any test/cache file to make a check pass. - 2. Classify the root cause first and act accordingly: - - Python port bug (a wrapper/default/return-shape/algorithm in src/nns/** drifted from R): fix it in src/nns/**. - - R changed behavior (the divergence is because upstream R itself changed): DO NOT edit code to chase a cache value. Stop and open the PR/issue as an escalation describing the change; cache regeneration is a separate, reviewed step. - - Native kernel change (the difference originates in C++ src/** / NNS-core): DO NOT edit. Native code enters Python only through accepted NNS-core commits. Stop and escalate. - 3. Verify any fix against LIVE R, not the committed cache. The fix must keep ALL of these green: - - python -m pytest -q -n 0 tests/invariants - - the mapped parity tests in the plan (run with -n 0), against live R - - ruff check . ; mypy ; python -m build - Do not weaken tests or tolerances. - 4. Keep the change minimal and in the style of the surrounding code. - - Deliverable: - - Create a NEW branch named parity-fix/${{ steps.payload.outputs.r_commit }} and open a SEPARATE parity-correction pull request into the default branch. Do NOT merge it. - - The PR body MUST contain, for each divergence: the function, the arguments, the R output, the Python output, the first divergent intermediate, the affected source files, and the proposed fix (or, for escalations, why no code fix was made and what must happen upstream / in NNS-core). - - Title: "Parity fix: R NNS ${{ steps.payload.outputs.r_commit }}". - claude_args: | - --model ${{ steps.payload.outputs.model }} - --max-turns 40 - --allowedTools Edit,Read,Write,Glob,Grep,Bash(python:*),Bash(python -m pytest:*),Bash(ruff:*),Bash(mypy:*),Bash(pip:*),Bash(Rscript:*),Bash(git:*),Bash(gh:*) - --disallowedTools Bash(rm:*) - - - name: Escalate when live R could not verify - if: steps.gate.outputs.escalate == 'true' + - name: Open parity-review PR for human fix + if: steps.gate.outputs.open_pr == 'true' uses: peter-evans/create-pull-request@v6 with: - branch: parity-escalation-${{ steps.payload.outputs.r_commit }} - title: Parity escalation (live R unavailable) ${{ steps.payload.outputs.r_commit }} + token: ${{ secrets.OVVO_SYNC_TOKEN || github.token }} + branch: parity-review-${{ steps.payload.outputs.r_commit }} + title: Parity review needed - R NNS ${{ steps.payload.outputs.r_commit }} body: | - Live R NNS could not be installed in CI for R commit - `${{ steps.payload.outputs.r_commit }}` (version - `${{ steps.payload.outputs.r_version }}`), so a parity fix was NOT - attempted automatically — fidelity must be verified against live R, - never guessed. + Live-R parity for the changed R API needs human review. + + Cause: ${{ steps.gate.outputs.cause }} + + R commit: `${{ steps.payload.outputs.r_commit }}` + R version: `${{ steps.payload.outputs.r_version }}` - Reports: + Reports committed on this branch: - `sync/last_r_api_inspection.md` - `sync/last_r_api_plan.json` - `sync/last_live_r_parity_report.md` - Action required: run `scripts/install_local_r_nns.py` in an - environment with R available, reproduce the divergence, and apply a - reviewed parity fix. Native differences must route through NNS-core. - commit-message: Parity escalation ${{ steps.payload.outputs.r_commit }} + Current method (automated fixing agent deferred): a maintainer + reproduces the divergence against live R and applies a reviewed + parity fix to `src/nns/**`. Record the function, arguments, R output, + Python output, first divergent intermediate, and affected files. + Native differences route through NNS-core. A DESCRIPTION/version or + behavior change is a separate, reviewed cache regeneration. Do not + edit `tests/_r_cache.json` to force a pass. + commit-message: Parity review reports for R NNS ${{ steps.payload.outputs.r_commit }} diff --git a/.github/workflows/sync-from-nns-core.yml b/.github/workflows/sync-from-nns-core.yml index a9d68308..d0daa2ee 100644 --- a/.github/workflows/sync-from-nns-core.yml +++ b/.github/workflows/sync-from-nns-core.yml @@ -117,6 +117,7 @@ jobs: - name: Open NNS-core sync PR uses: peter-evans/create-pull-request@v6 with: + token: ${{ secrets.OVVO_SYNC_TOKEN || github.token }} branch: sync-nns-core-${{ steps.payload.outputs.core_commit }} title: Sync vendored NNS-core from ${{ steps.payload.outputs.core_commit }} body: | diff --git a/docs/parity_autofix.md b/docs/parity_autofix.md index 89d07fa1..22d199c0 100644 --- a/docs/parity_autofix.md +++ b/docs/parity_autofix.md @@ -1,90 +1,63 @@ -# Automated parity check and fix +# Automated parity check (with deferred autofix agent) -`NNS-python` automates both halves of R-behavior fidelity: +`NNS-python` automates R-behavior fidelity detection. The automated **fixing +agent is deferred for a later date**; for now the chain verifies against live R +and hands divergences to a maintainer via a parity-review PR (the current +method). 1. **Check** — when upstream `OVVO-Financial/NNS` changes an R API, the `inspect-r-api-update` workflow plans which Python modules / parity tests are affected and records a report. -2. **Fix if required** — the `parity-autofix` workflow verifies the affected - public behavior against **live R** at the recorded R commit and, if behavior - drifted, hands a structured divergence report to an agent - (`anthropics/claude-code-action`) that drafts a fix and opens a **separate, - human-reviewed** parity-correction PR. +2. **Verify** — the `parity-autofix` workflow installs **live R** at the + recorded R commit and re-runs the mapped parity. If behavior diverged (or live + R could not be verified), it opens a **parity-review PR** carrying the reports + for a maintainer to fix. ```text NNS R API change -> inspect-r-api-update.yml (plan + cache gates + inspection PR) - -> dispatch nns-parity-divergence - -> parity-autofix.yml (live-R verify -> fix or escalate -> PR) - -> human review + merge + -> dispatch nns-parity-divergence (via OVVO_SYNC_TOKEN) + -> parity-autofix.yml (live-R verify -> parity-review PR) + -> maintainer applies reviewed fix -> merge ``` -## What the agent may and may not do - -Hard rules enforced in the agent prompt and the workflow gate: +## Fix rules (applied by the maintainer; later by the agent) * Edit **`src/nns/**` only**. Never edit `extern/NNS-core/**`, `tools/NNS/**`, or `tests/_r_cache.json` to make a check pass. * Classify the root cause and act accordingly: * **Python port bug** → fix in `src/nns/**`. - * **R changed behavior** → do not chase a cache value; escalate (cache - regeneration is a separate, reviewed step). + * **R changed behavior** → do not chase a cache value; cache regeneration is a + separate, reviewed step. * **Native kernel change** → do not edit; native code enters Python only - through accepted `NNS-core` commits. Escalate. -* Verify against **live R**, not the committed cache. If live R cannot be - installed in CI, the workflow opens an **escalation PR** instead of letting - the agent guess a fix. -* **Human-merge only.** The workflow never merges; every fix PR is reviewed. - -## Required setup + through accepted `NNS-core` commits. +* Verify against **live R**, not the committed cache. +* **Human-merge only.** The workflow never merges. -### 1. Anthropic credentials (required) +## Tokens -The autofix agent needs an API key exposed as the `ANTHROPIC_API_KEY` repo -secret. +Everything runs on a single GitHub secret while the agent is deferred: -Easiest path — from Claude Code, run: +| Secret | Kind | Used for | Required? | +| --- | --- | --- | --- | +| `OVVO_SYNC_TOKEN` | GitHub fine-grained PAT — **Contents: R/W**, **Pull requests: R/W** on `NNS-python` | emit the `inspect -> autofix` `repository_dispatch` and open all sync / inspection / parity-review PRs (so they trigger `native-backend-ci`) | yes (workflows fall back to `github.token`, but then PRs won't trigger CI and the auto-chain is skipped) | -```text -/install-github-app -``` +Add it at repo **Settings → Secrets and variables → Actions → New repository +secret**. Without it, `inspect-r-api-update` prints the manual trigger command +and you run `parity-autofix` yourself via **workflow_dispatch** (inputs: +`r_commit`, `r_version`). -This installs the official Claude GitHub App, adds the `ANTHROPIC_API_KEY` -secret, and (optionally) scaffolds a workflow. You must be a repo admin. +## Enabling the autofix agent later -Manual path: +When you want the agent to draft fixes automatically, re-add the +`anthropics/claude-code-action` step to `parity-autofix.yml` (gated on a live-R +divergence) with the fix rules above as its prompt, and add: -1. Repo **Settings → Secrets and variables → Actions → New repository secret**. -2. Name `ANTHROPIC_API_KEY`, value = your key from `console.anthropic.com`. +* **`ANTHROPIC_API_KEY`** — an Anthropic key (`sk-ant-…`) from + `console.anthropic.com` (or run `/install-github-app`). This authenticates the + agent to Claude and is **not** substitutable by a GitHub token. -Bedrock / Vertex are supported via the action's `use_bedrock` / `use_vertex` -inputs with OIDC (`id-token: write`) instead of a static key; see the +The agent would open a `src/nns/**`-only parity-correction PR, still +human-merged. Bedrock / Vertex are alternatives via the action's `use_bedrock` / +`use_vertex` inputs with OIDC; see the [cloud providers docs](https://github.com/anthropics/claude-code-action/blob/main/docs/cloud-providers.md). - -### 2. Fix-PR token so CI runs on the fix (recommended) - -A PR opened with the default `GITHUB_TOKEN` does **not** trigger other workflows -(GitHub's recursion guard), so `native-backend-ci` would not run on the fix PR. -To get CI on fix PRs, let the agent open the PR with a token that does trigger -workflows: - -* Installing the **GitHub App** (above) already provides this, or -* add a fine-grained PAT / GitHub App token as the `PARITY_APP_TOKEN` secret; - the workflow passes it to the action as `github_token`. If unset, it falls - back to `github.token`. - -### 3. Auto-chain token (optional) - -`inspect-r-api-update` chains to `parity-autofix` via a `repository_dispatch`, -which the default `GITHUB_TOKEN` cannot emit. To enable the automatic chain, add -a PAT with `contents: write` (or `repo`) scope as the `PARITY_DISPATCH_TOKEN` -secret. Without it, `inspect-r-api-update` prints the manual trigger command and -you run `parity-autofix` yourself via **workflow_dispatch** (inputs: `r_commit`, -`r_version`). - -## Model - -The agent model defaults to `claude-sonnet-4-6` and is overridable via the -`workflow_dispatch` `model` input. Use a more capable model (e.g. -`claude-opus-4-8`) for harder divergences. Every fix is still verified by the -full gate set against live R and reviewed by a human before merge.