diff --git a/agentops-cli/src/commands/dashboard.js b/agentops-cli/src/commands/dashboard.js index daa7837..2658683 100644 --- a/agentops-cli/src/commands/dashboard.js +++ b/agentops-cli/src/commands/dashboard.js @@ -557,6 +557,7 @@ function validateDashboardUx() { policy_review: true, semantic_review: true, promotion_approvals: true, + robust_eval_center: true, alert_handoff_review: true, schema_version_coverage: true, exporter_failure_review: true, diff --git a/agentops-cli/src/commands/product.js b/agentops-cli/src/commands/product.js index de08d27..805dd57 100644 --- a/agentops-cli/src/commands/product.js +++ b/agentops-cli/src/commands/product.js @@ -228,6 +228,40 @@ function productAudit(options = {}) { ux.errors )); + checks.push(check( + 'robust-eval-center-contract', + ux.ok + && ux.contracts?.robust_eval_center === true + && fileIncludes('agentops-cli/src/legacy.js', [ + 'hiddenCheckPacks', + 'permissionProfiles', + 'macos-network-blocked', + 'commandFileSeal', + 'semanticChecks', + 'llm-judge', + 'artifactDiff', + 'promotionApproval' + ]) + && fileIncludes('grafana/dashboards/v2/08-evals-quality.json', [ + 'BenchmarkArtifactContentDiffs', + 'BenchmarkApproval' + ]) + && fileIncludes('docs/agentops-architecture-product-audit.md', [ + 'Robust Eval Center', + 'Hidden check packs', + 'Rubric and semantic scoring', + 'Artifact diffing', + 'Promotion policy', + 'Cross-platform managed OS-level' + ]), + [ + 'agentops-cli/src/legacy.js', + 'grafana/dashboards/v2/08-evals-quality.json', + 'docs/agentops-architecture-product-audit.md' + ], + ux.errors + )); + checks.push(check( 'azure-ingest-privacy-plan', fileIncludes('agentops-cli/src/lib/azure/v2-ingest-plan.js', ['--allow-content', 'AgentOpsContent_CL', 'schema_versioning', 'schema_migration_policy']) diff --git a/agentops-cli/test/index.test.js b/agentops-cli/test/index.test.js index af604f6..5b32b8e 100644 --- a/agentops-cli/test/index.test.js +++ b/agentops-cli/test/index.test.js @@ -4332,6 +4332,7 @@ test('product audit proves the local AgentOps control-room contract', () => { 'grafana-v2-pack', 'kql-library', 'run-centric-ui-contract', + 'robust-eval-center-contract', 'content-transcript-opt-in', 'first-run-loop', 'ask-agentops-response-flow', diff --git a/docs/agentops-architecture-product-audit.md b/docs/agentops-architecture-product-audit.md index e590684..a4022cd 100644 --- a/docs/agentops-architecture-product-audit.md +++ b/docs/agentops-architecture-product-audit.md @@ -1239,15 +1239,21 @@ Success criterion: Users can compare baseline vs candidate agents and detect safety, quality, cost, and anti-cheat regressions before promotion. ``` +Implemented: + +- Hidden check packs are supported as masked, sealed check packs, with signed fixture pack manifests, trust roots, revoked signing key IDs, and rotation windows. +- Rubric and semantic scoring supports deterministic file-content, regex, file-rubric, and command-backed `llm-judge` checks with reusable hosted judge provider command templates. +- Network and permission controls include per-task permission profiles, read-only copied fixtures, allowed-tool risk policies, and opt-in macOS `sandbox-exec` network blocking that fails closed when requested on unsupported hosts. +- Local harness tamper resistance includes copied fixture workspaces, read-only benchmark profiles, sealed fixture packs, and `commandFileSeal` checks for test scripts and command files. +- Artifact diffing is available in CLI reports and the Evals & Quality dashboard, including per-file review and capped content diff previews. +- Scorecard UI is present in the Evals & Quality dashboard through eval scorecards, regression follow-up, before/after comparisons, hidden check review, policy review, semantic review, artifact review, and promotion approvals. +- Promotion policy can require gate results, approval evidence, approver identity/counts, approved external review metadata, and optional external review verification for GitHub PRs, Azure DevOps PRs, and Jira issues. +- `agentops product audit` now includes `robust-eval-center-contract` so this eval-center shape remains a local release gate. + Required work: -- Hidden tests. -- Rubric/semantic scoring beyond deterministic file-content adapters. -- Network and permission profiles. -- Managed immutable harness isolation beyond command-file seals. -- Artifact diffing. -- Scorecard UI. -- Promotion policy. +- Cross-platform managed OS-level immutable harness and network egress isolation beyond the current copied fixture, read-only profile, command-file seals, signed fixture packs, and opt-in macOS network sandbox. +- Hosted judge service deployment for `llm-judge` beyond the current non-mutating Azure Container Apps provisioning plan. ### 5. Trustworthy Data Quality