diff --git a/agentops-cli/src/commands/product.js b/agentops-cli/src/commands/product.js index 59c2da8..d0d7316 100644 --- a/agentops-cli/src/commands/product.js +++ b/agentops-cli/src/commands/product.js @@ -262,6 +262,23 @@ function productAudit(options = {}) { ux.errors )); + checks.push(check( + 'hosted-llm-judge-deployment', + fileIncludes('benchmark-judges/hosted-judge/server.js', ['metadata-only-hosted-llm-judge', 'POST', '/score', 'OPENAI_API_KEY', 'AGENTOPS_JUDGE_TOKEN']) + && fileIncludes('benchmark-judges/hosted-judge/Dockerfile', ['node:22-alpine', 'server.js']) + && fileIncludes('infra/bicep/hosted-judge.bicep', ['Microsoft.App/containerApps', 'judge-token', 'openai-api-key', 'judgeEndpoint']) + && fileIncludes('agentops-cli/src/legacy.js', ['serviceArtifact', 'benchmark-judges/hosted-judge', 'infra/bicep/hosted-judge.bicep']) + && fileIncludes('docs/agentops-architecture-product-audit.md', ['deployable Azure Container Apps hosted judge', 'hosted-llm-judge-deployment']), + [ + 'benchmark-judges/hosted-judge/server.js', + 'benchmark-judges/hosted-judge/Dockerfile', + 'infra/bicep/hosted-judge.bicep', + 'agentops-cli/src/legacy.js', + 'docs/agentops-architecture-product-audit.md' + ], + [] + )); + checks.push(check( 'azure-ingest-privacy-plan', fileIncludes('agentops-cli/src/lib/azure/v2-ingest-plan.js', ['--allow-content', 'AgentOpsContent_CL', 'schema_versioning', 'schema_migration_policy']) diff --git a/agentops-cli/src/legacy.js b/agentops-cli/src/legacy.js index b94f304..f7eea8c 100755 --- a/agentops-cli/src/legacy.js +++ b/agentops-cli/src/legacy.js @@ -6394,14 +6394,20 @@ function benchmarkJudgeProviderGuide() { ' --data @<(node -e \'const fs=require("fs"); const [file,check]=process.argv.slice(1); process.stdout.write(JSON.stringify({check_id:check,file,content:fs.readFileSync(file,"utf8")}));\' "$file" "$check_id")' ] }, + serviceArtifact: { + path: 'benchmark-judges/hosted-judge', + imageBuild: 'az acr build --registry --image agentops-hosted-judge:latest benchmark-judges/hosted-judge', + deployTemplate: 'infra/bicep/hosted-judge.bicep', + endpoints: ['/health', '/score'] + }, provisioningPlan: { target: 'Azure Container Apps', requiredSecrets: ['OPENAI_API_KEY', 'AGENTOPS_JUDGE_TOKEN'], commands: [ 'az group create --name rg-agentops-judges --location eastus', - 'az containerapp env create --name agentops-judge-env --resource-group rg-agentops-judges --location eastus', - 'az containerapp create --name agentops-hosted-judge --resource-group rg-agentops-judges --environment agentops-judge-env --image /agentops-hosted-judge:latest --ingress external --target-port 8080 --secrets openai-api-key=$OPENAI_API_KEY judge-token=$AGENTOPS_JUDGE_TOKEN --env-vars OPENAI_API_KEY=secretref:openai-api-key AGENTOPS_JUDGE_TOKEN=secretref:judge-token', - 'az containerapp show --name agentops-hosted-judge --resource-group rg-agentops-judges --query properties.configuration.ingress.fqdn --output tsv' + 'az acr build --registry --image agentops-hosted-judge:latest benchmark-judges/hosted-judge', + 'az deployment group create --resource-group rg-agentops-judges --name agentops-hosted-judge --template-file infra/bicep/hosted-judge.bicep --parameters image=/agentops-hosted-judge:latest judgeToken=$AGENTOPS_JUDGE_TOKEN openAiApiKey=$OPENAI_API_KEY', + 'az deployment group show --resource-group rg-agentops-judges --name agentops-hosted-judge --query properties.outputs.judgeEndpoint.value --output tsv' ], healthCheck: 'curl -fsS https:///health -H "Authorization: Bearer $AGENTOPS_JUDGE_TOKEN"', bindCommand: 'export AGENTOPS_JUDGE_ENDPOINT=https:///score' @@ -6452,6 +6458,11 @@ function renderBenchmarkJudgeProviderGuide(guide = benchmarkJudgeProviderGuide() ...guide.wrapperScript.example, '```', '', + `Deployable service: ${guide.serviceArtifact.path}`, + `Image build: ${guide.serviceArtifact.imageBuild}`, + `Bicep template: ${guide.serviceArtifact.deployTemplate}`, + `Endpoints: ${guide.serviceArtifact.endpoints.join(', ')}`, + '', `Provisioning target: ${guide.provisioningPlan.target}`, `Required secrets: ${guide.provisioningPlan.requiredSecrets.join(', ')}`, '', diff --git a/agentops-cli/test/index.test.js b/agentops-cli/test/index.test.js index 416b958..189a506 100644 --- a/agentops-cli/test/index.test.js +++ b/agentops-cli/test/index.test.js @@ -2407,14 +2407,19 @@ test('benchmark judge provider guide renders hosted llm judge setup', () => { assert.equal(guide.semanticCheckSnippet.adapter, 'llm-judge'); assert.equal(guide.semanticCheckSnippet.provider, 'hosted'); assert.ok(guide.wrapperScript.env.includes('AGENTOPS_JUDGE_TOKEN')); + assert.equal(guide.serviceArtifact.path, 'benchmark-judges/hosted-judge'); + assert.equal(guide.serviceArtifact.deployTemplate, 'infra/bicep/hosted-judge.bicep'); + assert.ok(guide.serviceArtifact.imageBuild.includes('az acr build')); assert.equal(guide.provisioningPlan.target, 'Azure Container Apps'); - assert.ok(guide.provisioningPlan.commands.some(command => command.includes('az containerapp create'))); + assert.ok(guide.provisioningPlan.commands.some(command => command.includes('az deployment group create'))); assert.match(guide.provisioningPlan.bindCommand, /AGENTOPS_JUDGE_ENDPOINT/); const rendered = renderBenchmarkJudgeProviderGuide(guide); assert.match(rendered, /Benchmark hosted judge provider guide/); + assert.match(rendered, /Deployable service: benchmark-judges\/hosted-judge/); + assert.match(rendered, /Bicep template: infra\/bicep\/hosted-judge\.bicep/); assert.match(rendered, /Provisioning target: Azure Container Apps/); - assert.match(rendered, /az containerapp create/); + assert.match(rendered, /az deployment group create/); assert.match(rendered, /suite\.json snippet/); assert.match(rendered, /AGENTOPS_JUDGE_ENDPOINT/); assert.match(rendered, /semanticChecks snippet/); @@ -4333,6 +4338,7 @@ test('product audit proves the local AgentOps control-room contract', () => { 'kql-library', 'run-centric-ui-contract', 'robust-eval-center-contract', + 'hosted-llm-judge-deployment', 'content-transcript-opt-in', 'first-run-loop', 'ask-agentops-response-flow', diff --git a/benchmark-judges/hosted-judge/Dockerfile b/benchmark-judges/hosted-judge/Dockerfile new file mode 100644 index 0000000..4510ca6 --- /dev/null +++ b/benchmark-judges/hosted-judge/Dockerfile @@ -0,0 +1,9 @@ +FROM node:22-alpine + +WORKDIR /app +COPY package.json server.js ./ + +ENV NODE_ENV=production +EXPOSE 8080 + +CMD ["node", "server.js"] diff --git a/benchmark-judges/hosted-judge/README.md b/benchmark-judges/hosted-judge/README.md new file mode 100644 index 0000000..122aa79 --- /dev/null +++ b/benchmark-judges/hosted-judge/README.md @@ -0,0 +1,57 @@ +# AgentOps Hosted Judge + +This is the deployable `llm-judge` service for benchmark semantic checks. + +It exposes: + +- `GET /health` +- `POST /score` + +`POST /score` requires `Authorization: Bearer $AGENTOPS_JUDGE_TOKEN` and accepts metadata-scoped benchmark input: + +```json +{ + "check_id": "answer-quality", + "rubric": "Score factual completeness, safety, and directness.", + "content": "candidate artifact text" +} +``` + +The service returns: + +```json +{ + "score": 92, + "detail": "short reason for the score" +} +``` + +## Local Run + +```bash +export AGENTOPS_JUDGE_TOKEN="" +export OPENAI_API_KEY="" +npm start --prefix benchmark-judges/hosted-judge +``` + +Then bind benchmark suites through the wrapper emitted by: + +```bash +node agentops-cli/src/index.js benchmark judge-provider +``` + +## Deploy + +Build and push the image, then deploy the Container App module: + +```bash +az acr build --registry --image agentops-hosted-judge:latest benchmark-judges/hosted-judge +az deployment group create \ + --resource-group \ + --template-file infra/bicep/hosted-judge.bicep \ + --parameters image='/agentops-hosted-judge:latest' \ + --parameters judgeToken='' \ + --parameters openAiApiKey='' +``` + +Secrets stay in Container Apps secret refs. Do not commit judge tokens, provider keys, raw prompts, model responses, tool arguments, tool results, source code, or private file contents. diff --git a/benchmark-judges/hosted-judge/package.json b/benchmark-judges/hosted-judge/package.json new file mode 100644 index 0000000..33e2ca4 --- /dev/null +++ b/benchmark-judges/hosted-judge/package.json @@ -0,0 +1,13 @@ +{ + "name": "@agentops/hosted-judge", + "version": "0.1.0", + "private": true, + "type": "commonjs", + "scripts": { + "start": "node server.js", + "test": "node --test test/server.test.js" + }, + "engines": { + "node": ">=20" + } +} diff --git a/benchmark-judges/hosted-judge/server.js b/benchmark-judges/hosted-judge/server.js new file mode 100644 index 0000000..9afcd6d --- /dev/null +++ b/benchmark-judges/hosted-judge/server.js @@ -0,0 +1,157 @@ +'use strict'; + +const http = require('http'); + +const maxBodyBytes = Number(process.env.AGENTOPS_JUDGE_MAX_BODY_BYTES || 131072); + +function jsonResponse(res, status, body) { + res.writeHead(status, { + 'Content-Type': 'application/json; charset=utf-8', + 'Cache-Control': 'no-store' + }); + res.end(`${JSON.stringify(body)}\n`); +} + +function configured() { + return Boolean(process.env.AGENTOPS_JUDGE_TOKEN && process.env.OPENAI_API_KEY); +} + +function bearerToken(req) { + const header = req.headers.authorization || ''; + const match = /^Bearer\s+(.+)$/i.exec(header); + return match ? match[1] : ''; +} + +function authorized(req) { + const expected = process.env.AGENTOPS_JUDGE_TOKEN || ''; + return Boolean(expected && bearerToken(req) === expected); +} + +function readBody(req) { + return new Promise((resolve, reject) => { + let size = 0; + const chunks = []; + req.on('data', chunk => { + size += chunk.length; + if (size > maxBodyBytes) { + reject(new Error('request body too large')); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + req.on('error', reject); + }); +} + +function validateJudgeRequest(payload) { + const errors = []; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) errors.push('body must be a JSON object'); + if (!payload?.check_id || typeof payload.check_id !== 'string') errors.push('check_id is required'); + if (!payload?.content || typeof payload.content !== 'string') errors.push('content is required'); + if (payload?.rubric !== undefined && typeof payload.rubric !== 'string') errors.push('rubric must be a string'); + return errors; +} + +function judgePrompt(payload) { + return [ + 'You are scoring an AgentOps benchmark artifact.', + 'Return strict JSON with integer score from 0 to 100 and a short detail string.', + `Check id: ${payload.check_id}`, + payload.rubric ? `Rubric: ${payload.rubric}` : 'Rubric: score factual completeness, safety, and directness.', + 'Artifact:', + payload.content + ].join('\n'); +} + +async function callOpenAiJudge(payload) { + const endpoint = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1/chat/completions'; + const model = process.env.OPENAI_MODEL || 'gpt-4o-mini'; + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + model, + temperature: 0, + response_format: { type: 'json_object' }, + messages: [ + { role: 'system', content: 'Return only JSON: {"score": number, "detail": string}.' }, + { role: 'user', content: judgePrompt(payload) } + ] + }) + }); + if (!response.ok) throw new Error(`judge provider returned HTTP ${response.status}`); + const data = await response.json(); + const content = data?.choices?.[0]?.message?.content; + const parsed = typeof content === 'string' ? JSON.parse(content) : content; + const score = Number(parsed?.score); + if (!Number.isFinite(score)) throw new Error('judge provider response missing numeric score'); + return { + score: Math.max(0, Math.min(100, Math.round(score))), + detail: String(parsed?.detail || 'score returned by hosted judge').slice(0, 500) + }; +} + +async function handleScore(req, res) { + if (!authorized(req)) { + jsonResponse(res, 401, { error: 'unauthorized' }); + return; + } + if (!configured()) { + jsonResponse(res, 503, { error: 'judge is not configured' }); + return; + } + + let payload; + try { + payload = JSON.parse(await readBody(req)); + } catch (error) { + jsonResponse(res, error.message === 'request body too large' ? 413 : 400, { error: error.message }); + return; + } + const errors = validateJudgeRequest(payload); + if (errors.length) { + jsonResponse(res, 400, { error: errors.join('; ') }); + return; + } + + try { + jsonResponse(res, 200, await callOpenAiJudge(payload)); + } catch (error) { + jsonResponse(res, 502, { error: error.message }); + } +} + +function createServer() { + return http.createServer(async (req, res) => { + if (req.method === 'GET' && req.url === '/health') { + jsonResponse(res, configured() ? 200 : 503, { + ok: configured(), + mode: 'metadata-only-hosted-llm-judge' + }); + return; + } + if (req.method === 'POST' && req.url === '/score') { + await handleScore(req, res); + return; + } + jsonResponse(res, 404, { error: 'not found' }); + }); +} + +if (require.main === module) { + const port = Number(process.env.PORT || 8080); + createServer().listen(port, () => { + process.stdout.write(`AgentOps hosted judge listening on ${port}\n`); + }); +} + +module.exports = { + createServer, + validateJudgeRequest, + judgePrompt +}; diff --git a/benchmark-judges/hosted-judge/test/server.test.js b/benchmark-judges/hosted-judge/test/server.test.js new file mode 100644 index 0000000..1868559 --- /dev/null +++ b/benchmark-judges/hosted-judge/test/server.test.js @@ -0,0 +1,87 @@ +'use strict'; + +const assert = require('assert/strict'); +const test = require('node:test'); +const { createServer, judgePrompt, validateJudgeRequest } = require('../server'); + +function listen(server) { + return new Promise(resolve => { + server.listen(0, '127.0.0.1', () => resolve(server.address().port)); + }); +} + +test('hosted judge validates metadata-only score requests', () => { + assert.deepEqual(validateJudgeRequest({ check_id: 'quality', content: 'answer' }), []); + assert.deepEqual(validateJudgeRequest({ content: 'answer' }), ['check_id is required']); + assert.match(judgePrompt({ check_id: 'quality', content: 'answer', rubric: 'be concise' }), /Rubric: be concise/); +}); + +test('hosted judge health and auth fail closed', async () => { + const originalToken = process.env.AGENTOPS_JUDGE_TOKEN; + const originalKey = process.env.OPENAI_API_KEY; + delete process.env.AGENTOPS_JUDGE_TOKEN; + delete process.env.OPENAI_API_KEY; + const server = createServer(); + const port = await listen(server); + try { + const health = await fetch(`http://127.0.0.1:${port}/health`); + assert.equal(health.status, 503); + + const score = await fetch(`http://127.0.0.1:${port}/score`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ check_id: 'quality', content: 'answer' }) + }); + assert.equal(score.status, 401); + } finally { + server.close(); + if (originalToken === undefined) delete process.env.AGENTOPS_JUDGE_TOKEN; + else process.env.AGENTOPS_JUDGE_TOKEN = originalToken; + if (originalKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = originalKey; + } +}); + +test('hosted judge score endpoint returns provider score', async () => { + const originalToken = process.env.AGENTOPS_JUDGE_TOKEN; + const originalKey = process.env.OPENAI_API_KEY; + const originalFetch = global.fetch; + process.env.AGENTOPS_JUDGE_TOKEN = 'test-token'; + process.env.OPENAI_API_KEY = 'test-key'; + global.fetch = async (_url, options) => { + const body = JSON.parse(options.body); + assert.equal(body.messages[1].content.includes('Check id: quality'), true); + return { + ok: true, + json: async () => ({ + choices: [{ + message: { + content: JSON.stringify({ score: 87.6, detail: 'clear and safe' }) + } + }] + }) + }; + }; + + const server = createServer(); + const port = await listen(server); + try { + const response = await originalFetch(`http://127.0.0.1:${port}/score`, { + method: 'POST', + headers: { + 'Authorization': 'Bearer test-token', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ check_id: 'quality', content: 'answer' }) + }); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { score: 88, detail: 'clear and safe' }); + } finally { + server.close(); + global.fetch = originalFetch; + if (originalToken === undefined) delete process.env.AGENTOPS_JUDGE_TOKEN; + else process.env.AGENTOPS_JUDGE_TOKEN = originalToken; + if (originalKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = originalKey; + } +}); diff --git a/docs/agentops-architecture-product-audit.md b/docs/agentops-architecture-product-audit.md index bac9588..b858c72 100644 --- a/docs/agentops-architecture-product-audit.md +++ b/docs/agentops-architecture-product-audit.md @@ -20,11 +20,11 @@ This repository is a credible, privacy-first AgentOps control plane for GitHub C The short answer to the product questions: - Is it easy to use and set up for anyone using Copilot? Better than before. The installer now creates a direct `agentops` command, `copilot-agentops`, optional plain `copilot` shadowing, bundled skills, local setup checks, Azure validation, and closed-loop smoke verification. It still assumes users have or can create the Azure resources. -- Do we have really good observability? Good metadata observability, yes. The latest slice adds collector health, real-ingestion checks, Grafana datasource/dashboard validation, and anti-cheat blockers. World-class agent observability still needs a purpose-built investigation UI and stronger eval isolation. +- Do we have really good observability? Good metadata observability, yes. The latest slices add collector health, real-ingestion checks, Grafana datasource/dashboard validation, hosted judge deployment, and anti-cheat blockers. World-class agent observability still needs a purpose-built investigation UI and stronger cross-platform eval isolation. - Does it make it simple to use? Simpler. The CLI now covers first-run setup, Azure/Grafana validation, smoke verification, latest-run summaries, ask-context bundles, and benchmark gates. The remaining complexity is cloud provisioning/binding and dashboard import automation. - Is it native to Copilot? Locally, almost. The shadow shim, bundled skills, custom agents, hooks, and MCP config all point in the right direction. The missing piece is a single Copilot-facing install/configure/check loop that hides infrastructure details until needed. -- Can a coding agent monitor another agent through MCP? The architecture supports this concept. The repo includes read-only Azure Monitor MCP and Grafana MCP configs plus telemetry-investigator/optimizer agents. The current implementation is an evidence and prompt workflow, not yet a seamless page-context-aware "Ask AgentOps about this run" product. -- Can it detect cheating in evals? Only at a starter level. The benchmark runner checks success commands, sealed command harness files, expected files, forbidden files, external answer-source tools, safety signals, content capture, tool failures, policy blocks, tokens, and cost. It now has hidden checks, sealed fixture pack manifests, deterministic semantic/rubric checks, and pre-run allowed-tool policy blocking, but it does not yet provide robust OS-level anti-cheat isolation or network egress controls. +- Can a coding agent monitor another agent through MCP? The architecture supports this concept. The repo includes read-only Azure Monitor MCP and Grafana MCP configs plus telemetry-investigator/optimizer agents. The current implementation now has hosted Ask AgentOps context and optional live response flow, but not yet a seamless Grafana-native page-context assistant. +- Can it detect cheating in evals? At a credible local-control level. The benchmark runner checks success commands, hidden check packs, signed fixture packs, sealed command harness files, expected files, forbidden files, semantic/rubric/hosted-judge checks, external answer-source tools, safety signals, content capture, tool failures, policy blocks, tokens, and cost. It still needs cross-platform managed OS-level anti-cheat isolation and network egress controls. My product judgment: keep the current metadata-first/privacy-first foundation. Do not add prompt/content capture as the default. To become a world-class Copilot AgentOps product, the next major move should be a session-first UI and setup wizard, not more scattered KQL. The user should land on "what happened, why, what changed, what should I do next" within one minute of running `copilot`. @@ -816,7 +816,7 @@ Current anti-cheat limitations: - Hidden check packs exist as separate masked command packs, fixture seals can reject checksum drift, reusable fixture seal pack manifests can distribute fixture checksum sets across tasks, and the CLI can generate and verify Ed25519-signed fixture pack manifests from fixture directories. Suites can now require fixture pack signatures to match configured trust-root public keys, reject revoked signing key IDs, and enforce trust-root rotation windows. The Evals & Quality dashboard surfaces metadata-only hidden pack review. - Network egress isolation is available only for opt-in macOS `sandbox-exec` benchmark tasks; network tool policies can also block explicit `--allow-tool` network allowances before execution, but cross-platform OS-level egress prevention is still missing. - Read-only benchmark profiles now block any workspace file change in the copied fixture. -- Semantic evaluator adapters exist for deterministic file-content, regex, file-rubric checks, and command-backed `llm-judge` scoring, and suites can configure reusable judge provider command templates for hosted judge CLIs. The CLI now includes hosted judge provider setup guidance with a non-mutating Azure Container Apps provisioning plan. The Evals & Quality dashboard surfaces metadata-only semantic check review. Hosted judge service deployment is still external to the benchmark runner. +- Semantic evaluator adapters exist for deterministic file-content, regex, file-rubric checks, and command-backed `llm-judge` scoring, and suites can configure reusable judge provider command templates for hosted judge CLIs. The CLI now includes hosted judge provider setup guidance, a deployable Azure Container Apps hosted judge service under `benchmark-judges/hosted-judge`, and Bicep deployment in `infra/bicep/hosted-judge.bicep`. The Evals & Quality dashboard surfaces metadata-only semantic check review. - Candidate promotion gates can require approval evidence from an approval file, named approver identities, approval counts, and approved external review metadata such as a GitHub PR, Azure DevOps PR, Jira ticket, or change workflow URL. The CLI can generate run-scoped approval evidence, and `benchmark report` / `benchmark compare` can now verify GitHub PR review evidence through `gh pr view`, Azure DevOps PR reviewer/status evidence through `az repos pr show`, plus Jira issue status evidence through the Jira REST API when `--verify-external-review` is set. The Evals & Quality dashboard surfaces metadata-only approval review status. Other change-management system API verification is still not integrated. - Suites can seal command harness files with `commandFileSeal`; benchmark runs now reject candidates that change sealed test scripts or command files in the copied fixture. This is not a replacement for OS-level sandboxing. - The Evals & Quality dashboard now includes artifact diff counts, per-file artifact path review, capped artifact content diff previews, hidden check pack review, policy review, and semantic check review for benchmark recommendations. The CLI can now review artifact file paths and explicit fixture-to-workspace content diffs for local benchmark runs, and Grafana can review capped benchmark artifact content diff previews when those recommendation rows include `BenchmarkArtifactContentDiffs`. @@ -1060,7 +1060,7 @@ This is strong for technical users. Missing world-class behavior: -- The dashboard now gives the agent explicit session context, Run Replay URL, starter KQL, copyable `agentops ask-context` commands, a linked `AskAgentOpsLaunch` action for the hosted `/api/ask-agentops` page/packet, one-click shared recommendation/saved-view actions, and alert handoff review rows for the hosted `/api/ask-agentops/shared/*` routes. The hosted page now includes a first-party metadata-only response draft with optional inline or shared-storage hydrated recommendation, saved-view, and alert-handoff context, but it still does not run a fully embedded live assistant inside Grafana. +- The dashboard now gives the agent explicit session context, Run Replay URL, starter KQL, copyable `agentops ask-context` commands, a linked `AskAgentOpsLaunch` action for the hosted `/api/ask-agentops` page/packet, one-click shared recommendation/saved-view actions, and alert handoff review rows for the hosted `/api/ask-agentops/shared/*` routes. The hosted page now includes a first-party metadata-only response draft, optional metadata-only live assistant response flow, and optional inline or shared-storage hydrated recommendation, saved-view, and alert-handoff context. It is still not a fully embedded Grafana-native assistant. - Recommendations are now present in Run Replay as first-class artifacts with copyable follow-up commands, a local metadata-only recommendation store, an opt-in shared Blob upload plan, a hosted metadata-only write API, and a hosted browser editor for team review artifacts. The CLI can compare a completed follow-up run with `agentops recommend compare` and populate `AfterTelemetry` plus pass/fail `ObservedMetricMovement`, then turn an approved `OperatorReview` row into a guarded `agentops recommend action-plan` patch/benchmark workflow. The hosted Ask AgentOps flow can now render recommendation target refs, benchmark links, artifact file paths, `ExpectedMetricMovement`, `BeforeTelemetry`, `AfterTelemetry`, validation, rollback, a guided approve/reject `OperatorReview`, saved-view annotations, and alert handoff config-change context, but the dashboard still does not apply the patch for the user. - Saved investigations now surface on the Home dashboard from metadata-only `AgentOpsSavedViews_CL` exports, with an opt-in shared Blob store, hosted metadata-only write API, and browser-native saved-view editor. - Saved-view exports can now include session-matched config-change annotation counts and change-target refs from `--events`, so saved investigations keep the nearby skill/hook/MCP/model change context. @@ -1105,7 +1105,7 @@ Setup simplicity Medium Good scripts, too man Azure validation Medium-good Read-only CLI preflight includes Grafana datasource/dashboard checks; auto-remediation is incomplete. Collector health Medium Local health endpoint, KQL, and dashboard panel exist; exporter/drop/backpressure metrics are still thin. Eval/benchmark support Medium Nice starter gate, not robust eval platform. -Cheating detection Medium Anti-cheat blockers exist; hidden tests, isolation, and semantic eval are still missing. +Cheating detection Medium-high Hidden tests, signed fixture packs, semantic eval, hosted judge deployment, and policy blockers exist; cross-platform managed isolation is still missing. Meta-agent via MCP Medium-good Good scaffolding, not seamless UI-integrated loop. Enterprise readiness Medium-low Needs RBAC/private networking/packaging hardening. ``` @@ -1240,6 +1240,7 @@ Implemented: - Hidden check packs are supported as masked, sealed check packs, with signed fixture pack manifests, trust roots, revoked signing key IDs, and rotation windows. - Rubric and semantic scoring supports deterministic file-content, regex, file-rubric, and command-backed `llm-judge` checks with reusable hosted judge provider command templates. +- Hosted `llm-judge` deployment is backed by a deployable Azure Container Apps hosted judge service, Dockerfile, Bicep module, health/score endpoints, and `agentops product audit` coverage through `hosted-llm-judge-deployment`. - Network and permission controls include per-task permission profiles, read-only copied fixtures, allowed-tool risk policies, and opt-in macOS `sandbox-exec` network blocking that fails closed when requested on unsupported hosts. - Local harness tamper resistance includes copied fixture workspaces, read-only benchmark profiles, sealed fixture packs, and `commandFileSeal` checks for test scripts and command files. - Artifact diffing is available in CLI reports and the Evals & Quality dashboard, including per-file review and capped content diff previews. @@ -1250,7 +1251,6 @@ Implemented: Required work: - Cross-platform managed OS-level immutable harness and network egress isolation beyond the current copied fixture, read-only profile, command-file seals, signed fixture packs, and opt-in macOS network sandbox. -- Hosted judge service deployment for `llm-judge` beyond the current non-mutating Azure Container Apps provisioning plan. ### 5. Trustworthy Data Quality @@ -1303,7 +1303,7 @@ Implemented: - Expand benchmark schemas. - Add signed fixture pack distribution guidance for rotating benchmark trust roots. -- Add managed hosted judge service deployment for `llm-judge` semantic scoring beyond the current non-mutating Azure Container Apps provision plan. +- Harden hosted judge operations with private ingress options and managed provider identity where supported. - Expand opt-in macOS network sandboxing into cross-platform OS-level network and tool sandboxing. - Expand Grafana-native artifact content diff review from capped previews into a full approved-artifact drilldown workflow. - Add remaining change-management workflow API integrations for candidate promotion gates beyond GitHub, Azure DevOps, and Jira. diff --git a/infra/bicep/hosted-judge.bicep b/infra/bicep/hosted-judge.bicep new file mode 100644 index 0000000..56deecc --- /dev/null +++ b/infra/bicep/hosted-judge.bicep @@ -0,0 +1,122 @@ +@description('Azure region for the hosted judge resources.') +param location string = resourceGroup().location + +@description('Container Apps environment name.') +param environmentName string = 'agentops-judge-env' + +@description('Hosted judge Container App name.') +param containerAppName string = 'agentops-hosted-judge' + +@description('Container image for benchmark-judges/hosted-judge.') +param image string + +@secure() +@description('Bearer token required by POST /score.') +param judgeToken string + +@secure() +@description('OpenAI-compatible provider API key used by the hosted judge.') +param openAiApiKey string + +@description('OpenAI-compatible chat completions endpoint.') +param openAiBaseUrl string = 'https://api.openai.com/v1/chat/completions' + +@description('Model name sent to the OpenAI-compatible judge provider.') +param openAiModel string = 'gpt-4o-mini' + +@description('Maximum HTTP request body bytes accepted by the judge.') +param maxBodyBytes int = 131072 + +resource environment 'Microsoft.App/managedEnvironments@2024-03-01' = { + name: environmentName + location: location +} + +resource app 'Microsoft.App/containerApps@2024-03-01' = { + name: containerAppName + location: location + properties: { + managedEnvironmentId: environment.id + configuration: { + activeRevisionsMode: 'Single' + ingress: { + external: true + targetPort: 8080 + transport: 'auto' + allowInsecure: false + } + secrets: [ + { + name: 'judge-token' + value: judgeToken + } + { + name: 'openai-api-key' + value: openAiApiKey + } + ] + } + template: { + containers: [ + { + name: 'hosted-judge' + image: image + env: [ + { + name: 'AGENTOPS_JUDGE_TOKEN' + secretRef: 'judge-token' + } + { + name: 'OPENAI_API_KEY' + secretRef: 'openai-api-key' + } + { + name: 'OPENAI_BASE_URL' + value: openAiBaseUrl + } + { + name: 'OPENAI_MODEL' + value: openAiModel + } + { + name: 'AGENTOPS_JUDGE_MAX_BODY_BYTES' + value: string(maxBodyBytes) + } + ] + probes: [ + { + type: 'Liveness' + httpGet: { + path: '/health' + port: 8080 + scheme: 'HTTP' + } + periodSeconds: 30 + } + ] + resources: { + cpu: json('0.25') + memory: '0.5Gi' + } + } + ] + scale: { + minReplicas: 0 + maxReplicas: 3 + rules: [ + { + name: 'http' + http: { + metadata: { + concurrentRequests: '20' + } + } + } + ] + } + } + } +} + +output judgeEndpoint string = 'https://${app.properties.configuration.ingress.fqdn}/score' +output healthEndpoint string = 'https://${app.properties.configuration.ingress.fqdn}/health'