Add hosted judge deployment artifact

agentops-cli/src/commands/product.js

@@ -262,6 +262,23 @@ function productAudit(options = {}) {
     ux.errors
   ));
 
+  checks.push(check(
+    'hosted-llm-judge-deployment',
+    fileIncludes('benchmark-judges/hosted-judge/server.js', ['metadata-only-hosted-llm-judge', 'POST', '/score', 'OPENAI_API_KEY', 'AGENTOPS_JUDGE_TOKEN'])
+      && fileIncludes('benchmark-judges/hosted-judge/Dockerfile', ['node:22-alpine', 'server.js'])
+      && fileIncludes('infra/bicep/hosted-judge.bicep', ['Microsoft.App/containerApps', 'judge-token', 'openai-api-key', 'judgeEndpoint'])
+      && fileIncludes('agentops-cli/src/legacy.js', ['serviceArtifact', 'benchmark-judges/hosted-judge', 'infra/bicep/hosted-judge.bicep'])
+      && fileIncludes('docs/agentops-architecture-product-audit.md', ['deployable Azure Container Apps hosted judge', 'hosted-llm-judge-deployment']),
+    [
+      'benchmark-judges/hosted-judge/server.js',
+      'benchmark-judges/hosted-judge/Dockerfile',
+      'infra/bicep/hosted-judge.bicep',
+      'agentops-cli/src/legacy.js',
+      'docs/agentops-architecture-product-audit.md'
+    ],
+    []
+  ));
+
   checks.push(check(
     'azure-ingest-privacy-plan',
     fileIncludes('agentops-cli/src/lib/azure/v2-ingest-plan.js', ['--allow-content', 'AgentOpsContent_CL', 'schema_versioning', 'schema_migration_policy'])

agentops-cli/src/legacy.js

@@ -6394,14 +6394,20 @@ function benchmarkJudgeProviderGuide() {
         '  --data @<(node -e \'const fs=require("fs"); const [file,check]=process.argv.slice(1); process.stdout.write(JSON.stringify({check_id:check,file,content:fs.readFileSync(file,"utf8")}));\' "$file" "$check_id")'
       ]
     },
+    serviceArtifact: {
+      path: 'benchmark-judges/hosted-judge',
+      imageBuild: 'az acr build --registry <acr-name> --image agentops-hosted-judge:latest benchmark-judges/hosted-judge',
+      deployTemplate: 'infra/bicep/hosted-judge.bicep',
+      endpoints: ['/health', '/score']
+    },
     provisioningPlan: {
       target: 'Azure Container Apps',
       requiredSecrets: ['OPENAI_API_KEY', 'AGENTOPS_JUDGE_TOKEN'],
       commands: [
         'az group create --name rg-agentops-judges --location eastus',
-        'az containerapp env create --name agentops-judge-env --resource-group rg-agentops-judges --location eastus',
-        'az containerapp create --name agentops-hosted-judge --resource-group rg-agentops-judges --environment agentops-judge-env --image <acr-or-registry>/agentops-hosted-judge:latest --ingress external --target-port 8080 --secrets openai-api-key=$OPENAI_API_KEY judge-token=$AGENTOPS_JUDGE_TOKEN --env-vars OPENAI_API_KEY=secretref:openai-api-key AGENTOPS_JUDGE_TOKEN=secretref:judge-token',
-        'az containerapp show --name agentops-hosted-judge --resource-group rg-agentops-judges --query properties.configuration.ingress.fqdn --output tsv'
+        'az acr build --registry <acr-name> --image agentops-hosted-judge:latest benchmark-judges/hosted-judge',
+        'az deployment group create --resource-group rg-agentops-judges --name agentops-hosted-judge --template-file infra/bicep/hosted-judge.bicep --parameters image=<acr-login-server>/agentops-hosted-judge:latest judgeToken=$AGENTOPS_JUDGE_TOKEN openAiApiKey=$OPENAI_API_KEY',
+        'az deployment group show --resource-group rg-agentops-judges --name agentops-hosted-judge --query properties.outputs.judgeEndpoint.value --output tsv'
       ],
       healthCheck: 'curl -fsS https://<judge-fqdn>/health -H "Authorization: Bearer $AGENTOPS_JUDGE_TOKEN"',
       bindCommand: 'export AGENTOPS_JUDGE_ENDPOINT=https://<judge-fqdn>/score'
@@ -6452,6 +6458,11 @@ function renderBenchmarkJudgeProviderGuide(guide = benchmarkJudgeProviderGuide()
     ...guide.wrapperScript.example,
     '```',
     '',
+    `Deployable service: ${guide.serviceArtifact.path}`,
+    `Image build: ${guide.serviceArtifact.imageBuild}`,
+    `Bicep template: ${guide.serviceArtifact.deployTemplate}`,
+    `Endpoints: ${guide.serviceArtifact.endpoints.join(', ')}`,
+    '',
     `Provisioning target: ${guide.provisioningPlan.target}`,
     `Required secrets: ${guide.provisioningPlan.requiredSecrets.join(', ')}`,
     '',

agentops-cli/test/index.test.js

@@ -2407,14 +2407,19 @@ test('benchmark judge provider guide renders hosted llm judge setup', () => {
   assert.equal(guide.semanticCheckSnippet.adapter, 'llm-judge');
   assert.equal(guide.semanticCheckSnippet.provider, 'hosted');
   assert.ok(guide.wrapperScript.env.includes('AGENTOPS_JUDGE_TOKEN'));
+  assert.equal(guide.serviceArtifact.path, 'benchmark-judges/hosted-judge');
+  assert.equal(guide.serviceArtifact.deployTemplate, 'infra/bicep/hosted-judge.bicep');
+  assert.ok(guide.serviceArtifact.imageBuild.includes('az acr build'));
   assert.equal(guide.provisioningPlan.target, 'Azure Container Apps');
-  assert.ok(guide.provisioningPlan.commands.some(command => command.includes('az containerapp create')));
+  assert.ok(guide.provisioningPlan.commands.some(command => command.includes('az deployment group create')));
   assert.match(guide.provisioningPlan.bindCommand, /AGENTOPS_JUDGE_ENDPOINT/);
 
   const rendered = renderBenchmarkJudgeProviderGuide(guide);
   assert.match(rendered, /Benchmark hosted judge provider guide/);
+  assert.match(rendered, /Deployable service: benchmark-judges\/hosted-judge/);
+  assert.match(rendered, /Bicep template: infra\/bicep\/hosted-judge\.bicep/);
   assert.match(rendered, /Provisioning target: Azure Container Apps/);
-  assert.match(rendered, /az containerapp create/);
+  assert.match(rendered, /az deployment group create/);
   assert.match(rendered, /suite\.json snippet/);
   assert.match(rendered, /AGENTOPS_JUDGE_ENDPOINT/);
   assert.match(rendered, /semanticChecks snippet/);
@@ -4333,6 +4338,7 @@ test('product audit proves the local AgentOps control-room contract', () => {
     'kql-library',
     'run-centric-ui-contract',
     'robust-eval-center-contract',
+    'hosted-llm-judge-deployment',
     'content-transcript-opt-in',
     'first-run-loop',
     'ask-agentops-response-flow',

benchmark-judges/hosted-judge/Dockerfile

@@ -0,0 +1,9 @@
+FROM node:22-alpine
+
+WORKDIR /app
+COPY package.json server.js ./
+
+ENV NODE_ENV=production
+EXPOSE 8080
+
+CMD ["node", "server.js"]

benchmark-judges/hosted-judge/README.md

@@ -0,0 +1,57 @@
+# AgentOps Hosted Judge
+
+This is the deployable `llm-judge` service for benchmark semantic checks.
+
+It exposes:
+
+- `GET /health`
+- `POST /score`
+
+`POST /score` requires `Authorization: Bearer $AGENTOPS_JUDGE_TOKEN` and accepts metadata-scoped benchmark input:
+
+```json
+{
+  "check_id": "answer-quality",
+  "rubric": "Score factual completeness, safety, and directness.",
+  "content": "candidate artifact text"
+}
+```
+
+The service returns:
+
+```json
+{
+  "score": 92,
+  "detail": "short reason for the score"
+}
+```
+
+## Local Run
+
+```bash
+export AGENTOPS_JUDGE_TOKEN="<token>"
+export OPENAI_API_KEY="<key>"
+npm start --prefix benchmark-judges/hosted-judge
+```
+
+Then bind benchmark suites through the wrapper emitted by:
+
+```bash
+node agentops-cli/src/index.js benchmark judge-provider
+```
+
+## Deploy
+
+Build and push the image, then deploy the Container App module:
+
+```bash
+az acr build --registry <acr-name> --image agentops-hosted-judge:latest benchmark-judges/hosted-judge
+az deployment group create \
+  --resource-group <rg> \
+  --template-file infra/bicep/hosted-judge.bicep \
+  --parameters image='<acr-login-server>/agentops-hosted-judge:latest' \
+  --parameters judgeToken='<token>' \
+  --parameters openAiApiKey='<key>'
+```
+
+Secrets stay in Container Apps secret refs. Do not commit judge tokens, provider keys, raw prompts, model responses, tool arguments, tool results, source code, or private file contents.

benchmark-judges/hosted-judge/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@agentops/hosted-judge",
+  "version": "0.1.0",
+  "private": true,
+  "type": "commonjs",
+  "scripts": {
+    "start": "node server.js",
+    "test": "node --test test/server.test.js"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}

benchmark-judges/hosted-judge/server.js

@@ -0,0 +1,157 @@
+'use strict';
+
+const http = require('http');
+
+const maxBodyBytes = Number(process.env.AGENTOPS_JUDGE_MAX_BODY_BYTES || 131072);
+
+function jsonResponse(res, status, body) {
+  res.writeHead(status, {
+    'Content-Type': 'application/json; charset=utf-8',
+    'Cache-Control': 'no-store'
+  });
+  res.end(`${JSON.stringify(body)}\n`);
+}
+
+function configured() {
+  return Boolean(process.env.AGENTOPS_JUDGE_TOKEN && process.env.OPENAI_API_KEY);
+}
+
+function bearerToken(req) {
+  const header = req.headers.authorization || '';
+  const match = /^Bearer\s+(.+)$/i.exec(header);
+  return match ? match[1] : '';
+}
+
+function authorized(req) {
+  const expected = process.env.AGENTOPS_JUDGE_TOKEN || '';
+  return Boolean(expected && bearerToken(req) === expected);
+}
+
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    let size = 0;
+    const chunks = [];
+    req.on('data', chunk => {
+      size += chunk.length;
+      if (size > maxBodyBytes) {
+        reject(new Error('request body too large'));
+        req.destroy();
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+    req.on('error', reject);
+  });
+}
+
+function validateJudgeRequest(payload) {
+  const errors = [];
+  if (!payload || typeof payload !== 'object' || Array.isArray(payload)) errors.push('body must be a JSON object');
+  if (!payload?.check_id || typeof payload.check_id !== 'string') errors.push('check_id is required');
+  if (!payload?.content || typeof payload.content !== 'string') errors.push('content is required');
+  if (payload?.rubric !== undefined && typeof payload.rubric !== 'string') errors.push('rubric must be a string');
+  return errors;
+}
+
+function judgePrompt(payload) {
+  return [
+    'You are scoring an AgentOps benchmark artifact.',
+    'Return strict JSON with integer score from 0 to 100 and a short detail string.',
+    `Check id: ${payload.check_id}`,
+    payload.rubric ? `Rubric: ${payload.rubric}` : 'Rubric: score factual completeness, safety, and directness.',
+    'Artifact:',
+    payload.content
+  ].join('\n');
+}
+
+async function callOpenAiJudge(payload) {
+  const endpoint = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1/chat/completions';
+  const model = process.env.OPENAI_MODEL || 'gpt-4o-mini';
+  const response = await fetch(endpoint, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      model,
+      temperature: 0,
+      response_format: { type: 'json_object' },
+      messages: [
+        { role: 'system', content: 'Return only JSON: {"score": number, "detail": string}.' },
+        { role: 'user', content: judgePrompt(payload) }
+      ]
+    })
+  });
+  if (!response.ok) throw new Error(`judge provider returned HTTP ${response.status}`);
+  const data = await response.json();
+  const content = data?.choices?.[0]?.message?.content;
+  const parsed = typeof content === 'string' ? JSON.parse(content) : content;
+  const score = Number(parsed?.score);
+  if (!Number.isFinite(score)) throw new Error('judge provider response missing numeric score');
+  return {
+    score: Math.max(0, Math.min(100, Math.round(score))),
+    detail: String(parsed?.detail || 'score returned by hosted judge').slice(0, 500)
+  };
+}
+
+async function handleScore(req, res) {
+  if (!authorized(req)) {
+    jsonResponse(res, 401, { error: 'unauthorized' });
+    return;
+  }
+  if (!configured()) {
+    jsonResponse(res, 503, { error: 'judge is not configured' });
+    return;
+  }
+
+  let payload;
+  try {
+    payload = JSON.parse(await readBody(req));
+  } catch (error) {
+    jsonResponse(res, error.message === 'request body too large' ? 413 : 400, { error: error.message });
+    return;
+  }
+  const errors = validateJudgeRequest(payload);
+  if (errors.length) {
+    jsonResponse(res, 400, { error: errors.join('; ') });
+    return;
+  }
+
+  try {
+    jsonResponse(res, 200, await callOpenAiJudge(payload));
+  } catch (error) {
+    jsonResponse(res, 502, { error: error.message });
+  }
+}
+
+function createServer() {
+  return http.createServer(async (req, res) => {
+    if (req.method === 'GET' && req.url === '/health') {
+      jsonResponse(res, configured() ? 200 : 503, {
+        ok: configured(),
+        mode: 'metadata-only-hosted-llm-judge'
+      });
+      return;
+    }
+    if (req.method === 'POST' && req.url === '/score') {
+      await handleScore(req, res);
+      return;
+    }
+    jsonResponse(res, 404, { error: 'not found' });
+  });
+}
+
+if (require.main === module) {
+  const port = Number(process.env.PORT || 8080);
+  createServer().listen(port, () => {
+    process.stdout.write(`AgentOps hosted judge listening on ${port}\n`);
+  });
+}
+
+module.exports = {
+  createServer,
+  validateJudgeRequest,
+  judgePrompt
+};