diff --git a/actioner/README.md b/actioner/README.md index d0b2a35..c1b9087 100644 --- a/actioner/README.md +++ b/actioner/README.md @@ -25,7 +25,7 @@ If required metadata is missing, it returns `needs-review` and does not create a ## Ask AgentOps Launcher -Use the hosted launcher to open an assistant with run-scoped metadata already assembled. It does not call an LLM by itself; it returns a safe prompt, a first-party metadata-only response draft, and, when `AGENTOPS_ASSISTANT_URL` is configured, an assistant launch URL with the prompt encoded. POST bodies can include schema-valid metadata-only `recommendation`, `saved_view`, and `alert_handoff` packets, or POST to `/api/ask-agentops/shared` with shared Blob ids so the Function App hydrates those packets through input bindings. The page links recommendation `ChangeTargetRefs`, benchmark run id, artifact file paths, `ExpectedMetricMovement`, `BeforeTelemetry`, `AfterTelemetry`, `ObservedMetricMovement`, validation steps, rollback condition, saved-view query/tag/annotation context, and alert handoff owner/query/config-change context without rendering raw diff content. When recommendation evidence is present, the page also shows a guided review section with approve/reject controls that write an `OperatorReview` metadata object back through the shared-store API when configured, plus an `agentops recommend action-plan --recommendation-id ` handoff for the guarded patch/benchmark workflow. Approved recommendations also render a guarded apply packet with a patch handoff only after operator approval, benchmark evidence, after-run metric movement, validation, rollback, and change-target refs are present. +Use the hosted launcher to open an assistant with run-scoped metadata already assembled. It returns a safe prompt, a first-party metadata-only response draft, and, when `AGENTOPS_ASSISTANT_URL` is configured, an assistant launch URL with the prompt encoded. When `AGENTOPS_ASSISTANT_API_URL` or `assistant_api_url` is configured, the hosted page also renders an inline live assistant form that POSTs the metadata-only prompt and compact context to that endpoint and displays the response inside the page. POST bodies can include schema-valid metadata-only `recommendation`, `saved_view`, and `alert_handoff` packets, or POST to `/api/ask-agentops/shared` with shared Blob ids so the Function App hydrates those packets through input bindings. The page links recommendation `ChangeTargetRefs`, benchmark run id, artifact file paths, `ExpectedMetricMovement`, `BeforeTelemetry`, `AfterTelemetry`, `ObservedMetricMovement`, validation steps, rollback condition, saved-view query/tag/annotation context, and alert handoff owner/query/config-change context without rendering raw diff content. When recommendation evidence is present, the page also shows a guided review section with approve/reject controls that write an `OperatorReview` metadata object back through the shared-store API when configured, plus an `agentops recommend action-plan --recommendation-id ` handoff for the guarded patch/benchmark workflow. Approved recommendations also render a guarded apply packet with a patch handoff only after operator approval, benchmark evidence, after-run metric movement, validation, rollback, and change-target refs are present. HTTP route: diff --git a/actioner/index.js b/actioner/index.js index 49ae351..d01a065 100644 --- a/actioner/index.js +++ b/actioner/index.js @@ -320,6 +320,7 @@ function buildAskAgentOpsLaunch(payload = {}, options = {}) { const savedView = savedViewEvidenceFromPayload(payload); const alertHandoff = alertHandoffEvidenceFromPayload(payload); const assistantBaseUrl = stringValue(options.assistantBaseUrl || payload.assistant_url || process.env.AGENTOPS_ASSISTANT_URL).trim(); + const assistantApiUrl = stringValue(options.assistantApiUrl || payload.assistant_api_url || process.env.AGENTOPS_ASSISTANT_API_URL).trim(); const errors = []; if (!runId && !sessionId && !traceId) errors.push('run_id, session_id, or trace_id is required'); @@ -378,6 +379,20 @@ function buildAskAgentOpsLaunch(payload = {}, options = {}) { shared_context: hydration.context, prompt: errors.length ? null : prompt, assistant_response: assistantResponse, + live_assistant: errors.length ? null : buildLiveAssistantRequest({ + apiUrl: assistantApiUrl, + prompt, + runId, + sessionId, + traceId, + last, + dashboardUrl, + selectedEvent, + benchmark, + recommendation: recommendation?.summary || null, + savedView: savedView?.summary || null, + alertHandoff: alertHandoff?.summary || null + }), launch_url: !errors.length && assistantBaseUrl ? `${assistantBaseUrl}${assistantBaseUrl.includes('?') ? '&' : '?'}q=${encodeURIComponent(prompt)}` : null, @@ -394,6 +409,48 @@ function buildAskAgentOpsLaunch(payload = {}, options = {}) { }; } +function buildLiveAssistantRequest(context = {}) { + const recommendation = context.recommendation || null; + const savedView = context.savedView || null; + const alertHandoff = context.alertHandoff || null; + const apiUrl = stringValue(context.apiUrl).trim(); + const metadata = { + run_id: context.runId || null, + session_id: context.sessionId || null, + trace_id: context.traceId || null, + last: context.last || null, + dashboard_url_present: Boolean(context.dashboardUrl), + selected_event: context.selectedEvent || null, + benchmark_run_id: context.benchmark || recommendation?.benchmark_run_id || null, + recommendation_id: recommendation?.recommendation_id || null, + saved_view_id: savedView?.saved_view_id || null, + alert_rule: alertHandoff?.rule || null, + change_target_refs: [ + ...(recommendation?.change_target_refs || []), + ...(savedView?.change_target_refs || []) + ].filter(Boolean) + }; + return { + mode: 'metadata-only-live-assistant-request', + status: apiUrl ? 'ready' : 'disabled', + api_url: apiUrl || null, + request: apiUrl ? { + prompt: context.prompt, + context: metadata, + privacy: { + mode: 'metadata-only', + excluded: ['raw user prompts', 'model responses', 'tool arguments', 'tool results', 'source code', 'file contents', 'request bodies', 'response bodies', 'secrets'] + } + } : null, + fallback_response: 'metadata-only-assistant-response', + guardrails: [ + 'Post only the metadata-only prompt and compact context.', + 'Render the assistant answer inside this hosted page.', + 'Do not send raw user prompt text, model responses, tool arguments, tool results, source code, file contents, request bodies, response bodies, or secrets.' + ] + }; +} + function objectPayload(value) { if (!value) return null; if (typeof value === 'object' && !Array.isArray(value)) return value; @@ -982,6 +1039,14 @@ function renderAskAgentOpsLaunch(packet) { const launch = packet.launch_url ? `

Open Assistant

` : '

Set AGENTOPS_ASSISTANT_URL to enable a direct assistant launch URL. The metadata-only prompt is ready to copy.

'; + const liveAssistant = packet.live_assistant?.status === 'ready' + ? `
+

Live Assistant

+

Run a metadata-only live assistant response inside this page.

+ + 
${htmlEscape(JSON.stringify({ status: 'ready', mode: packet.live_assistant.mode }, null, 2))}
+
` + : '

Set AGENTOPS_ASSISTANT_API_URL to enable an inline live assistant response flow. The first-party response draft remains available below.

'; return ` @@ -1013,6 +1078,7 @@ function renderAskAgentOpsLaunch(packet) {

Ask AgentOps

Metadata-only assistant context for ${htmlEscape(packet.run_id || packet.session_id || packet.trace_id || 'selected run')}.

${packet.status === 'ready' ? launch : `

${htmlEscape(packet.errors.join('; '))}

`} + ${packet.status === 'ready' ? liveAssistant : ''} ${packet.assistant_response ? `

Response Draft

${htmlEscape(packet.assistant_response.summary)}

@@ -1070,6 +1136,26 @@ function renderAskAgentOpsLaunch(packet) { }); }); } + const liveBox = document.querySelector('.live-assistant'); + if (liveBox) { + const output = document.getElementById('live-assistant-output'); + document.getElementById('live-assistant-run').addEventListener('click', async () => { + const request = JSON.parse(liveBox.dataset.request); + output.textContent = JSON.stringify({ status: 'running', mode: 'metadata-only-live-assistant-request' }, null, 2); + try { + const response = await fetch(liveBox.dataset.apiUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request) + }); + const contentType = response.headers.get('content-type') || ''; + const body = contentType.includes('application/json') ? await response.json() : { text: await response.text() }; + output.textContent = JSON.stringify({ status: response.ok ? 'ok' : 'error', response: body }, null, 2); + } catch (error) { + output.textContent = JSON.stringify({ status: 'error', error: error.message }, null, 2); + } + }); + } `; diff --git a/agentops-cli/src/commands/product.js b/agentops-cli/src/commands/product.js index 805dd57..59c2da8 100644 --- a/agentops-cli/src/commands/product.js +++ b/agentops-cli/src/commands/product.js @@ -297,6 +297,15 @@ function productAudit(options = {}) { [] )); + checks.push(check( + 'ask-agentops-live-response-flow', + fileIncludes('actioner/index.js', ['metadata-only-live-assistant-request', 'AGENTOPS_ASSISTANT_API_URL', 'live-assistant-run', 'fetch(liveBox.dataset.apiUrl']) + && fileIncludes('actioner/README.md', ['AGENTOPS_ASSISTANT_API_URL', 'inline live assistant form', 'metadata-only prompt and compact context']) + && fileIncludes('docs/agentops-architecture-product-audit.md', ['browser-native metadata-only live assistant response flow', 'AGENTOPS_ASSISTANT_API_URL']), + ['actioner/index.js', 'actioner/README.md', 'docs/agentops-architecture-product-audit.md'], + [] + )); + checks.push(check( 'ask-agentops-shared-context', fileIncludes('actioner/index.js', ['savedViewEvidenceFromPayload', 'alertHandoffEvidenceFromPayload', 'hydrateAskAgentOpsPayload', 'shared_context', 'recommendationBlob', 'savedViewBlob', 'alertHandoffBlob']) diff --git a/agentops-cli/test/index.test.js b/agentops-cli/test/index.test.js index 5b32b8e..416b958 100644 --- a/agentops-cli/test/index.test.js +++ b/agentops-cli/test/index.test.js @@ -4336,6 +4336,7 @@ test('product audit proves the local AgentOps control-room contract', () => { 'content-transcript-opt-in', 'first-run-loop', 'ask-agentops-response-flow', + 'ask-agentops-live-response-flow', 'agent-improvement-guarded-apply' ]) { assert.equal(byName[name].ok, true, name); @@ -9059,7 +9060,8 @@ test('ask agentops launcher builds metadata-only assistant context', async () => saved_view: savedViewRow, alert_handoff: alertHandoff }, { - assistantBaseUrl: 'https://assistant.example/ask' + assistantBaseUrl: 'https://assistant.example/ask', + assistantApiUrl: 'https://assistant.example/api/respond' }); assert.equal(packet.schema_version, 'agentops.ask-agentops-launch.v1'); @@ -9073,6 +9075,13 @@ test('ask agentops launcher builds metadata-only assistant context', async () => assert.equal(packet.alert_handoff.rule, 'failed-spans'); assert.equal(packet.assistant_response.mode, 'metadata-only-assistant-response'); assert.equal(packet.assistant_response.status, 'draft'); + assert.equal(packet.live_assistant.mode, 'metadata-only-live-assistant-request'); + assert.equal(packet.live_assistant.status, 'ready'); + assert.equal(packet.live_assistant.api_url, 'https://assistant.example/api/respond'); + assert.equal(packet.live_assistant.request.context.run_id, 'run-123'); + assert.equal(packet.live_assistant.request.context.recommendation_id, 'rec-123'); + assert.ok(packet.live_assistant.request.context.change_target_refs.includes('skill:agentops-latest-run')); + assert.match(packet.live_assistant.request.prompt, /Use only metadata/); assert.ok(packet.assistant_response.evidence.includes('RunId=run-123')); assert.ok(packet.assistant_response.evidence.includes('RecommendationId=rec-123')); assert.ok(packet.assistant_response.evidence.includes('RecommendationBenchmarkRunId=bench-123')); @@ -9168,6 +9177,7 @@ test('ask agentops launcher builds metadata-only assistant context', async () => assert.equal(context.res.headers['Content-Type'], 'application/json'); assert.equal(context.res.body.mode, 'metadata-only-assistant-launch'); assert.equal(context.res.body.assistant_response.mode, 'metadata-only-assistant-response'); + assert.equal(context.res.body.live_assistant.status, 'disabled'); const sharedContext = { bindings: { recommendationBlob: sharedRecommendationBlob, @@ -9223,6 +9233,7 @@ test('ask agentops launcher builds metadata-only assistant context', async () => assert.equal(htmlContext.res.headers['Content-Type'], 'text/html; charset=utf-8'); assert.match(htmlContext.res.body, /Response Draft/); assert.match(htmlContext.res.body, /Root-cause candidates/); + assert.match(htmlContext.res.body, /AGENTOPS_ASSISTANT_API_URL/); const recommendationHtmlContext = {}; await askAgentOps(recommendationHtmlContext, { @@ -9230,13 +9241,17 @@ test('ask agentops launcher builds metadata-only assistant context', async () => run_id: 'run-123', recommendation: recommendationRow, saved_view: savedViewRow, - alert_handoff: alertHandoff + alert_handoff: alertHandoff, + assistant_api_url: 'https://assistant.example/api/respond' }, headers: { accept: 'text/html' } }); assert.equal(recommendationHtmlContext.res.status, 200); + assert.match(recommendationHtmlContext.res.body, /Live Assistant/); + assert.match(recommendationHtmlContext.res.body, /metadata-only-live-assistant-request/); + assert.match(recommendationHtmlContext.res.body, /live-assistant-run/); assert.match(recommendationHtmlContext.res.body, /Recommendation/); assert.match(recommendationHtmlContext.res.body, /skill:agentops-latest-run/); assert.match(recommendationHtmlContext.res.body, /benchmarks\/output\.md/); diff --git a/docs/agentops-architecture-product-audit.md b/docs/agentops-architecture-product-audit.md index a4022cd..bac9588 100644 --- a/docs/agentops-architecture-product-audit.md +++ b/docs/agentops-architecture-product-audit.md @@ -1223,12 +1223,9 @@ Success criterion: Every suggested agent/skill/hook/MCP change has telemetry evidence, predicted metric movement, validation, and rollback. ``` -Required work: - -- Replace the deterministic first-party response draft with a live assistant response flow inside Grafana or the hosted AgentOps page. - Implemented: +- `AskAgentOps` now supports a browser-native metadata-only live assistant response flow inside the hosted page: when `AGENTOPS_ASSISTANT_API_URL` or `assistant_api_url` is configured, the page POSTs the safe prompt and compact run/recommendation context to that endpoint and renders the response inline. The deterministic first-party response draft remains as the fallback. - Approved recommendation patch application now has a hosted guarded apply packet: `AskAgentOps` returns `metadata-only-guarded-apply` with branch/action-plan/benchmark/compare commands and a patch handoff only after operator approval, benchmark evidence, after-run metric movement, validation, rollback, and change-target refs are present. ### 4. Robust Eval Center