Add hosted Ask AgentOps live response flow

actioner/README.md

@@ -25,7 +25,7 @@ If required metadata is missing, it returns `needs-review` and does not create a
 
 ## Ask AgentOps Launcher
 
-Use the hosted launcher to open an assistant with run-scoped metadata already assembled. It does not call an LLM by itself; it returns a safe prompt, a first-party metadata-only response draft, and, when `AGENTOPS_ASSISTANT_URL` is configured, an assistant launch URL with the prompt encoded. POST bodies can include schema-valid metadata-only `recommendation`, `saved_view`, and `alert_handoff` packets, or POST to `/api/ask-agentops/shared` with shared Blob ids so the Function App hydrates those packets through input bindings. The page links recommendation `ChangeTargetRefs`, benchmark run id, artifact file paths, `ExpectedMetricMovement`, `BeforeTelemetry`, `AfterTelemetry`, `ObservedMetricMovement`, validation steps, rollback condition, saved-view query/tag/annotation context, and alert handoff owner/query/config-change context without rendering raw diff content. When recommendation evidence is present, the page also shows a guided review section with approve/reject controls that write an `OperatorReview` metadata object back through the shared-store API when configured, plus an `agentops recommend action-plan --recommendation-id <id>` handoff for the guarded patch/benchmark workflow. Approved recommendations also render a guarded apply packet with a patch handoff only after operator approval, benchmark evidence, after-run metric movement, validation, rollback, and change-target refs are present.
+Use the hosted launcher to open an assistant with run-scoped metadata already assembled. It returns a safe prompt, a first-party metadata-only response draft, and, when `AGENTOPS_ASSISTANT_URL` is configured, an assistant launch URL with the prompt encoded. When `AGENTOPS_ASSISTANT_API_URL` or `assistant_api_url` is configured, the hosted page also renders an inline live assistant form that POSTs the metadata-only prompt and compact context to that endpoint and displays the response inside the page. POST bodies can include schema-valid metadata-only `recommendation`, `saved_view`, and `alert_handoff` packets, or POST to `/api/ask-agentops/shared` with shared Blob ids so the Function App hydrates those packets through input bindings. The page links recommendation `ChangeTargetRefs`, benchmark run id, artifact file paths, `ExpectedMetricMovement`, `BeforeTelemetry`, `AfterTelemetry`, `ObservedMetricMovement`, validation steps, rollback condition, saved-view query/tag/annotation context, and alert handoff owner/query/config-change context without rendering raw diff content. When recommendation evidence is present, the page also shows a guided review section with approve/reject controls that write an `OperatorReview` metadata object back through the shared-store API when configured, plus an `agentops recommend action-plan --recommendation-id <id>` handoff for the guarded patch/benchmark workflow. Approved recommendations also render a guarded apply packet with a patch handoff only after operator approval, benchmark evidence, after-run metric movement, validation, rollback, and change-target refs are present.
 
 HTTP route:
 

actioner/index.js

@@ -320,6 +320,7 @@ function buildAskAgentOpsLaunch(payload = {}, options = {}) {
   const savedView = savedViewEvidenceFromPayload(payload);
   const alertHandoff = alertHandoffEvidenceFromPayload(payload);
   const assistantBaseUrl = stringValue(options.assistantBaseUrl || payload.assistant_url || process.env.AGENTOPS_ASSISTANT_URL).trim();
+  const assistantApiUrl = stringValue(options.assistantApiUrl || payload.assistant_api_url || process.env.AGENTOPS_ASSISTANT_API_URL).trim();
   const errors = [];
 
   if (!runId && !sessionId && !traceId) errors.push('run_id, session_id, or trace_id is required');
@@ -378,6 +379,20 @@ function buildAskAgentOpsLaunch(payload = {}, options = {}) {
     shared_context: hydration.context,
     prompt: errors.length ? null : prompt,
     assistant_response: assistantResponse,
+    live_assistant: errors.length ? null : buildLiveAssistantRequest({
+      apiUrl: assistantApiUrl,
+      prompt,
+      runId,
+      sessionId,
+      traceId,
+      last,
+      dashboardUrl,
+      selectedEvent,
+      benchmark,
+      recommendation: recommendation?.summary || null,
+      savedView: savedView?.summary || null,
+      alertHandoff: alertHandoff?.summary || null
+    }),
     launch_url: !errors.length && assistantBaseUrl
       ? `${assistantBaseUrl}${assistantBaseUrl.includes('?') ? '&' : '?'}q=${encodeURIComponent(prompt)}`
       : null,
@@ -394,6 +409,48 @@ function buildAskAgentOpsLaunch(payload = {}, options = {}) {
   };
 }
 
+function buildLiveAssistantRequest(context = {}) {
+  const recommendation = context.recommendation || null;
+  const savedView = context.savedView || null;
+  const alertHandoff = context.alertHandoff || null;
+  const apiUrl = stringValue(context.apiUrl).trim();
+  const metadata = {
+    run_id: context.runId || null,
+    session_id: context.sessionId || null,
+    trace_id: context.traceId || null,
+    last: context.last || null,
+    dashboard_url_present: Boolean(context.dashboardUrl),
+    selected_event: context.selectedEvent || null,
+    benchmark_run_id: context.benchmark || recommendation?.benchmark_run_id || null,
+    recommendation_id: recommendation?.recommendation_id || null,
+    saved_view_id: savedView?.saved_view_id || null,
+    alert_rule: alertHandoff?.rule || null,
+    change_target_refs: [
+      ...(recommendation?.change_target_refs || []),
+      ...(savedView?.change_target_refs || [])
+    ].filter(Boolean)
+  };
+  return {
+    mode: 'metadata-only-live-assistant-request',
+    status: apiUrl ? 'ready' : 'disabled',
+    api_url: apiUrl || null,
+    request: apiUrl ? {
+      prompt: context.prompt,
+      context: metadata,
+      privacy: {
+        mode: 'metadata-only',
+        excluded: ['raw user prompts', 'model responses', 'tool arguments', 'tool results', 'source code', 'file contents', 'request bodies', 'response bodies', 'secrets']
+      }
+    } : null,
+    fallback_response: 'metadata-only-assistant-response',
+    guardrails: [
+      'Post only the metadata-only prompt and compact context.',
+      'Render the assistant answer inside this hosted page.',
+      'Do not send raw user prompt text, model responses, tool arguments, tool results, source code, file contents, request bodies, response bodies, or secrets.'
+    ]
+  };
+}
+
 function objectPayload(value) {
   if (!value) return null;
   if (typeof value === 'object' && !Array.isArray(value)) return value;
@@ -982,6 +1039,14 @@ function renderAskAgentOpsLaunch(packet) {
   const launch = packet.launch_url
     ? `<p><a class="button" href="${htmlEscape(packet.launch_url)}">Open Assistant</a></p>`
     : '<p class="note">Set <code>AGENTOPS_ASSISTANT_URL</code> to enable a direct assistant launch URL. The metadata-only prompt is ready to copy.</p>';
+  const liveAssistant = packet.live_assistant?.status === 'ready'
+    ? `<section class="note live-assistant" data-api-url="${htmlEscape(packet.live_assistant.api_url)}" data-request='${htmlEscape(JSON.stringify(packet.live_assistant.request))}'>
+      <h2>Live Assistant</h2>
+      <p>Run a metadata-only live assistant response inside this page.</p>
+      <button type="button" id="live-assistant-run">Ask Live Assistant</button>
+      <pre id="live-assistant-output">${htmlEscape(JSON.stringify({ status: 'ready', mode: packet.live_assistant.mode }, null, 2))}</pre>
+    </section>`
+    : '<p class="note">Set <code>AGENTOPS_ASSISTANT_API_URL</code> to enable an inline live assistant response flow. The first-party response draft remains available below.</p>';
   return `<!doctype html>
 <html lang="en">
 <head>
@@ -1013,6 +1078,7 @@ function renderAskAgentOpsLaunch(packet) {
     <h1>Ask AgentOps</h1>
     <p class="note">Metadata-only assistant context for ${htmlEscape(packet.run_id || packet.session_id || packet.trace_id || 'selected run')}.</p>
     ${packet.status === 'ready' ? launch : `<p class="note">${htmlEscape(packet.errors.join('; '))}</p>`}
+    ${packet.status === 'ready' ? liveAssistant : ''}
     ${packet.assistant_response ? `<section class="note">
       <h2>Response Draft</h2>
       <p>${htmlEscape(packet.assistant_response.summary)}</p>
@@ -1070,6 +1136,26 @@ function renderAskAgentOpsLaunch(packet) {
         });
       });
     }
+    const liveBox = document.querySelector('.live-assistant');
+    if (liveBox) {
+      const output = document.getElementById('live-assistant-output');
+      document.getElementById('live-assistant-run').addEventListener('click', async () => {
+        const request = JSON.parse(liveBox.dataset.request);
+        output.textContent = JSON.stringify({ status: 'running', mode: 'metadata-only-live-assistant-request' }, null, 2);
+        try {
+          const response = await fetch(liveBox.dataset.apiUrl, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(request)
+          });
+          const contentType = response.headers.get('content-type') || '';
+          const body = contentType.includes('application/json') ? await response.json() : { text: await response.text() };
+          output.textContent = JSON.stringify({ status: response.ok ? 'ok' : 'error', response: body }, null, 2);
+        } catch (error) {
+          output.textContent = JSON.stringify({ status: 'error', error: error.message }, null, 2);
+        }
+      });
+    }
   </script>
 </body>
 </html>`;

agentops-cli/src/commands/product.js

@@ -297,6 +297,15 @@ function productAudit(options = {}) {
     []
   ));
 
+  checks.push(check(
+    'ask-agentops-live-response-flow',
+    fileIncludes('actioner/index.js', ['metadata-only-live-assistant-request', 'AGENTOPS_ASSISTANT_API_URL', 'live-assistant-run', 'fetch(liveBox.dataset.apiUrl'])
+      && fileIncludes('actioner/README.md', ['AGENTOPS_ASSISTANT_API_URL', 'inline live assistant form', 'metadata-only prompt and compact context'])
+      && fileIncludes('docs/agentops-architecture-product-audit.md', ['browser-native metadata-only live assistant response flow', 'AGENTOPS_ASSISTANT_API_URL']),
+    ['actioner/index.js', 'actioner/README.md', 'docs/agentops-architecture-product-audit.md'],
+    []
+  ));
+
   checks.push(check(
     'ask-agentops-shared-context',
     fileIncludes('actioner/index.js', ['savedViewEvidenceFromPayload', 'alertHandoffEvidenceFromPayload', 'hydrateAskAgentOpsPayload', 'shared_context', 'recommendationBlob', 'savedViewBlob', 'alertHandoffBlob'])

agentops-cli/test/index.test.js

@@ -4336,6 +4336,7 @@ test('product audit proves the local AgentOps control-room contract', () => {
     'content-transcript-opt-in',
     'first-run-loop',
     'ask-agentops-response-flow',
+    'ask-agentops-live-response-flow',
     'agent-improvement-guarded-apply'
   ]) {
     assert.equal(byName[name].ok, true, name);
@@ -9059,7 +9060,8 @@ test('ask agentops launcher builds metadata-only assistant context', async () =>
     saved_view: savedViewRow,
     alert_handoff: alertHandoff
   }, {
-    assistantBaseUrl: 'https://assistant.example/ask'
+    assistantBaseUrl: 'https://assistant.example/ask',
+    assistantApiUrl: 'https://assistant.example/api/respond'
   });
 
   assert.equal(packet.schema_version, 'agentops.ask-agentops-launch.v1');
@@ -9073,6 +9075,13 @@ test('ask agentops launcher builds metadata-only assistant context', async () =>
   assert.equal(packet.alert_handoff.rule, 'failed-spans');
   assert.equal(packet.assistant_response.mode, 'metadata-only-assistant-response');
   assert.equal(packet.assistant_response.status, 'draft');
+  assert.equal(packet.live_assistant.mode, 'metadata-only-live-assistant-request');
+  assert.equal(packet.live_assistant.status, 'ready');
+  assert.equal(packet.live_assistant.api_url, 'https://assistant.example/api/respond');
+  assert.equal(packet.live_assistant.request.context.run_id, 'run-123');
+  assert.equal(packet.live_assistant.request.context.recommendation_id, 'rec-123');
+  assert.ok(packet.live_assistant.request.context.change_target_refs.includes('skill:agentops-latest-run'));
+  assert.match(packet.live_assistant.request.prompt, /Use only metadata/);
   assert.ok(packet.assistant_response.evidence.includes('RunId=run-123'));
   assert.ok(packet.assistant_response.evidence.includes('RecommendationId=rec-123'));
   assert.ok(packet.assistant_response.evidence.includes('RecommendationBenchmarkRunId=bench-123'));
@@ -9168,6 +9177,7 @@ test('ask agentops launcher builds metadata-only assistant context', async () =>
   assert.equal(context.res.headers['Content-Type'], 'application/json');
   assert.equal(context.res.body.mode, 'metadata-only-assistant-launch');
   assert.equal(context.res.body.assistant_response.mode, 'metadata-only-assistant-response');
+  assert.equal(context.res.body.live_assistant.status, 'disabled');
 
   const sharedContext = { bindings: {
     recommendationBlob: sharedRecommendationBlob,
@@ -9223,20 +9233,25 @@ test('ask agentops launcher builds metadata-only assistant context', async () =>
   assert.equal(htmlContext.res.headers['Content-Type'], 'text/html; charset=utf-8');
   assert.match(htmlContext.res.body, /Response Draft/);
   assert.match(htmlContext.res.body, /Root-cause candidates/);
+  assert.match(htmlContext.res.body, /AGENTOPS_ASSISTANT_API_URL/);
 
   const recommendationHtmlContext = {};
   await askAgentOps(recommendationHtmlContext, {
     body: {
       run_id: 'run-123',
       recommendation: recommendationRow,
       saved_view: savedViewRow,
-      alert_handoff: alertHandoff
+      alert_handoff: alertHandoff,
+      assistant_api_url: 'https://assistant.example/api/respond'
     },
     headers: {
       accept: 'text/html'
     }
   });
   assert.equal(recommendationHtmlContext.res.status, 200);
+  assert.match(recommendationHtmlContext.res.body, /Live Assistant/);
+  assert.match(recommendationHtmlContext.res.body, /metadata-only-live-assistant-request/);
+  assert.match(recommendationHtmlContext.res.body, /live-assistant-run/);
   assert.match(recommendationHtmlContext.res.body, /Recommendation/);
   assert.match(recommendationHtmlContext.res.body, /skill:agentops-latest-run/);
   assert.match(recommendationHtmlContext.res.body, /benchmarks\/output\.md/);

docs/agentops-architecture-product-audit.md

@@ -1223,12 +1223,9 @@ Success criterion:
 Every suggested agent/skill/hook/MCP change has telemetry evidence, predicted metric movement, validation, and rollback.
 ```
 
-Required work:
-
-- Replace the deterministic first-party response draft with a live assistant response flow inside Grafana or the hosted AgentOps page.
-
 Implemented:
 
+- `AskAgentOps` now supports a browser-native metadata-only live assistant response flow inside the hosted page: when `AGENTOPS_ASSISTANT_API_URL` or `assistant_api_url` is configured, the page POSTs the safe prompt and compact run/recommendation context to that endpoint and renders the response inline. The deterministic first-party response draft remains as the fallback.
 - Approved recommendation patch application now has a hosted guarded apply packet: `AskAgentOps` returns `metadata-only-guarded-apply` with branch/action-plan/benchmark/compare commands and a patch handoff only after operator approval, benchmark evidence, after-run metric movement, validation, rollback, and change-target refs are present.
 
 ### 4. Robust Eval Center