diff --git a/agentops-cli/src/commands/product.js b/agentops-cli/src/commands/product.js index ed33cc7..635fa3a 100644 --- a/agentops-cli/src/commands/product.js +++ b/agentops-cli/src/commands/product.js @@ -316,8 +316,10 @@ function productAudit(options = {}) { 'azure-ingest-privacy-plan', fileIncludes('agentops-cli/src/lib/azure/v2-ingest-plan.js', ['--allow-content', 'AgentOpsContent_CL', 'schema_versioning', 'schema_migration_policy', 'logs-ingestion-upload-plan']) && fileIncludes('agentops-cli/src/commands/azure-ingest.js', ['logs-upload', '--yes', 'az', 'rest']) + && fileIncludes('infra/bicep/v2-ingestion.bicep', ['AgentOpsRunSummary_CL', 'dataCollectionRules', 'streamDeclarations', 'logsIngestionEndpoint']) + && fileIncludes('infra/bicep/main.bicep', ['deployV2Ingestion', 'AGENTOPS_LOGS_INGESTION_ENDPOINT', 'AGENTOPS_DCR_IMMUTABLE_ID']) && fileIncludes('docs/azure-v2-ingestion.md', ['AgentOpsContent_CL', '--allow-content', 'SchemaVersion', 'schema migration policy', 'azure-ingest logs-upload']), - ['agentops-cli/src/lib/azure/v2-ingest-plan.js', 'agentops-cli/src/commands/azure-ingest.js', 'docs/azure-v2-ingestion.md'], + ['agentops-cli/src/lib/azure/v2-ingest-plan.js', 'agentops-cli/src/commands/azure-ingest.js', 'infra/bicep/v2-ingestion.bicep', 'infra/bicep/main.bicep', 'docs/azure-v2-ingestion.md'], [] )); diff --git a/docs/azure-v2-ingestion.md b/docs/azure-v2-ingestion.md index 4666b15..c559c9f 100644 --- a/docs/azure-v2-ingestion.md +++ b/docs/azure-v2-ingestion.md @@ -31,10 +31,25 @@ It does not create Azure resources or upload data. For metadata-only saved-view, Use Azure Monitor Logs Ingestion API with a Data Collection Rule: -1. Create one custom Log Analytics table per `AgentOps*_CL` table. -2. Create one DCR stream per table using the columns reported by `agentops azure-ingest plan`. -3. Send each JSONL row to the matching DCR stream. -4. Import the V2 dashboard pack from `grafana/dashboards/v2/`. +1. Deploy the optional V2 ingestion infrastructure: + +```bash +az deployment group create \ + --resource-group \ + --template-file infra/bicep/main.bicep \ + --parameters deployV2Ingestion=true +``` + +2. Export the deployment outputs: + +```bash +export AGENTOPS_LOGS_INGESTION_ENDPOINT="" +export AGENTOPS_DCR_IMMUTABLE_ID="" +``` + +3. Confirm one custom Log Analytics table per `AgentOps*_CL` table and one DCR stream per table exist. +4. Send each JSONL row to the matching DCR stream. +5. Import the V2 dashboard pack from `grafana/dashboards/v2/`. `azure-ingest logs-upload` turns the reviewed plan into Azure Monitor Logs Ingestion API calls. It is dry-run by default and requires `--yes` before it runs `az rest`: @@ -72,7 +87,7 @@ agentops azure-ingest upload-plan \ --container agentops-shared \ --prefix team-a/latest ``` -5. Run `agentops validate-azure --last 24h`. +Run `agentops validate-azure --last 24h` after sharing artifacts. The dashboards never require raw prompts, responses, file contents, tool arguments, or tool results. If you intentionally enable content capture, keep it in `AgentOpsContent_CL`, use `agentops azure-ingest plan --allow-content`, and expose it only through a separate, access-controlled workspace/dashboard. diff --git a/infra/bicep/main.bicep b/infra/bicep/main.bicep index 6a1f020..e7f6b4c 100644 --- a/infra/bicep/main.bicep +++ b/infra/bicep/main.bicep @@ -36,6 +36,9 @@ param actionerAssistantUrl string = '' @description('Deploy shared Azure Blob storage for metadata-only saved-view and recommendation exports.') param deploySharedStore bool = false +@description('Deploy AgentOps V2 custom tables plus Data Collection Endpoint/Rule for Logs Ingestion API uploads.') +param deployV2Ingestion bool = false + @allowed([ 'Enabled' 'Disabled' @@ -186,6 +189,18 @@ module sharedStore 'shared-store.bicep' = if (deploySharedStore) { } } +module v2Ingestion 'v2-ingestion.bicep' = if (deployV2Ingestion) { + name: 'v2-ingestion' + params: { + location: location + workspaceName: logAnalytics.outputs.name + baseName: baseName + environmentName: environmentName + retentionInDays: effectiveLogRetentionDays + tags: tags + } +} + resource sharedStoreAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = if (deployActioner && deploySharedStore) { name: sharedStoreName } @@ -257,3 +272,8 @@ output SHARED_STORE_ACCOUNT_NAME string = deploySharedStore ? sharedStore!.outpu output SHARED_STORE_BLOB_ENDPOINT string = deploySharedStore ? sharedStore!.outputs.blobEndpoint : '' output SHARED_STORE_CONTAINER_NAME string = deploySharedStore ? sharedStore!.outputs.containerName : '' output SHARED_STORE_RESOURCE_ID string = deploySharedStore ? sharedStore!.outputs.resourceId : '' +output V2_INGESTION_DEPLOYED bool = deployV2Ingestion +output AGENTOPS_LOGS_INGESTION_ENDPOINT string = deployV2Ingestion ? v2Ingestion!.outputs.logsIngestionEndpoint : '' +output AGENTOPS_DCR_IMMUTABLE_ID string = deployV2Ingestion ? v2Ingestion!.outputs.dataCollectionRuleImmutableId : '' +output AGENTOPS_DCR_RESOURCE_ID string = deployV2Ingestion ? v2Ingestion!.outputs.dataCollectionRuleResourceId : '' +output AGENTOPS_V2_TABLE_COUNT int = deployV2Ingestion ? v2Ingestion!.outputs.tableCount : 0 diff --git a/infra/bicep/v2-ingestion.bicep b/infra/bicep/v2-ingestion.bicep new file mode 100644 index 0000000..28e6dc6 --- /dev/null +++ b/infra/bicep/v2-ingestion.bicep @@ -0,0 +1,346 @@ +@description('Azure region for V2 ingestion resources.') +param location string = resourceGroup().location + +@description('Log Analytics workspace name that receives AgentOps V2 custom-table rows.') +param workspaceName string + +@description('Base resource name used for the DCE and DCR.') +param baseName string = 'copilot-agentops' + +@description('Deployment environment name.') +param environmentName string = 'dev' + +@description('Retention in days for AgentOps V2 custom tables.') +@maxValue(730) +param retentionInDays int = 30 + +@description('Tags applied to V2 ingestion resources.') +param tags object = {} + +var destinationName = 'agentops-log-analytics' +var tablePlan = 'Analytics' +var effectiveRetentionInDays = retentionInDays < 4 ? 4 : retentionInDays +var v2Tables = [ + { + name: 'AgentOpsRunSummary_CL' + stream: 'Custom-AgentOpsRunSummary_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RunId', type: 'string' } + { name: 'SessionId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'OutcomeStatus', type: 'string' } + { name: 'OutcomeReason', type: 'string' } + { name: 'AgentName', type: 'string' } + { name: 'SkillName', type: 'string' } + { name: 'SubAgentName', type: 'string' } + { name: 'ParentAgentName', type: 'string' } + { name: 'ModelRequested', type: 'string' } + { name: 'ModelActual', type: 'string' } + { name: 'InputTokens', type: 'long' } + { name: 'OutputTokens', type: 'long' } + { name: 'ReasoningTokens', type: 'long' } + { name: 'CacheReadTokens', type: 'long' } + { name: 'CacheCreationTokens', type: 'long' } + { name: 'EstimatedCostUsd', type: 'real' } + { name: 'DurationMs', type: 'long' } + { name: 'ToolCount', type: 'long' } + { name: 'ToolFailureCount', type: 'long' } + { name: 'ToolDeniedCount', type: 'long' } + { name: 'TestsRan', type: 'boolean' } + { name: 'TestsPassed', type: 'boolean' } + { name: 'PrOpened', type: 'boolean' } + { name: 'CiStatus', type: 'string' } + { name: 'ContentCaptureSignal', type: 'boolean' } + { name: 'ContentCaptureMode', type: 'string' } + { name: 'PrivacyMode', type: 'string' } + { name: 'RepoHash', type: 'string' } + { name: 'BranchHash', type: 'string' } + { name: 'PrNumberHash', type: 'string' } + { name: 'Surface', type: 'string' } + { name: 'TaskType', type: 'string' } + { name: 'ScenarioName', type: 'string' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsEvents_CL' + stream: 'Custom-AgentOpsEvents_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RunId', type: 'string' } + { name: 'SessionId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'EventName', type: 'string' } + { name: 'SpanName', type: 'string' } + { name: 'Status', type: 'string' } + { name: 'ToolName', type: 'string' } + { name: 'AgentName', type: 'string' } + { name: 'SkillName', type: 'string' } + { name: 'SubAgentName', type: 'string' } + { name: 'ParentAgentName', type: 'string' } + { name: 'ModelActual', type: 'string' } + { name: 'InputTokens', type: 'long' } + { name: 'OutputTokens', type: 'long' } + { name: 'DurationMs', type: 'long' } + { name: 'EstimatedCostUsd', type: 'long' } + { name: 'ContentCaptureSignal', type: 'boolean' } + { name: 'PrivacyMode', type: 'string' } + { name: 'Surface', type: 'string' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsToolCalls_CL' + stream: 'Custom-AgentOpsToolCalls_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RunId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'SpanId', type: 'string' } + { name: 'ToolName', type: 'string' } + { name: 'ToolType', type: 'string' } + { name: 'ToolRisk', type: 'string' } + { name: 'Status', type: 'string' } + { name: 'Allowed', type: 'boolean' } + { name: 'DeniedReason', type: 'string' } + { name: 'DurationMs', type: 'long' } + { name: 'OutputSizeBytes', type: 'long' } + { name: 'ArgsSchemaHash', type: 'string' } + { name: 'AgentName', type: 'string' } + { name: 'Surface', type: 'string' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsMcpCalls_CL' + stream: 'Custom-AgentOpsMcpCalls_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RunId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'SpanId', type: 'string' } + { name: 'McpServerName', type: 'string' } + { name: 'McpServerHash', type: 'string' } + { name: 'McpClientName', type: 'string' } + { name: 'McpSessionId', type: 'string' } + { name: 'McpTransport', type: 'string' } + { name: 'ToolName', type: 'string' } + { name: 'ToolRisk', type: 'string' } + { name: 'Status', type: 'string' } + { name: 'Allowed', type: 'boolean' } + { name: 'Sandboxed', type: 'boolean' } + { name: 'DurationMs', type: 'long' } + { name: 'ResultSizeBytes', type: 'long' } + { name: 'ArgsSchemaHash', type: 'string' } + { name: 'AgentName', type: 'string' } + { name: 'Surface', type: 'string' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsPrivacy_CL' + stream: 'Custom-AgentOpsPrivacy_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RunId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'ContentKind', type: 'string' } + { name: 'Action', type: 'string' } + { name: 'Observed', type: 'boolean' } + { name: 'LeakDetected', type: 'boolean' } + { name: 'DroppedCount', type: 'long' } + { name: 'RedactedCount', type: 'long' } + { name: 'PrivacyMode', type: 'string' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsEval_CL' + stream: 'Custom-AgentOpsEval_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RunId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'RepoHash', type: 'string' } + { name: 'TaskType', type: 'string' } + { name: 'ModelActual', type: 'string' } + { name: 'EvalOverall', type: 'long' } + { name: 'EvalBucket', type: 'string' } + { name: 'Reliability', type: 'long' } + { name: 'Security', type: 'long' } + { name: 'TestDiscipline', type: 'long' } + { name: 'ToolEfficiency', type: 'long' } + { name: 'ContextEfficiency', type: 'long' } + { name: 'CodeOutcome', type: 'long' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsGithubOutcomes_CL' + stream: 'Custom-AgentOpsGithubOutcomes_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RunId', type: 'string' } + { name: 'RepoHash', type: 'string' } + { name: 'BranchHash', type: 'string' } + { name: 'PrNumberHash', type: 'string' } + { name: 'PrOpened', type: 'boolean' } + { name: 'PrClosed', type: 'boolean' } + { name: 'PrMerged', type: 'boolean' } + { name: 'PrReverted', type: 'boolean' } + { name: 'CiStatus', type: 'string' } + { name: 'CommitCount', type: 'long' } + { name: 'FilesChangedCount', type: 'long' } + { name: 'ReviewCommentCount', type: 'long' } + { name: 'RunStartedAt', type: 'datetime' } + { name: 'PrCreatedAt', type: 'datetime' } + { name: 'PrMergedAt', type: 'datetime' } + { name: 'TimeToPrMinutes', type: 'long' } + { name: 'TimeToMergeMinutes', type: 'long' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsInsights_CL' + stream: 'Custom-AgentOpsInsights_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'InsightId', type: 'string' } + { name: 'InsightType', type: 'string' } + { name: 'Severity', type: 'string' } + { name: 'RunId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'Title', type: 'string' } + { name: 'Summary', type: 'string' } + { name: 'SuggestedNextStep', type: 'string' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsRecommendations_CL' + stream: 'Custom-AgentOpsRecommendations_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'RecommendationId', type: 'string' } + { name: 'Action', type: 'string' } + { name: 'Severity', type: 'string' } + { name: 'ObservedPattern', type: 'string' } + { name: 'NextAction', type: 'string' } + { name: 'RunId', type: 'string' } + { name: 'SessionId', type: 'string' } + { name: 'TraceId', type: 'string' } + { name: 'EvalOverall', type: 'long' } + { name: 'EvalBucket', type: 'string' } + { name: 'DashboardCount', type: 'long' } + { name: 'DashboardTitles', type: 'dynamic' } + { name: 'ChangeTargetRefs', type: 'dynamic' } + { name: 'Validation', type: 'dynamic' } + { name: 'ExpectedMetricMovement', type: 'dynamic' } + { name: 'BeforeTelemetry', type: 'dynamic' } + { name: 'AfterTelemetry', type: 'dynamic' } + { name: 'ObservedMetricMovement', type: 'dynamic' } + { name: 'BenchmarkArtifactFiles', type: 'dynamic' } + { name: 'BenchmarkArtifactContentDiffs', type: 'dynamic' } + { name: 'BenchmarkSemanticChecks', type: 'dynamic' } + { name: 'BenchmarkHiddenCheckPacks', type: 'dynamic' } + { name: 'BenchmarkPermissionProfiles', type: 'dynamic' } + { name: 'SchemaVersion', type: 'string' } + ] + } + { + name: 'AgentOpsCollectorHealth_CL' + stream: 'Custom-AgentOpsCollectorHealth_CL' + columns: [ + { name: 'TimeGenerated', type: 'datetime' } + { name: 'Component', type: 'string' } + { name: 'Status', type: 'string' } + { name: 'CheckName', type: 'string' } + { name: 'Detail', type: 'string' } + { name: 'CollectorMode', type: 'string' } + { name: 'PrivacyMode', type: 'string' } + { name: 'OtlpEndpoint', type: 'string' } + { name: 'AzureConfigured', type: 'boolean' } + { name: 'GrafanaConfigured', type: 'boolean' } + { name: 'PrivacyPoisonOk', type: 'boolean' } + { name: 'DroppedContentCount', type: 'long' } + { name: 'ExportErrors', type: 'long' } + { name: 'ExportFailureReason', type: 'string' } + { name: 'ExportFailureAction', type: 'string' } + { name: 'LastExportSuccess', type: 'datetime' } + { name: 'LastSpanReceived', type: 'datetime' } + { name: 'DashboardVersion', type: 'string' } + { name: 'SchemaVersion', type: 'string' } + ] + } +] + +resource workspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' existing = { + name: workspaceName +} + +resource tables 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = [for table in v2Tables: { + parent: workspace + name: table.name + properties: { + plan: tablePlan + retentionInDays: effectiveRetentionInDays + totalRetentionInDays: effectiveRetentionInDays + schema: { + name: table.name + columns: table.columns + } + } +}] + +resource endpoint 'Microsoft.Insights/dataCollectionEndpoints@2022-06-01' = { + name: 'dce-${baseName}-${environmentName}' + location: location + tags: tags + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } +} + +resource rule 'Microsoft.Insights/dataCollectionRules@2022-06-01' = { + name: 'dcr-${baseName}-${environmentName}-v2' + location: location + tags: tags + properties: { + dataCollectionEndpointId: endpoint.id + streamDeclarations: toObject(v2Tables, table => table.stream, table => { + columns: table.columns + }) + destinations: { + logAnalytics: [ + { + name: destinationName + workspaceResourceId: workspace.id + } + ] + } + dataFlows: [for table in v2Tables: { + streams: [ + table.stream + ] + destinations: [ + destinationName + ] + outputStream: table.stream + }] + } + dependsOn: [ + tables + ] +} + +output dataCollectionEndpointName string = endpoint.name +output dataCollectionEndpointResourceId string = endpoint.id +output logsIngestionEndpoint string = endpoint.properties.logsIngestion.endpoint +output dataCollectionRuleName string = rule.name +output dataCollectionRuleResourceId string = rule.id +output dataCollectionRuleImmutableId string = rule.properties.immutableId +output tableCount int = length(v2Tables) +output streams array = [for table in v2Tables: table.stream]