From 7ffa4484f03e65f28eedcb3a029935d2500a9ee8 Mon Sep 17 00:00:00 2001 From: Daniel Campagnoli Date: Tue, 14 Oct 2025 18:49:24 +0800 Subject: [PATCH 1/2] feat: Add asyncio concurrency helpers to Python agent --- .../codegen/codegenAutonomousAgent.ts | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/agent/autonomous/codegen/codegenAutonomousAgent.ts b/src/agent/autonomous/codegen/codegenAutonomousAgent.ts index 1656266a..cb2b4cdc 100644 --- a/src/agent/autonomous/codegen/codegenAutonomousAgent.ts +++ b/src/agent/autonomous/codegen/codegenAutonomousAgent.ts @@ -1,5 +1,6 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { type Span, SpanStatusCode } from '@opentelemetry/api'; import { type PyodideInterface, loadPyodide } from 'pyodide'; import type { AgentExecution } from '#agent/agentExecutions'; @@ -226,7 +227,6 @@ async function runAgentExecution(agent: AgentContext, span: Span): Promise Date: Wed, 15 Oct 2025 17:34:37 +0800 Subject: [PATCH 2/2] Update slackbot, google cloud logging, code gen py conversion --- .../agent-details.component.html | 2 +- .../autonomous/codegen/agentImageUtils.ts | 1 - .../codegen/codeGenAgentCodeReview.ts | 2 +- .../codegen/codegenAutonomousAgent.ts | 30 +- .../codegen/pyodideDeepConversion.test.ts | 119 ++++++ src/modules/slack/slackApi.ts | 36 +- src/modules/slack/slackBlockFormatter.ts | 17 +- src/modules/slack/slackChatBotService.ts | 32 +- src/routes/chat/sendMessageRoute.ts | 1 + .../webhooks/gitlab/gitlabPipelineHandler.ts | 18 +- src/utils/arrayUtils.test.ts | 377 ++++++++++++++++++ src/utils/arrayUtils.ts | 206 ++++++++++ 12 files changed, 783 insertions(+), 58 deletions(-) create mode 100644 src/agent/autonomous/codegen/pyodideDeepConversion.test.ts create mode 100644 src/utils/arrayUtils.test.ts create mode 100644 src/utils/arrayUtils.ts diff --git a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html index 4cdf9279..742970d7 100644 --- a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html +++ b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html @@ -71,7 +71,7 @@ - +
Human In Loop check diff --git a/src/agent/autonomous/codegen/agentImageUtils.ts b/src/agent/autonomous/codegen/agentImageUtils.ts index e8837ef4..9bf0bf36 100644 --- a/src/agent/autonomous/codegen/agentImageUtils.ts +++ b/src/agent/autonomous/codegen/agentImageUtils.ts @@ -30,7 +30,6 @@ export interface ImageSource { * @returns An array of ImagePartExt objects ready to be included in the LLM prompt. */ export async function checkForImageSources(result: Record, fileStore?: FileStore): Promise { - logger.info('checkForImageSources'); const imageParts: ImagePartExt[] = []; if (!result || typeof result !== 'object') return imageParts; diff --git a/src/agent/autonomous/codegen/codeGenAgentCodeReview.ts b/src/agent/autonomous/codegen/codeGenAgentCodeReview.ts index 42dbf528..0f4d9524 100644 --- a/src/agent/autonomous/codegen/codeGenAgentCodeReview.ts +++ b/src/agent/autonomous/codegen/codeGenAgentCodeReview.ts @@ -30,7 +30,7 @@ Your task is to review the code provided to ensure it follows the following inst ${agentPlanResponse} -First think through your review of the code in the tags against all the review instructions. +First output through your review of the code in the tags against each of the review instructions. Then output the updated code to go in main() method wrapped in tags without any extra indentation. If there are no changes to make then output the existing code as is in the result tags. `; diff --git a/src/agent/autonomous/codegen/codegenAutonomousAgent.ts b/src/agent/autonomous/codegen/codegenAutonomousAgent.ts index cb2b4cdc..4f483fc9 100644 --- a/src/agent/autonomous/codegen/codegenAutonomousAgent.ts +++ b/src/agent/autonomous/codegen/codegenAutonomousAgent.ts @@ -570,7 +570,7 @@ function setupPyodideFunctionProxies( /** * Generates Python code with a helper function and minimal wrappers - * to automatically perform a shallow conversion (.to_py(depth=1)) on JsProxy results. + * to automatically perform deep conversion to native Python types on JsProxy results. */ export function generatePythonWrapper(schemas: FunctionSchema[], generatedPythonCode: string): string { let helperAndWrapperCode = ` @@ -584,21 +584,21 @@ except ImportError: print("Warning: pyodide.ffi.JsProxy not found.", file=sys.stderr) class JsProxy: pass # Dummy class -def _try_shallow_convert_proxy(result, func_name_for_log: str): +def _convert_result(result, func_name_for_log: str): """ - Internal helper: Attempts shallow conversion (.to_py(depth=1)) if result is JsProxy. - Returns converted value or original result. + Internal helper: Deeply convert JsProxy results (lists/dicts/nested) to Python types. """ - if isinstance(result, JsProxy): - try: - # Attempt shallow conversion (converts top-level obj/arr) - return result.to_py(depth=1) - except Exception as e_conv: - # If conversion fails, log warning and return original proxy - print(f"Warning: Failed to shallow convert result of {func_name_for_log}: {e_conv}", file=sys.stderr) - return result # Fallback to the proxy - else: - # If not a proxy (e.g., primitive), return directly + try: + if isinstance(result, JsProxy): + # Full conversion of JsProxy to native Python types + return result.to_py() + if isinstance(result, (list, tuple)): + return [ _convert_result(x, func_name_for_log) for x in result ] + if isinstance(result, dict): + return { k: _convert_result(v, func_name_for_log) for k, v in result.items() } + return result + except Exception as e_conv: + print(f"Warning: Failed to convert result of {func_name_for_log}: {e_conv}", file=sys.stderr) return result # --- Concurrency helpers exposed to generated code (no explicit imports needed) --- @@ -630,7 +630,7 @@ def create_task(coro): async def ${originalName}(*args, **kwargs): try: raw_result = await ${internalName}(*args, **kwargs) - return _try_shallow_convert_proxy(raw_result, '${originalName}') + return _convert_result(raw_result, '${originalName}') except Exception as e_call: print(f"Error during call to underlying JS function '${internalName}': {e_call}", file=sys.stderr) # Optionally print traceback for detailed debugging diff --git a/src/agent/autonomous/codegen/pyodideDeepConversion.test.ts b/src/agent/autonomous/codegen/pyodideDeepConversion.test.ts new file mode 100644 index 00000000..4dacd9ab --- /dev/null +++ b/src/agent/autonomous/codegen/pyodideDeepConversion.test.ts @@ -0,0 +1,119 @@ +import { expect } from 'chai'; +import { type PyodideInterface, loadPyodide } from 'pyodide'; +import type { FunctionSchema } from '#functionSchema/functions'; +import { generatePythonWrapper } from './codegenAutonomousAgent'; +import { mainFnCodeToFullScript } from './pythonCodeGenUtils'; + +describe('Pyodide deep conversion in wrappers', () => { + let py: PyodideInterface; + + before(async function () { + this.timeout(60000); + py = await loadPyodide(); + }); + + it('allows subscripting and membership on returned arrays/dicts', async function () { + this.timeout(30000); + + const schemas: FunctionSchema[] = [ + { class: 'GitLab', name: 'GitLab_getProjects', description: 'list projects', returns: '', parameters: [], returnType: 'Array>' }, + ]; + // Ensure wrapper is generated for this function by including the name in "generatedPythonCode" + const wrapper = generatePythonWrapper(schemas, 'await GitLab_getProjects()'); + + const main = ` +projects = await GitLab_getProjects() +first = projects[0] +owner_name = first['owner']['name'] +ok = 'group/subgroup' in first['fullPath'] + +# Count matches across the list +count = 0 +for p in projects: + if 'group/subgroup' in p['fullPath']: + count += 1 + +# Mutate nested field to ensure dict/list semantics work +first['processed'] = True + +return { + 'ok': ok, + 'count': count, + 'ownerName': owner_name, + 'firstName': first['name'], + 'processed': first['processed'], + 'length': len(projects), +} + `.trim(); + + const script = wrapper + mainFnCodeToFullScript(main); + + const projects = [ + { name: 'project-a', fullPath: 'group/subgroup/project-a', owner: { name: 'team-a' } }, + { name: 'project-b', fullPath: 'group/subgroup/project-b', owner: { name: 'team-b' } }, + ]; + + const globals = py.toPy({ + _GitLab_getProjects: async () => projects, + }); + + const pyResult: any = await py.runPythonAsync(script, { globals }); + const result = pyResult?.toJs ? pyResult.toJs({ dict_converter: Object.fromEntries }) : pyResult; + if (pyResult?.destroy) pyResult.destroy?.(); + + expect(result.ok).to.equal(true); + expect(result.count).to.equal(2); + expect(result.ownerName).to.equal('team-a'); + expect(result.firstName).to.equal('project-a'); + expect(result.processed).to.equal(true); + expect(result.length).to.equal(2); + }); + + it('supports deep list/dict manipulation of returned structures', async function () { + this.timeout(30000); + + const schemas: FunctionSchema[] = [ + { class: 'TestApi', name: 'Api_getData', description: 'get data', returns: '', parameters: [], returnType: 'Record' }, + ]; + const wrapper = generatePythonWrapper(schemas, 'await Api_getData()'); + + const main = ` +data = await Api_getData() + +# Mutate nested list and compute aggregate from nested dict +data['items'][0]['tags'].append('new') +total = sum([v for v in data['metrics'].values()]) + +return { + 'firstTags': data['items'][0]['tags'], + 'total': total, + 'hasNew': 'new' in data['items'][0]['tags'], + 'hasNewInOriginal': 'new' in data['items'][0]['tags'], # Check if original object is modified +} + `.trim(); + + const script = wrapper + mainFnCodeToFullScript(main); + + const jsData = { + items: [ + { id: 1, tags: ['a', 'b'] }, + { id: 2, tags: [] }, + ], + metrics: { a: 1, b: 2, c: 3 }, + }; + + const globals = py.toPy({ + _Api_getData: async () => jsData, + }); + + const pyResult: any = await py.runPythonAsync(script, { globals }); + const result = pyResult?.toJs ? pyResult.toJs({ dict_converter: Object.fromEntries }) : pyResult; + if (pyResult?.destroy) pyResult.destroy?.(); + + expect(result.firstTags).to.deep.equal(['a', 'b', 'new']); + expect(result.total).to.equal(6); + expect(result.hasNew).to.equal(true); + // Ensure the original JS object is NOT modified by Python operations + expect(jsData.items[0].tags).to.deep.equal(['a', 'b']); + }); +}); diff --git a/src/modules/slack/slackApi.ts b/src/modules/slack/slackApi.ts index e0c2f363..74cd21f0 100644 --- a/src/modules/slack/slackApi.ts +++ b/src/modules/slack/slackApi.ts @@ -1,7 +1,10 @@ import { ConversationsHistoryResponse, ConversationsListResponse, ConversationsRepliesResponse, WebClient } from '@slack/web-api'; import { MessageElement } from '@slack/web-api/dist/types/response/ConversationsHistoryResponse'; +import { llms } from '#agent/agentContextLocalStorage'; import { logger } from '#o11y/logger'; +import { formatAsSlackBlocks } from './slackBlockFormatter'; import { SlackConfig, slackConfig } from './slackConfig'; +import { textToBlocks } from './slackMessageFormatter'; /** * A class to interact with the Slack API, specifically for fetching conversations and messages. @@ -10,13 +13,40 @@ export class SlackAPI { private client: WebClient; private config: SlackConfig = slackConfig(); - /** - * Constructs a new SlackAPI instance. - */ constructor() { this.client = new WebClient(this.config.botToken); } + async postMessage(channelId: string, threadTs: string, message: string, reply_ts?: string) { + const params: any = { + channel: channelId, + thread_ts: threadTs, + blocks: await formatAsSlackBlocks(message, llms().easy), + text: message, + }; + + try { + let result = await this.client.chat.postMessage(params); + + if (!result.ok && result.error === 'invalid_blocks_format') { + logger.info({ blocks: params.blocks }, 'Slack invalid_blocks_format. Retrying with medium LLM'); + params.blocks = await formatAsSlackBlocks(message, llms().medium); + result = await this.client.chat.postMessage(params); + } + + if (!result.ok && result.error === 'invalid_blocks_format') { + logger.info({ blocks: params.blocks }, 'Slack invalid_blocks_format. Retrying with textToBlocks'); + params.blocks = textToBlocks(message); + result = await this.client.chat.postMessage(params); + } + + if (!result.ok) throw new Error(`Failed to send message to Slack: ${result.error}`); + } catch (error) { + logger.error(error, 'Error sending message to Slack'); + throw error; + } + } + async getConversationReplies(channelId: string, threadTs: string, limit = 100): Promise { let cursor: string | undefined; const allMessages: MessageElement[] = []; diff --git a/src/modules/slack/slackBlockFormatter.ts b/src/modules/slack/slackBlockFormatter.ts index 08896cf9..59fe1521 100644 --- a/src/modules/slack/slackBlockFormatter.ts +++ b/src/modules/slack/slackBlockFormatter.ts @@ -1,4 +1,5 @@ import { llms } from '#agent/agentContextLocalStorage'; +import { LLM } from '#shared/llm/llm.model'; import { convertMarkdownToMrkdwn } from './slackMessageFormatter'; /* @@ -77,9 +78,7 @@ const SLACK_BLOCKS_SCHEMA = { required: ['blocks'], }; -interface SlackBlocks { - blocks: Array; -} +type SlackBlocks = Array; const SLACK_MARKDOWN_FORMATTING_RULES = [ '## Markdown Formatting Rules Overview', @@ -167,7 +166,7 @@ const SLACK_MARKDOWN_FORMATTING_RULES = [ * Formats markdown to Slack blocks, using markdown blocks, table blocks and divider blocks, as the Slack markdown doesn't support code block with syntax highlighting, horizontal lines, tables, and task list. * @param message */ -export async function formatAsSlackBlocks(markdown: string): Promise { +export async function formatAsSlackBlocks(markdown: string, llm: LLM): Promise { const prompt = `${markdown}\n\nYou are a Slack block formatter. Convert the message text/markdown to Slack blocks. @@ -201,12 +200,14 @@ interface TableBlock { column_settings?: Array<{ align?: string, is_wrapped?: boolean }> } -Return only a JSON object matching the type -{ +interface SlackBlocks { blocks: Array } + +Return only a JSON object matching the type SlackBlocks + `; - const blocks: SlackBlocks = await llms().easy.generateJson(prompt, { jsonSchema: SLACK_BLOCKS_SCHEMA, id: ' Markdown block formatter', temperature: 0 }); + const blocks: { blocks: SlackBlocks } = await llm.generateJson(prompt, { jsonSchema: SLACK_BLOCKS_SCHEMA, id: ' Markdown block formatter', temperature: 0 }); for (const block of blocks.blocks) if (block.type === 'markdown') block.text = convertMarkdownToMrkdwn(block.text); - return blocks; + return blocks.blocks; } diff --git a/src/modules/slack/slackChatBotService.ts b/src/modules/slack/slackChatBotService.ts index 207ab3d3..0cf81c93 100644 --- a/src/modules/slack/slackChatBotService.ts +++ b/src/modules/slack/slackChatBotService.ts @@ -1,5 +1,6 @@ import { App, type KnownEventFromType, type SayFn, StringIndexed } from '@slack/bolt'; import { MessageElement } from '@slack/web-api/dist/types/response/ConversationsHistoryResponse'; +import { llms } from '#agent/agentContextLocalStorage'; import { AgentExecution, isAgentExecuting } from '#agent/agentExecutions'; import { getLastFunctionCallArg } from '#agent/autonomous/agentCompletion'; import { resumeCompletedWithUpdatedUserRequest, startAgent } from '#agent/autonomous/autonomousAgentRunner'; @@ -22,6 +23,7 @@ import { SupportKnowledgebase } from '../../functions/supportKnowledgebase'; import { SlackAPI } from './slackApi'; import { formatAsSlackBlocks } from './slackBlockFormatter'; import { slackConfig } from './slackConfig'; +import { textToBlocks } from './slackMessageFormatter'; let slackApp: App | undefined; @@ -98,31 +100,21 @@ export class SlackChatBotService implements ChatBotService, AgentCompleted { return; } - if (agent.metadata.slack.reply_ts) this.api().removeReaction(agent.metadata.slack.channel, agent.metadata.slack.reply_ts, 'robot_face'); - - const params: any = { - channel: agent.metadata.slack.channel, - thread_ts: agent.metadata.slack.thread_ts, - blocks: await formatAsSlackBlocks(message), - text: message, - }; - - // TODO remove reaction from message it replied to - /* Only add thread_ts if we're in a real thread. - In a channel: event.thread_ts is set for replies - In the App DM: event.thread_ts is undefined */ // if (agent.metadata.thread_ts) { // params.thread_ts = agent.metadata.thread_ts; // } - + const replyTs = agent.metadata.slack.reply_ts; + const channelId = agent.metadata.slack.channel; + const threadTs = agent.metadata.slack.thread_ts; try { - const result = await slackApp.client.chat.postMessage(params); - - if (!result.ok) throw new Error(`Failed to send message to Slack: ${result.error}`); - } catch (error) { - logger.error(error, 'Error sending message to Slack'); - throw error; + this.api().postMessage(channelId, threadTs, message, replyTs); + if (replyTs) this.api().removeReaction(channelId, replyTs, 'robot_face'); + } catch (e) { + logger.error(e, 'Error sending message to Slack'); + if (replyTs) this.api().addReaction(channelId, replyTs, 'robot_face::boom'); } } @@ -330,8 +322,8 @@ export class SlackChatBotService implements ChatBotService, AgentCompleted { completedHandler: this, useSharedRepos: true, // Support bot is read only humanInLoop: { - budget: 2, - count: 10, + budget: 3, + count: 15, }, initialMemory: { 'core-documentation': await supportFuncs.getCoreDocumentation(), diff --git a/src/routes/chat/sendMessageRoute.ts b/src/routes/chat/sendMessageRoute.ts index a6f71105..7eb0b46a 100644 --- a/src/routes/chat/sendMessageRoute.ts +++ b/src/routes/chat/sendMessageRoute.ts @@ -68,6 +68,7 @@ export async function sendMessageRoute(fastify: AppFastifyInstance): Promise 50000) { - // ~50k tokens - // TODO use flash to reduce the size, or just remove the middle section - } - } + // for (const [k, v] of Object.entries(failedLogs)) { + // const lines = v.split('\n').length; + // const tokens = await countTokens(v); + // logger.info(`Failed pipeline job ${k}. Log size: ${tokens} tokens. ${lines} lines.`); + // if (tokens > 50000) { + // // ~50k tokens + // // TODO use flash to reduce the size, or just remove the middle section + // } + // } } } diff --git a/src/utils/arrayUtils.test.ts b/src/utils/arrayUtils.test.ts new file mode 100644 index 00000000..0972f0be --- /dev/null +++ b/src/utils/arrayUtils.test.ts @@ -0,0 +1,377 @@ +import { expect } from 'chai'; +import { deepEqual, extractCommonProperties } from './arrayUtils'; + +describe('arrayUtils', () => { + describe('extractCommonProperties', () => { + it('should return empty common properties for empty array', () => { + const result = extractCommonProperties([]); + + expect(result.commonProps).to.deep.equal({}); + expect(result.strippedItems).to.deep.equal([]); + }); + + it('should return empty common properties for single log', () => { + const logs = [{ id: '123', message: 'test' }]; + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({}); + expect(result.strippedItems).to.deep.equal(logs); + }); + + it('should extract common top-level properties', () => { + const logs = [ + { id: '1', severity: 'INFO', message: 'first' }, + { id: '2', severity: 'INFO', message: 'second' }, + { id: '3', severity: 'INFO', message: 'third' }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ severity: 'INFO' }); + expect(result.strippedItems).to.deep.equal([ + { id: '1', message: 'first' }, + { id: '2', message: 'second' }, + { id: '3', message: 'third' }, + ]); + }); + + it('should handle no common properties', () => { + const logs = [ + { id: '1', severity: 'INFO' }, + { id: '2', severity: 'ERROR' }, + { id: '3', severity: 'WARN' }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({}); + expect(result.strippedItems).to.deep.equal(logs); + }); + }); + + describe('Nested properties', () => { + it('should extract common nested properties', () => { + const logs = [ + { + id: '1', + resource: { + type: 'cloud_scheduler_job', + labels: { project_id: 'test-project', job_id: 'job-1' }, + }, + }, + { + id: '2', + resource: { + type: 'cloud_scheduler_job', + labels: { project_id: 'test-project', job_id: 'job-2' }, + }, + }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ + resource: { + type: 'cloud_scheduler_job', + labels: { project_id: 'test-project' }, + }, + }); + + expect(result.strippedItems).to.deep.equal([ + { id: '1', resource: { labels: { job_id: 'job-1' } } }, + { id: '2', resource: { labels: { job_id: 'job-2' } } }, + ]); + }); + + it('should handle deeply nested common properties', () => { + const logs = [ + { + id: '1', + metadata: { + level1: { + level2: { + level3: 'common-value', + }, + }, + }, + }, + { + id: '2', + metadata: { + level1: { + level2: { + level3: 'common-value', + }, + }, + }, + }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ + metadata: { + level1: { + level2: { + level3: 'common-value', + }, + }, + }, + }); + + expect(result.strippedItems).to.deep.equal([{ id: '1' }, { id: '2' }]); + }); + + it('should cleanup empty parent objects after stripping', () => { + const logs = [ + { + id: '1', + resource: { + labels: { + project_id: 'test-project', + }, + }, + }, + { + id: '2', + resource: { + labels: { + project_id: 'test-project', + }, + }, + }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ + resource: { + labels: { + project_id: 'test-project', + }, + }, + }); + + // Should not have empty resource or labels objects + expect(result.strippedItems).to.deep.equal([{ id: '1' }, { id: '2' }]); + }); + }); + + describe('Array handling', () => { + it('should treat arrays as leaf values', () => { + const logs = [ + { id: '1', tags: ['a', 'b', 'c'] }, + { id: '2', tags: ['a', 'b', 'c'] }, + { id: '3', tags: ['a', 'b', 'c'] }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ + tags: ['a', 'b', 'c'], + }); + + expect(result.strippedItems).to.deep.equal([{ id: '1' }, { id: '2' }, { id: '3' }]); + }); + + it('should not extract arrays with different values', () => { + const logs = [ + { id: '1', tags: ['a', 'b'] }, + { id: '2', tags: ['a', 'c'] }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({}); + expect(result.strippedItems).to.deep.equal(logs); + }); + + it('should handle arrays with different lengths', () => { + const logs = [ + { id: '1', tags: ['a', 'b'] }, + { id: '2', tags: ['a', 'b', 'c'] }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({}); + expect(result.strippedItems).to.deep.equal(logs); + }); + }); + + describe('Real-world Google Cloud log examples', () => { + it('should extract common properties from Cloud Scheduler logs', () => { + const logs = [ + { + insertId: 'abc123def456', + jsonPayload: { + jobName: 'projects/my-project-123/locations/us-central1/jobs/scheduled-job-name', + url: 'https://my-service-xyz.run.app/api/v1/endpoint', + targetType: 'HTTP', + scheduledTime: '2025-10-15T06:00:00.337059Z', + '@type': 'type.googleapis.com/google.cloud.scheduler.logging.AttemptStarted', + }, + resource: { + type: 'cloud_scheduler_job', + labels: { + project_id: 'my-project-123', + job_id: 'scheduled-job-name', + location: 'us-central1', + }, + }, + timestamp: '2025-10-15T06:00:02.109538210Z', + severity: 'INFO', + logName: 'projects/my-project-123/logs/cloudscheduler.googleapis.com%2Fexecutions', + receiveTimestamp: '2025-10-15T06:00:02.109538210Z', + }, + { + insertId: 'xyz789ghi012', + jsonPayload: { + jobName: 'projects/my-project-123/locations/us-central1/jobs/scheduled-job-name', + url: 'https://my-service-xyz.run.app/api/v1/endpoint', + targetType: 'HTTP', + scheduledTime: '2025-10-15T07:00:00.337059Z', + '@type': 'type.googleapis.com/google.cloud.scheduler.logging.AttemptStarted', + }, + resource: { + type: 'cloud_scheduler_job', + labels: { + project_id: 'my-project-123', + job_id: 'scheduled-job-name', + location: 'us-central1', + }, + }, + timestamp: '2025-10-15T07:00:02.109538210Z', + severity: 'INFO', + logName: 'projects/my-project-123/logs/cloudscheduler.googleapis.com%2Fexecutions', + receiveTimestamp: '2025-10-15T07:00:02.109538210Z', + }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ + jsonPayload: { + jobName: 'projects/my-project-123/locations/us-central1/jobs/scheduled-job-name', + url: 'https://my-service-xyz.run.app/api/v1/endpoint', + targetType: 'HTTP', + '@type': 'type.googleapis.com/google.cloud.scheduler.logging.AttemptStarted', + }, + resource: { + type: 'cloud_scheduler_job', + labels: { + project_id: 'my-project-123', + job_id: 'scheduled-job-name', + location: 'us-central1', + }, + }, + severity: 'INFO', + logName: 'projects/my-project-123/logs/cloudscheduler.googleapis.com%2Fexecutions', + }); + + expect(result.strippedItems[0]).to.have.property('insertId', 'abc123def456'); + expect(result.strippedItems[0]).to.have.property('timestamp', '2025-10-15T06:00:02.109538210Z'); + expect(result.strippedItems[0].jsonPayload).to.have.property('scheduledTime', '2025-10-15T06:00:00.337059Z'); + }); + }); + + describe('Edge cases', () => { + it('should handle null values', () => { + const logs = [ + { id: '1', value: null }, + { id: '2', value: null }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ value: null }); + expect(result.strippedItems).to.deep.equal([{ id: '1' }, { id: '2' }]); + }); + + it('should handle undefined values', () => { + const logs = [ + { id: '1', value: undefined }, + { id: '2', value: undefined }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ value: undefined }); + }); + + it('should handle mixed types correctly', () => { + const logs = [ + { id: '1', count: 42 }, + { id: '2', count: '42' }, // string instead of number + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({}); + expect(result.strippedItems).to.deep.equal(logs); + }); + + it('should handle boolean values', () => { + const logs = [ + { id: '1', isActive: true }, + { id: '2', isActive: true }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ isActive: true }); + }); + + it('should handle objects with different key sets', () => { + const logs = [ + { id: '1', metadata: { a: 1, b: 2 } }, + { id: '2', metadata: { a: 1, c: 3 } }, + ]; + + const result = extractCommonProperties(logs); + + expect(result.commonProps).to.deep.equal({ + metadata: { a: 1 }, + }); + }); + }); + + describe('deepEqual helper', () => { + it('should correctly compare primitives', () => { + expect(deepEqual(1, 1)).to.be.true; + expect(deepEqual('a', 'a')).to.be.true; + expect(deepEqual(true, true)).to.be.true; + expect(deepEqual(1, 2)).to.be.false; + expect(deepEqual('a', 'b')).to.be.false; + }); + + it('should correctly compare arrays', () => { + expect(deepEqual([1, 2, 3], [1, 2, 3])).to.be.true; + expect(deepEqual([1, 2], [1, 2, 3])).to.be.false; + expect(deepEqual([1, 2, 3], [1, 3, 2])).to.be.false; + }); + + it('should correctly compare objects', () => { + expect(deepEqual({ a: 1 }, { a: 1 })).to.be.true; + expect(deepEqual({ a: 1, b: 2 }, { b: 2, a: 1 })).to.be.true; + expect(deepEqual({ a: 1 }, { a: 2 })).to.be.false; + expect(deepEqual({ a: 1 }, { b: 1 })).to.be.false; + }); + + it('should handle null and undefined', () => { + expect(deepEqual(null, null)).to.be.true; + expect(deepEqual(undefined, undefined)).to.be.true; + expect(deepEqual(null, undefined)).to.be.false; + expect(deepEqual(null, 0)).to.be.false; + }); + + it('should compare nested structures', () => { + const obj1 = { a: { b: { c: [1, 2, 3] } } }; + const obj2 = { a: { b: { c: [1, 2, 3] } } }; + const obj3 = { a: { b: { c: [1, 2, 4] } } }; + + expect(deepEqual(obj1, obj2)).to.be.true; + expect(deepEqual(obj1, obj3)).to.be.false; + }); + }); +}); diff --git a/src/utils/arrayUtils.ts b/src/utils/arrayUtils.ts new file mode 100644 index 00000000..9965cabf --- /dev/null +++ b/src/utils/arrayUtils.ts @@ -0,0 +1,206 @@ +/** + * Utility functions for JSON manipulation and optimization + */ + +export interface CommonPropertiesResult { + commonProps: any; + strippedItems: any[]; +} + +/** + * Extracts properties that are identical across all items in an array + * Useful for reducing redundancy in JSON responses + * + * @param items - Array of objects to analyze + * @returns Object containing common properties and items with those properties removed + */ +export function extractCommonProperties(items: any[]): CommonPropertiesResult { + if (!Array.isArray(items) || items.length === 0) { + return { commonProps: {}, strippedItems: items }; + } + + // No commonality to extract from a single item + if (items.length === 1) { + return { commonProps: {}, strippedItems: items }; + } + + // Build a map of path -> value from first item + const pathMap = new Map(); + collectPaths(items[0], '', pathMap); + + // Filter out paths that aren't common to ALL items + for (let i = 1; i < items.length; i++) { + const itemPaths = new Map(); + collectPaths(items[i], '', itemPaths); + + // Remove paths that don't match + for (const [path, value] of pathMap) { + if (!itemPaths.has(path) || !deepEqual(itemPaths.get(path), value)) { + pathMap.delete(path); + } + } + + // Early exit if no common paths remain + if (pathMap.size === 0) { + return { commonProps: {}, strippedItems: items }; + } + } + + // Build common props object from paths + const commonProps = buildFromPaths(pathMap); + + // Strip common properties from items + const strippedItems = items.map((item) => stripPaths(item, pathMap)); + + return { commonProps, strippedItems }; +} + +/** + * Deep equality comparison for any values + * Handles primitives, arrays, objects, null, and undefined + * + * @param a - First value + * @param b - Second value + * @returns True if values are deeply equal + */ +export function deepEqual(a: any, b: any): boolean { + // Fast path for primitives and same reference + if (a === b) return true; + + // Fast path for null/undefined + if (a == null || b == null) return false; + + // Type check + if (typeof a !== typeof b) return false; + + // Fast path for primitives + if (typeof a !== 'object') return false; + + // Array comparison + if (Array.isArray(a)) { + if (!Array.isArray(b) || a.length !== b.length) return false; + return a.every((val, i) => deepEqual(val, b[i])); + } + + // Object comparison + const keysA = Object.keys(a); + const keysB = Object.keys(b); + if (keysA.length !== keysB.length) return false; + + return keysA.every((key) => { + if (!Object.prototype.hasOwnProperty.call(b, key)) return false; + return deepEqual(a[key], b[key]); + }); +} + +/** + * Collect all paths in an object using dot notation + * Arrays are treated as leaf values + * + * @param obj - Object to traverse + * @param prefix - Current path prefix + * @param pathMap - Map to store paths and values + */ +function collectPaths(obj: any, prefix: string, pathMap: Map): void { + if (obj === null || typeof obj !== 'object' || Array.isArray(obj)) { + if (prefix) { + pathMap.set(prefix, obj); + } + return; + } + + for (const key in obj) { + if (!Object.prototype.hasOwnProperty.call(obj, key)) continue; + + const path = prefix ? `${prefix}.${key}` : key; + const value = obj[key]; + + if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + collectPaths(value, path, pathMap); + } else { + pathMap.set(path, value); + } + } +} + +/** + * Build object from path map using dot notation + * + * @param pathMap - Map of dot-notation paths to values + * @returns Reconstructed object + */ +function buildFromPaths(pathMap: Map): any { + const result: any = {}; + + for (const [path, value] of pathMap) { + const parts = path.split('.'); + let current = result; + + for (let i = 0; i < parts.length - 1; i++) { + if (!current[parts[i]]) { + current[parts[i]] = {}; + } + current = current[parts[i]]; + } + + current[parts[parts.length - 1]] = value; + } + + return result; +} + +/** + * Strip common paths from object + * Creates a deep clone and removes specified paths + * + * @param obj - Object to strip paths from + * @param pathMap - Map of paths to remove + * @returns New object with paths removed + */ +function stripPaths(obj: any, pathMap: Map): any { + const result = JSON.parse(JSON.stringify(obj)); // Simple deep clone + + for (const path of pathMap.keys()) { + const parts = path.split('.'); + let current = result; + + // Navigate to parent + for (let i = 0; i < parts.length - 1; i++) { + if (!current[parts[i]]) break; + current = current[parts[i]]; + } + + // Delete the property + if (current && parts[parts.length - 1] in current) { + delete current[parts[parts.length - 1]]; + + // Clean up empty parents + cleanupEmpty(result, parts.slice(0, -1)); + } + } + + return result; +} + +/** + * Remove empty objects after stripping properties + * Recursively cleans up parent objects that become empty + * + * @param obj - Root object + * @param pathParts - Path parts to the object to check + */ +function cleanupEmpty(obj: any, pathParts: string[]): void { + if (pathParts.length === 0) return; + + let current = obj; + for (let i = 0; i < pathParts.length - 1; i++) { + if (!current[pathParts[i]]) return; + current = current[pathParts[i]]; + } + + const lastKey = pathParts[pathParts.length - 1]; + if (current[lastKey] && typeof current[lastKey] === 'object' && !Array.isArray(current[lastKey]) && Object.keys(current[lastKey]).length === 0) { + delete current[lastKey]; + cleanupEmpty(obj, pathParts.slice(0, -1)); + } +}