From 911f36cfe6978315480193d261221e04d40afde9 Mon Sep 17 00:00:00 2001 From: nicetrykiddo Date: Fri, 10 Oct 2025 17:00:22 +0530 Subject: [PATCH 1/2] feat: Implement network log preservation features and related tools --- src/McpContext.ts | 28 +++++++- src/PageCollector.ts | 135 +++++++++++++++++++++++++++++++++--- src/tools/ToolDefinition.ts | 10 +++ src/tools/network.ts | 98 ++++++++++++++++++++++++++ 4 files changed, 261 insertions(+), 10 deletions(-) diff --git a/src/McpContext.ts b/src/McpContext.ts index d1037935..4a7d9aa9 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -19,7 +19,7 @@ import type { PredefinedNetworkConditions, } from 'puppeteer-core'; -import {NetworkCollector, PageCollector} from './PageCollector.js'; +import {NetworkCollector, PageCollector, PreservedNetworkRequest} from './PageCollector.js'; import {listPages} from './tools/pages.js'; import {takeSnapshot} from './tools/snapshot.js'; import {CLOSE_PAGE_ERROR} from './tools/ToolDefinition.js'; @@ -134,6 +134,32 @@ export class McpContext implements Context { return this.#networkCollector.getData(page); } + getPreservedNetworkRequests(): PreservedNetworkRequest[] { + const page = this.getSelectedPage(); + return this.#networkCollector.getPreservedData(page); + } + + enableNetworkLogPreservation(options?: { + includeRequestBodies?: boolean; + includeResponseBodies?: boolean; + maxRequests?: number; + }): void { + this.#networkCollector.enablePreservation(options); + } + + disableNetworkLogPreservation(): void { + this.#networkCollector.disablePreservation(); + } + + isNetworkLogPreservationEnabled(): boolean { + return this.#networkCollector.isPreservationEnabled(); + } + + clearPreservedNetworkLogs(): void { + const page = this.getSelectedPage(); + this.#networkCollector.clearPreservedData(page); + } + getConsoleData(): Array { const page = this.getSelectedPage(); return this.#consoleCollector.getData(page); diff --git a/src/PageCollector.ts b/src/PageCollector.ts index 9b078d55..7dd1a8b9 100644 --- a/src/PageCollector.ts +++ b/src/PageCollector.ts @@ -4,16 +4,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type {Browser, HTTPRequest, Page} from 'puppeteer-core'; +import type {Browser, HTTPRequest, Page, HTTPResponse} from 'puppeteer-core'; + +export interface PreservedNetworkRequest { + request: HTTPRequest; + timestamp: number; + requestBody?: string; + responseBody?: string; +} export class PageCollector { #browser: Browser; #initializer: (page: Page, collector: (item: T) => void) => void; - /** - * The Array in this map should only be set once - * As we use the reference to it. - * Use methods that manipulate the array in place. - */ protected storage = new WeakMap(); constructor( @@ -24,6 +26,10 @@ export class PageCollector { this.#initializer = initializer; } + protected getBrowser(): Browser { + return this.#browser; + } + async init() { const pages = await this.#browser.pages(); for (const page of pages) { @@ -77,7 +83,86 @@ export class PageCollector { } export class NetworkCollector extends PageCollector { + #preservationEnabled = false; + #includeRequestBodies = true; + #includeResponseBodies = true; + #maxRequests?: number; + #preservedData = new WeakMap(); + + enablePreservation(options?: { + includeRequestBodies?: boolean; + includeResponseBodies?: boolean; + maxRequests?: number; + }): void { + this.#preservationEnabled = true; + this.#includeRequestBodies = options?.includeRequestBodies ?? true; + this.#includeResponseBodies = options?.includeResponseBodies ?? true; + this.#maxRequests = options?.maxRequests; + } + + disablePreservation(): void { + this.#preservationEnabled = false; + } + + isPreservationEnabled(): boolean { + return this.#preservationEnabled; + } + + clearPreservedData(page: Page): void { + const preserved = this.#preservedData.get(page); + if (preserved) { + preserved.length = 0; + } + } + + getPreservedData(page: Page): PreservedNetworkRequest[] { + return this.#preservedData.get(page) ?? []; + } + + async #captureRequestData(request: HTTPRequest): Promise { + const preserved: PreservedNetworkRequest = { + request, + timestamp: Date.now(), + }; + + if (this.#includeRequestBodies) { + try { + const postData = request.postData(); + if (postData) { + preserved.requestBody = postData; + } + } catch (error) { + } + } + + if (this.#includeResponseBodies) { + try { + const response = request.response(); + if (response) { + const buffer = await response.buffer(); + const contentType = response.headers()['content-type'] || ''; + + if (contentType.includes('text/') || + contentType.includes('application/json') || + contentType.includes('application/xml') || + contentType.includes('application/javascript')) { + preserved.responseBody = buffer.toString('utf-8'); + } else { + preserved.responseBody = `[Binary data: ${contentType}, ${buffer.length} bytes]`; + } + } + } catch (error) { + } + } + + return preserved; + } + override cleanup(page: Page) { + if (this.#preservationEnabled) { + return; + } + const requests = this.storage.get(page) ?? []; if (!requests) { return; @@ -87,9 +172,41 @@ export class NetworkCollector extends PageCollector { ? request.isNavigationRequest() : false; }); - // Keep all requests since the last navigation request including that - // navigation request itself. - // Keep the reference requests.splice(0, Math.max(lastRequestIdx, 0)); } + + public override addPage(page: Page): void { + super.addPage(page); + + if (this.#preservationEnabled) { + if (!this.#preservedData.has(page)) { + this.#preservedData.set(page, []); + } + + page.on('requestfinished', async (request: HTTPRequest) => { + const preserved = this.#preservedData.get(page); + if (!preserved) return; + + const data = await this.#captureRequestData(request); + preserved.push(data); + + if (this.#maxRequests && preserved.length > this.#maxRequests) { + preserved.shift(); + } + }); + } + } + + override async init() { + await super.init(); + + if (this.#preservationEnabled) { + const pages = await this.getBrowser().pages(); + for (const page of pages) { + if (!this.#preservedData.has(page)) { + this.#preservedData.set(page, []); + } + } + } + } } diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.ts index fe2fae7b..8dd5cc5c 100644 --- a/src/tools/ToolDefinition.ts +++ b/src/tools/ToolDefinition.ts @@ -7,6 +7,7 @@ import type {Dialog, ElementHandle, Page} from 'puppeteer-core'; import z from 'zod'; +import type {PreservedNetworkRequest} from '../PageCollector.js'; import type {TraceResult} from '../trace-processing/parse.js'; import type {ToolCategories} from './categories.js'; @@ -79,6 +80,15 @@ export type Context = Readonly<{ filename: string, ): Promise<{filename: string}>; waitForEventsAfterAction(action: () => Promise): Promise; + enableNetworkLogPreservation(options?: { + includeRequestBodies?: boolean; + includeResponseBodies?: boolean; + maxRequests?: number; + }): void; + disableNetworkLogPreservation(): void; + isNetworkLogPreservationEnabled(): boolean; + clearPreservedNetworkLogs(): void; + getPreservedNetworkRequests(): PreservedNetworkRequest[]; }>; export function defineTool( diff --git a/src/tools/network.ts b/src/tools/network.ts index 5943b0f5..8e7906a4 100644 --- a/src/tools/network.ts +++ b/src/tools/network.ts @@ -86,3 +86,101 @@ export const getNetworkRequest = defineTool({ response.attachNetworkRequest(request.params.url); }, }); + +export const enableNetworkLogPreservation = defineTool({ + name: 'enable_network_log_preservation', + description: `Enable network log preservation mode. When enabled, all network requests are preserved across page navigations and response bodies are automatically captured. By default, logs are cleaned on navigation for performance.`, + annotations: { + category: ToolCategories.NETWORK, + readOnlyHint: false, + }, + schema: { + includeRequestBodies: z + .boolean() + .optional() + .default(true) + .describe('Whether to capture and cache request bodies. Default: true'), + includeResponseBodies: z + .boolean() + .optional() + .default(true) + .describe('Whether to capture and cache response bodies. Default: true'), + maxRequests: z + .number() + .int() + .positive() + .optional() + .describe( + 'Maximum number of requests to preserve. Older requests are automatically removed when limit is reached. When omitted, no limit is applied.', + ), + }, + handler: async (request, response, context) => { + context.enableNetworkLogPreservation({ + includeRequestBodies: request.params.includeRequestBodies, + includeResponseBodies: request.params.includeResponseBodies, + maxRequests: request.params.maxRequests, + }); + response.appendResponseLine( + 'Network log preservation enabled. All network requests will be preserved across navigations.', + ); + if (request.params.includeRequestBodies) { + response.appendResponseLine('Request bodies will be captured.'); + } + if (request.params.includeResponseBodies) { + response.appendResponseLine('Response bodies will be captured.'); + } + if (request.params.maxRequests) { + response.appendResponseLine( + `Maximum ${request.params.maxRequests} requests will be preserved.`, + ); + } + }, +}); + +export const disableNetworkLogPreservation = defineTool({ + name: 'disable_network_log_preservation', + description: `Disable network log preservation mode and optionally clear existing preserved logs. After disabling, network logs will be cleaned on navigation (default behavior).`, + annotations: { + category: ToolCategories.NETWORK, + readOnlyHint: false, + }, + schema: { + clearExisting: z + .boolean() + .optional() + .default(true) + .describe( + 'Whether to clear existing preserved logs. Default: true', + ), + }, + handler: async (request, response, context) => { + context.disableNetworkLogPreservation(); + if (request.params.clearExisting) { + context.clearPreservedNetworkLogs(); + response.appendResponseLine( + 'Network log preservation disabled and existing logs cleared.', + ); + } else { + response.appendResponseLine( + 'Network log preservation disabled. Existing logs retained.', + ); + } + }, +}); + +export const clearPreservedNetworkLogs = defineTool({ + name: 'clear_preserved_network_logs', + description: `Clear all preserved network logs for the currently selected page. This does not disable preservation mode.`, + annotations: { + category: ToolCategories.NETWORK, + readOnlyHint: false, + }, + schema: {}, + handler: async (_request, response, context) => { + const preservedCount = context.getPreservedNetworkRequests().length; + context.clearPreservedNetworkLogs(); + response.appendResponseLine( + `Cleared ${preservedCount} preserved network request(s).`, + ); + }, +}); From 65c1635fac0e0baae359f0cc0a3312e5a26f68a8 Mon Sep 17 00:00:00 2001 From: nicetrykiddo Date: Fri, 10 Oct 2025 17:06:41 +0530 Subject: [PATCH 2/2] feat: Enhance network log preservation features with improved messaging and descriptions for AI Agents --- src/McpResponse.ts | 9 ++++++- src/tools/network.ts | 64 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/src/McpResponse.ts b/src/McpResponse.ts index bf7603bf..96a69e58 100644 --- a/src/McpResponse.ts +++ b/src/McpResponse.ts @@ -223,7 +223,6 @@ Call ${handleDialog.name} to handle it before continuing.`); if (this.#networkRequestsOptions?.include) { let requests = context.getNetworkRequests(); - // Apply resource type filtering if specified if (this.#networkRequestsOptions.resourceTypes?.length) { const normalizedTypes = new Set( this.#networkRequestsOptions.resourceTypes, @@ -234,7 +233,15 @@ Call ${handleDialog.name} to handle it before continuing.`); }); } + const preservationEnabled = context.isNetworkLogPreservationEnabled(); + response.push('## Network requests'); + if (preservationEnabled) { + response.push('šŸ”’ **Preservation Mode: ACTIVE** (requests persist across navigations)'); + } else { + response.push('šŸ“‹ Normal mode (logs cleared on navigation). Enable preservation to keep history.'); + } + if (requests.length) { const data = this.#dataWithPagination( requests, diff --git a/src/tools/network.ts b/src/tools/network.ts index 8e7906a4..094950ea 100644 --- a/src/tools/network.ts +++ b/src/tools/network.ts @@ -34,7 +34,7 @@ const FILTERABLE_RESOURCE_TYPES: readonly [ResourceType, ...ResourceType[]] = [ export const listNetworkRequests = defineTool({ name: 'list_network_requests', - description: `List all requests for the currently selected page`, + description: `List all requests for the currently selected page. By default, only shows requests since the last navigation. To preserve requests across navigations, first call enable_network_log_preservation before navigating.`, annotations: { category: ToolCategories.NETWORK, readOnlyHint: true, @@ -89,7 +89,7 @@ export const getNetworkRequest = defineTool({ export const enableNetworkLogPreservation = defineTool({ name: 'enable_network_log_preservation', - description: `Enable network log preservation mode. When enabled, all network requests are preserved across page navigations and response bodies are automatically captured. By default, logs are cleaned on navigation for performance.`, + description: `Enable network log preservation mode to keep ALL network requests across navigations. IMPORTANT: Call this BEFORE navigating or interacting with the page if you want to analyze request patterns across multiple actions. When enabled, all request/response bodies are automatically captured and cached for later analysis. Use this when you need to compare requests before/after certain actions or track API calls across page transitions.`, annotations: { category: ToolCategories.NETWORK, readOnlyHint: false, @@ -115,31 +115,44 @@ export const enableNetworkLogPreservation = defineTool({ ), }, handler: async (request, response, context) => { + const wasAlreadyEnabled = context.isNetworkLogPreservationEnabled(); + context.enableNetworkLogPreservation({ includeRequestBodies: request.params.includeRequestBodies, includeResponseBodies: request.params.includeResponseBodies, maxRequests: request.params.maxRequests, }); - response.appendResponseLine( - 'Network log preservation enabled. All network requests will be preserved across navigations.', - ); + + if (wasAlreadyEnabled) { + response.appendResponseLine( + 'āš ļø Network log preservation was already enabled. Settings updated.', + ); + } else { + response.appendResponseLine( + 'āœ… Network log preservation enabled. All network requests will be preserved across navigations.', + ); + } + if (request.params.includeRequestBodies) { - response.appendResponseLine('Request bodies will be captured.'); + response.appendResponseLine('šŸ“¤ Request bodies will be captured.'); } if (request.params.includeResponseBodies) { - response.appendResponseLine('Response bodies will be captured.'); + response.appendResponseLine('šŸ“„ Response bodies will be captured.'); } if (request.params.maxRequests) { response.appendResponseLine( - `Maximum ${request.params.maxRequests} requests will be preserved.`, + `šŸ”¢ Maximum ${request.params.maxRequests} requests will be preserved.`, ); } + response.appendResponseLine( + '\nšŸ’” TIP: Preservation is now active. Navigate, click buttons, or interact with the page - all network activity will be recorded.', + ); }, }); export const disableNetworkLogPreservation = defineTool({ name: 'disable_network_log_preservation', - description: `Disable network log preservation mode and optionally clear existing preserved logs. After disabling, network logs will be cleaned on navigation (default behavior).`, + description: `Disable network log preservation mode and optionally clear existing preserved logs. After disabling, network logs will be cleaned on navigation (default behavior). Call this when you're done analyzing preserved requests to restore normal performance.`, annotations: { category: ToolCategories.NETWORK, readOnlyHint: false, @@ -154,33 +167,58 @@ export const disableNetworkLogPreservation = defineTool({ ), }, handler: async (request, response, context) => { + const wasEnabled = context.isNetworkLogPreservationEnabled(); + + if (!wasEnabled) { + response.appendResponseLine( + 'āš ļø Network log preservation was not enabled. No action taken.', + ); + return; + } + context.disableNetworkLogPreservation(); if (request.params.clearExisting) { context.clearPreservedNetworkLogs(); response.appendResponseLine( - 'Network log preservation disabled and existing logs cleared.', + 'āœ… Network log preservation disabled and existing logs cleared.', ); } else { response.appendResponseLine( - 'Network log preservation disabled. Existing logs retained.', + 'āœ… Network log preservation disabled. Existing logs retained.', ); } + response.appendResponseLine( + 'šŸ’” Normal behavior restored: network logs will be cleared on navigation.', + ); }, }); export const clearPreservedNetworkLogs = defineTool({ name: 'clear_preserved_network_logs', - description: `Clear all preserved network logs for the currently selected page. This does not disable preservation mode.`, + description: `Clear all preserved network logs for the currently selected page without disabling preservation mode. Use this to reset the preserved request history while keeping preservation active for future requests.`, annotations: { category: ToolCategories.NETWORK, readOnlyHint: false, }, schema: {}, handler: async (_request, response, context) => { + if (!context.isNetworkLogPreservationEnabled()) { + response.appendResponseLine( + 'āš ļø Network log preservation is not enabled. No preserved logs to clear.', + ); + response.appendResponseLine( + 'šŸ’” TIP: Call enable_network_log_preservation first to start preserving logs.', + ); + return; + } + const preservedCount = context.getPreservedNetworkRequests().length; context.clearPreservedNetworkLogs(); response.appendResponseLine( - `Cleared ${preservedCount} preserved network request(s).`, + `āœ… Cleared ${preservedCount} preserved network request(s).`, + ); + response.appendResponseLine( + 'šŸ’” Preservation mode is still active. New requests will continue to be preserved.', ); }, });