From dfbb6e6ce1973016f5b22e13825515ca9fa84833 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:26:04 +0000 Subject: [PATCH 1/2] Initial plan From 48aa311d5155e8b23f3ca265b4b36e03c3054882 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:31:20 +0000 Subject: [PATCH 2/2] feat: Add WebView interaction MCP tools (maui_webview_*) Add 9 new MCP tools for WebView interaction: - maui_webview_navigate: Navigate WebView to a URL - maui_webview_click: Click element by CSS selector - maui_webview_fill: Fill text into input by CSS selector - maui_webview_type: Type text into focused element - maui_webview_console: Get WebView console logs - maui_webview_dom: Get WebView DOM tree - maui_webview_dom_query: Query DOM by CSS selector - maui_webview_network: Get WebView network requests - maui_webview_screenshot: Capture WebView screenshot Add 5 new AgentClient methods for endpoints that had no client methods: GetWebViewConsoleAsync, GetWebViewDomAsync, QueryWebViewDomAsync, GetWebViewNetworkAsync, GetWebViewScreenshotAsync Register WebViewTools in McpServerHost. Add unit tests for all new AgentClient methods. Agent-Logs-Url: https://github.com/dotnet/maui-labs/sessions/b3e00c21-cbea-4ef6-8c6c-b9daf9d91c2e Co-authored-by: rmarinho <1235097+rmarinho@users.noreply.github.com> --- .../DevFlow/Mcp/McpServerHost.cs | 1 + .../DevFlow/Mcp/Tools/WebViewTools.cs | 139 ++++++++++++++++++ .../AgentClient.cs | 84 +++++++++++ .../WebViewDriverTests.cs | 72 +++++++++ 4 files changed, 296 insertions(+) create mode 100644 src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/Tools/WebViewTools.cs diff --git a/src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/McpServerHost.cs b/src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/McpServerHost.cs index 3f9b3ea1b..1673b7eb1 100644 --- a/src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/McpServerHost.cs +++ b/src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/McpServerHost.cs @@ -32,6 +32,7 @@ public static async Task RunAsync() .WithTools() .WithTools() .WithTools() + .WithTools() .WithTools() .WithTools() .WithTools() diff --git a/src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/Tools/WebViewTools.cs b/src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/Tools/WebViewTools.cs new file mode 100644 index 000000000..6b90f9510 --- /dev/null +++ b/src/Cli/Microsoft.Maui.Cli/DevFlow/Mcp/Tools/WebViewTools.cs @@ -0,0 +1,139 @@ +using System.ComponentModel; +using ModelContextProtocol; +using ModelContextProtocol.Server; +using ModelContextProtocol.Protocol; +using Microsoft.Maui.Cli.DevFlow.Mcp; + +namespace Microsoft.Maui.Cli.DevFlow.Mcp.Tools; + +[McpServerToolType] +public sealed class WebViewTools +{ + [McpServerTool(Name = "maui_webview_navigate"), Description("Navigate a Blazor WebView to a URL. Use this for high-level WebView navigation instead of low-level CDP commands.")] + public static async Task WebViewNavigate( + McpAgentSession session, + [Description("URL to navigate the WebView to (e.g., '/counter', 'https://example.com')")] string url, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var success = await agent.NavigateWebViewAsync(url, contextId); + return success + ? $"WebView navigated to '{url}'." + : $"Failed to navigate WebView to '{url}'. Is a Blazor WebView active?"; + } + + [McpServerTool(Name = "maui_webview_click"), Description("Click an element in a Blazor WebView by CSS selector. Scrolls the element into view, focuses it, and clicks it.")] + public static async Task WebViewClick( + McpAgentSession session, + [Description("CSS selector of the element to click (e.g., 'button', '#submit-btn', '.nav-link')")] string selector, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var success = await agent.ClickWebViewAsync(selector, contextId); + return success + ? $"Clicked WebView element '{selector}'." + : $"Failed to click WebView element '{selector}'. Element may not exist or WebView is not active."; + } + + [McpServerTool(Name = "maui_webview_fill"), Description("Fill text into an input element in a Blazor WebView by CSS selector. Replaces existing value, focuses the element, and dispatches input/change events.")] + public static async Task WebViewFill( + McpAgentSession session, + [Description("CSS selector of the input element to fill (e.g., 'input#email', 'textarea.comment')")] string selector, + [Description("Text to fill into the element")] string text, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var success = await agent.FillWebViewAsync(selector, text, contextId); + return success + ? $"Filled WebView element '{selector}' with text." + : $"Failed to fill WebView element '{selector}'. Element may not exist or is not an input."; + } + + [McpServerTool(Name = "maui_webview_type"), Description("Type text into the currently focused element in a Blazor WebView using CDP Input.insertText. Use maui_webview_click to focus an element first.")] + public static async Task WebViewType( + McpAgentSession session, + [Description("Text to type into the focused element")] string text, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var success = await agent.InsertWebViewTextAsync(text, contextId); + return success + ? "Text typed into WebView." + : "Failed to type text into WebView. Is an element focused?"; + } + + [McpServerTool(Name = "maui_webview_console"), Description("Get console log messages from a Blazor WebView. Returns log entries filtered to the WebView source.")] + public static async Task WebViewConsole( + McpAgentSession session, + [Description("Maximum number of log entries to return (default: 100)")] int limit = 100, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var logs = await agent.GetWebViewConsoleAsync(limit, contextId); + return string.IsNullOrEmpty(logs) || logs == "[]" + ? "No WebView console messages captured." + : logs; + } + + [McpServerTool(Name = "maui_webview_dom"), Description("Get the DOM tree (HTML source) of a Blazor WebView. Returns the full outerHTML of the document element.")] + public static async Task WebViewDom( + McpAgentSession session, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var dom = await agent.GetWebViewDomAsync(contextId); + return string.IsNullOrEmpty(dom) + ? "No WebView DOM available. Is a Blazor WebView active?" + : dom; + } + + [McpServerTool(Name = "maui_webview_dom_query"), Description("Query the WebView DOM by CSS selector. Returns matching elements with their tag name, id, class, text content, and outer HTML.")] + public static async Task WebViewDomQuery( + McpAgentSession session, + [Description("CSS selector to query (e.g., 'button', '.nav-link', '#main-content h1')")] string selector, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var result = await agent.QueryWebViewDomAsync(selector, contextId); + return string.IsNullOrEmpty(result) || result == "[]" + ? $"No elements found matching '{selector}'." + : result; + } + + [McpServerTool(Name = "maui_webview_network"), Description("Get captured network requests from a Blazor WebView.")] + public static async Task WebViewNetwork( + McpAgentSession session, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var requests = await agent.GetWebViewNetworkAsync(contextId); + return string.IsNullOrEmpty(requests) || requests == "[]" + ? "No WebView network requests captured." + : requests; + } + + [McpServerTool(Name = "maui_webview_screenshot"), Description("Capture a screenshot of a Blazor WebView. Returns the image directly. This captures only the WebView content, not the native MAUI UI.")] + public static async Task WebViewScreenshot( + McpAgentSession session, + [Description("WebView context ID (optional if only one WebView)")] string? contextId = null, + [Description("Agent HTTP port (optional if only one agent connected)")] int? agentPort = null) + { + var agent = await session.GetAgentClientAsync(agentPort); + var bytes = await agent.GetWebViewScreenshotAsync(contextId); + if (bytes == null || bytes.Length == 0) + throw new McpException("Failed to capture WebView screenshot. Is a Blazor WebView active?"); + + return [ + new TextContentBlock { Text = $"WebView screenshot captured ({bytes.Length} bytes)" }, + ImageContentBlock.FromBytes(bytes, "image/png") + ]; + } +} diff --git a/src/DevFlow/Microsoft.Maui.DevFlow.Driver/AgentClient.cs b/src/DevFlow/Microsoft.Maui.DevFlow.Driver/AgentClient.cs index 5467b0735..46bde4ebb 100644 --- a/src/DevFlow/Microsoft.Maui.DevFlow.Driver/AgentClient.cs +++ b/src/DevFlow/Microsoft.Maui.DevFlow.Driver/AgentClient.cs @@ -386,6 +386,90 @@ public async Task InsertWebViewTextAsync(string text, string? contextId = return await PostActionAsync($"{WebViewApi}/input/text", payload); } + /// + /// Get WebView console logs (filtered by source=webview). + /// + public async Task GetWebViewConsoleAsync(int limit = 100, string? contextId = null) + { + var path = $"{WebViewApi}/console?limit={limit}"; + if (!string.IsNullOrWhiteSpace(contextId)) + path += $"&contextId={Uri.EscapeDataString(contextId)}"; + try + { + return await _http.GetStringAsync($"{_baseUrl}{path}"); + } + catch { return "[]"; } + } + + /// + /// Get the WebView DOM tree (HTML source). + /// + public async Task GetWebViewDomAsync(string? contextId = null) + { + var path = $"{WebViewApi}/dom"; + if (!string.IsNullOrWhiteSpace(contextId)) + path += $"?contextId={Uri.EscapeDataString(contextId)}"; + try + { + return await _http.GetStringAsync($"{_baseUrl}{path}"); + } + catch { return string.Empty; } + } + + /// + /// Query the WebView DOM by CSS selector. + /// + public async Task QueryWebViewDomAsync(string selector, string? contextId = null) + { + var payload = new JsonObject + { + ["selector"] = selector + }; + + if (!string.IsNullOrWhiteSpace(contextId)) + payload["contextId"] = contextId; + + try + { + using var content = DriverJson.CreateJsonContent(payload); + var response = await _http.PostAsync($"{_baseUrl}{WebViewApi}/dom/query", content); + return await response.Content.ReadAsStringAsync(); + } + catch { return "[]"; } + } + + /// + /// Get WebView network requests. + /// + public async Task GetWebViewNetworkAsync(string? contextId = null) + { + var path = $"{WebViewApi}/network"; + if (!string.IsNullOrWhiteSpace(contextId)) + path += $"?contextId={Uri.EscapeDataString(contextId)}"; + try + { + return await _http.GetStringAsync($"{_baseUrl}{path}"); + } + catch { return "[]"; } + } + + /// + /// Capture a WebView-specific screenshot. Returns PNG bytes. + /// + public async Task GetWebViewScreenshotAsync(string? contextId = null) + { + var path = $"{WebViewApi}/screenshot"; + if (!string.IsNullOrWhiteSpace(contextId)) + path += $"?contextId={Uri.EscapeDataString(contextId)}"; + try + { + var response = await _http.GetAsync($"{_baseUrl}{path}"); + if (!response.IsSuccessStatusCode) return null; + return await response.Content.ReadAsByteArrayAsync(); + } + catch { return null; } + } + public async Task HitTestAsync(double x, double y, int? window = null) { var path = $"{UiApi}/hit-test?x={x}&y={y}"; diff --git a/src/DevFlow/Microsoft.Maui.DevFlow.Tests/WebViewDriverTests.cs b/src/DevFlow/Microsoft.Maui.DevFlow.Tests/WebViewDriverTests.cs index 5223cbb67..9b87b4ddc 100644 --- a/src/DevFlow/Microsoft.Maui.DevFlow.Tests/WebViewDriverTests.cs +++ b/src/DevFlow/Microsoft.Maui.DevFlow.Tests/WebViewDriverTests.cs @@ -81,6 +81,78 @@ public async Task Focus_WhenAgentNotRunning_ReturnsFalse() var result = await client.FocusAsync("test-id"); Assert.False(result); } + + [Fact] + public async Task NavigateWebView_WhenAgentNotRunning_ReturnsFalse() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.NavigateWebViewAsync("https://example.com"); + Assert.False(result); + } + + [Fact] + public async Task ClickWebView_WhenAgentNotRunning_ReturnsFalse() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.ClickWebViewAsync("button"); + Assert.False(result); + } + + [Fact] + public async Task FillWebView_WhenAgentNotRunning_ReturnsFalse() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.FillWebViewAsync("input", "test text"); + Assert.False(result); + } + + [Fact] + public async Task InsertWebViewText_WhenAgentNotRunning_ReturnsFalse() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.InsertWebViewTextAsync("hello"); + Assert.False(result); + } + + [Fact] + public async Task GetWebViewConsole_WhenAgentNotRunning_ReturnsEmptyArray() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.GetWebViewConsoleAsync(); + Assert.Equal("[]", result); + } + + [Fact] + public async Task GetWebViewDom_WhenAgentNotRunning_ReturnsEmpty() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.GetWebViewDomAsync(); + Assert.Empty(result); + } + + [Fact] + public async Task QueryWebViewDom_WhenAgentNotRunning_ReturnsEmptyArray() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.QueryWebViewDomAsync("button"); + Assert.Equal("[]", result); + } + + [Fact] + public async Task GetWebViewNetwork_WhenAgentNotRunning_ReturnsEmptyArray() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.GetWebViewNetworkAsync(); + Assert.Equal("[]", result); + } + + [Fact] + public async Task GetWebViewScreenshot_WhenAgentNotRunning_ReturnsNull() + { + using var client = new AgentClient("localhost", 19999); + var result = await client.GetWebViewScreenshotAsync(); + Assert.Null(result); + } } public class AppDriverFactoryTests