From 69e9155172af5660f74db42463a89fa3aec2368f Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 15:42:28 -0500 Subject: [PATCH 01/12] feat: make CLI AI-agent friendly (Issue #28, Phases 1-2) - Add central OutputWriter abstraction for consistent JSON/human output - Add global --json and --no-json flags (available on all commands) - TTY auto-detection: default to JSON when stdout is piped/redirected - Support MAUIDEVFLOW_OUTPUT=json environment variable override - Structured error output on stderr with error type, retryable flag, and suggestions - Add JSON output to all MAUI commands: status, tree, query, element, hittest, tap, fill, clear, focus, navigate, scroll, resize, property, set-property, screenshot, alert detect/dismiss/tree - Add JSON output to: list, broker status, network detail/clear, logs - Migrate existing per-command --json options (logs, network, wait, webviews) to use the global --json flag - Add --fields option to tree and query for client-side field projection (e.g. --fields "id,type,text,automationId") - Add --format compact option for tree/query (minimal field set) - Add --wait-until exists|gone with --timeout to query command (eliminates agent polling loops, saves tokens) - Error types distinguish InvocationError vs RuntimeError Closes #28 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/MauiDevFlow.CLI/OutputWriter.cs | 173 ++++++++ src/MauiDevFlow.CLI/Program.cs | 633 +++++++++++++++++++--------- 2 files changed, 607 insertions(+), 199 deletions(-) create mode 100644 src/MauiDevFlow.CLI/OutputWriter.cs diff --git a/src/MauiDevFlow.CLI/OutputWriter.cs b/src/MauiDevFlow.CLI/OutputWriter.cs new file mode 100644 index 0000000..b392e18 --- /dev/null +++ b/src/MauiDevFlow.CLI/OutputWriter.cs @@ -0,0 +1,173 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace MauiDevFlow.CLI; + +/// +/// Central output abstraction for consistent JSON/human output across all CLI commands. +/// JSON mode: raw data on stdout, structured errors on stderr. +/// Human mode: formatted text on stdout, plain errors on stderr. +/// +static class OutputWriter +{ + private static readonly JsonSerializerOptions s_jsonOptions = new() + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }; + + private static readonly JsonSerializerOptions s_compactJsonOptions = new() + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }; + + /// + /// Resolves whether JSON output mode is active. + /// Priority: --no-json flag > --json flag > MAUIDEVFLOW_OUTPUT env var > TTY auto-detection. + /// + public static bool ResolveJsonMode(bool jsonFlag, bool noJsonFlag) + { + if (noJsonFlag) return false; + if (jsonFlag) return true; + + var envVar = Environment.GetEnvironmentVariable("MAUIDEVFLOW_OUTPUT"); + if (string.Equals(envVar, "json", StringComparison.OrdinalIgnoreCase)) + return true; + + if (Console.IsOutputRedirected) + return true; + + return false; + } + + /// + /// Write a successful result to stdout. + /// In JSON mode, serializes the data. In human mode, calls the humanFormatter. + /// + public static void WriteResult(T data, bool json, Action? humanFormatter = null) + { + if (json) + { + Console.WriteLine(JsonSerializer.Serialize(data, s_jsonOptions)); + } + else if (humanFormatter != null) + { + humanFormatter(data); + } + else + { + Console.WriteLine(JsonSerializer.Serialize(data, s_jsonOptions)); + } + } + + /// + /// Write raw JSON string to stdout (for data already serialized or from HTTP responses). + /// + public static void WriteRawJson(string jsonString) + { + Console.WriteLine(jsonString); + } + + /// + /// Write a JsonElement to stdout with indentation. + /// + public static void WriteJsonElement(JsonElement element, bool json) + { + if (json) + { + Console.WriteLine(element.GetRawText()); + } + else + { + Console.WriteLine(JsonSerializer.Serialize(element, s_jsonOptions)); + } + } + + /// + /// Write a simple success action result (for tap, fill, clear, etc.). + /// + public static void WriteActionResult(bool success, string action, string? elementId, bool json, string? humanMessage = null) + { + if (json) + { + var result = new ActionResult { Success = success, Action = action, ElementId = elementId }; + Console.WriteLine(JsonSerializer.Serialize(result, s_compactJsonOptions)); + } + else + { + Console.WriteLine(humanMessage ?? (success ? $"{action}: {elementId}" : $"Failed to {action.ToLowerInvariant()}: {elementId}")); + } + } + + /// + /// Write a structured error to stderr and set error state. + /// In JSON mode, outputs structured error JSON. In human mode, plain text. + /// + public static void WriteError(string message, bool json, string errorType = "RuntimeError", + bool retryable = false, string[]? suggestions = null) + { + if (json) + { + var error = new ErrorResult + { + Error = message, + Type = errorType, + Retryable = retryable, + Suggestions = suggestions + }; + Console.Error.WriteLine(JsonSerializer.Serialize(error, s_compactJsonOptions)); + } + else + { + Console.Error.WriteLine($"Error: {message}"); + } + } + + /// + /// Write a single JSONL line (for streaming commands). + /// + public static void WriteJsonLine(T data) + { + Console.WriteLine(JsonSerializer.Serialize(data, s_compactJsonOptions)); + } + + /// + /// Serialize an object to indented JSON string. + /// + public static string FormatJson(T data) + { + return JsonSerializer.Serialize(data, s_jsonOptions); + } + + // DTOs for structured output + + private class ActionResult + { + [JsonPropertyName("success")] + public bool Success { get; set; } + + [JsonPropertyName("action")] + public string? Action { get; set; } + + [JsonPropertyName("elementId")] + public string? ElementId { get; set; } + } + + private class ErrorResult + { + [JsonPropertyName("error")] + public string? Error { get; set; } + + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("retryable")] + public bool Retryable { get; set; } + + [JsonPropertyName("suggestions")] + public string[]? Suggestions { get; set; } + } +} diff --git a/src/MauiDevFlow.CLI/Program.cs b/src/MauiDevFlow.CLI/Program.cs index 2112386..5135ed4 100644 --- a/src/MauiDevFlow.CLI/Program.cs +++ b/src/MauiDevFlow.CLI/Program.cs @@ -34,10 +34,20 @@ static async Task Main(string[] args) ["--platform", "-p"], () => "maccatalyst", "Target platform (maccatalyst, android, ios, windows)"); + var jsonOption = new Option( + "--json", + () => false, + "Output as JSON (auto-enabled when stdout is piped/redirected)"); + var noJsonOption = new Option( + "--no-json", + () => false, + "Force human-readable output even when piped"); rootCommand.AddGlobalOption(agentPortOption); rootCommand.AddGlobalOption(agentHostOption); rootCommand.AddGlobalOption(platformOption); + rootCommand.AddGlobalOption(jsonOption); + rootCommand.AddGlobalOption(noJsonOption); // ===== CDP commands (Blazor WebView) ===== @@ -141,9 +151,7 @@ static async Task Main(string[] args) cdpCommand.Add(snapshotCmd); var webviewsCmd = new Command("webviews", "List available CDP WebViews"); - var webviewsJsonOption = new Option("--json", () => false, "Output as JSON"); - webviewsCmd.Add(webviewsJsonOption); - webviewsCmd.SetHandler(async (host, port, json) => await CdpWebViewsAsync(host, port, json), agentHostOption, agentPortOption, webviewsJsonOption); + webviewsCmd.SetHandler(async (host, port, json, noJson) => await CdpWebViewsAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson)), agentHostOption, agentPortOption, jsonOption, noJsonOption); cdpCommand.Add(webviewsCmd); var sourceCmd = new Command("source", "Get page HTML source from a WebView"); @@ -161,13 +169,15 @@ static async Task Main(string[] args) // MAUI status var mauiStatusCmd = new Command("status", "Check agent connection") { windowOption }; - mauiStatusCmd.SetHandler(async (host, port, window) => await MauiStatusAsync(host, port, window), agentHostOption, agentPortOption, windowOption); + mauiStatusCmd.SetHandler(async (host, port, json, noJson, window) => await MauiStatusAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), window), agentHostOption, agentPortOption, jsonOption, noJsonOption, windowOption); mauiCommand.Add(mauiStatusCmd); // MAUI tree var treeDepthOption = new Option("--depth", () => 0, "Max tree depth (0=unlimited)"); - var mauiTreeCmd = new Command("tree", "Dump visual tree") { treeDepthOption, windowOption }; - mauiTreeCmd.SetHandler(async (host, port, depth, window) => await MauiTreeAsync(host, port, depth, window), agentHostOption, agentPortOption, treeDepthOption, windowOption); + var treeFieldsOption = new Option("--fields", "Comma-separated fields to include (e.g. id,type,text,automationId,bounds)"); + var treeFormatOption = new Option("--format", "Output format: compact (id,type,text,automationId,bounds only)"); + var mauiTreeCmd = new Command("tree", "Dump visual tree") { treeDepthOption, treeFieldsOption, treeFormatOption, windowOption }; + mauiTreeCmd.SetHandler(async (host, port, json, noJson, depth, window, fields, format) => await MauiTreeAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), depth, window, fields, format), agentHostOption, agentPortOption, jsonOption, noJsonOption, treeDepthOption, windowOption, treeFieldsOption, treeFormatOption); mauiCommand.Add(mauiTreeCmd); // MAUI query @@ -175,37 +185,55 @@ static async Task Main(string[] args) var queryAutoIdOption = new Option("--automationId", "Filter by AutomationId"); var queryTextOption = new Option("--text", "Filter by text content"); var querySelectorOption = new Option("--selector", "CSS selector (e.g. 'Button:visible', 'StackLayout > Label[Text^=\"Hello\"]')"); - var mauiQueryCmd = new Command("query", "Find elements") { queryTypeOption, queryAutoIdOption, queryTextOption, querySelectorOption }; - mauiQueryCmd.SetHandler(async (host, port, type, autoId, text, selector) => - await MauiQueryAsync(host, port, type, autoId, text, selector), - agentHostOption, agentPortOption, queryTypeOption, queryAutoIdOption, queryTextOption, querySelectorOption); + var queryFieldsOption = new Option("--fields", "Comma-separated fields to include (e.g. id,type,text,automationId,bounds)"); + var queryFormatOption = new Option("--format", "Output format: compact (id,type,text,automationId,bounds only)"); + var queryWaitUntilOption = new Option("--wait-until", "Wait condition: exists or gone"); + var queryTimeoutOption = new Option("--timeout", () => 10, "Timeout in seconds for --wait-until"); + var mauiQueryCmd = new Command("query", "Find elements") { queryTypeOption, queryAutoIdOption, queryTextOption, querySelectorOption, queryFieldsOption, queryFormatOption, queryWaitUntilOption, queryTimeoutOption }; + mauiQueryCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode( + ctx.ParseResult.GetValueForOption(jsonOption), + ctx.ParseResult.GetValueForOption(noJsonOption)); + var type = ctx.ParseResult.GetValueForOption(queryTypeOption); + var autoId = ctx.ParseResult.GetValueForOption(queryAutoIdOption); + var text = ctx.ParseResult.GetValueForOption(queryTextOption); + var selector = ctx.ParseResult.GetValueForOption(querySelectorOption); + var fields = ctx.ParseResult.GetValueForOption(queryFieldsOption); + var format = ctx.ParseResult.GetValueForOption(queryFormatOption); + var waitUntil = ctx.ParseResult.GetValueForOption(queryWaitUntilOption); + var timeout = ctx.ParseResult.GetValueForOption(queryTimeoutOption); + await MauiQueryAsync(host, port, isJson, type, autoId, text, selector, fields, format, waitUntil, timeout); + }); mauiCommand.Add(mauiQueryCmd); // MAUI hittest var hitTestXArg = new Argument("x", "X coordinate"); var hitTestYArg = new Argument("y", "Y coordinate"); var mauiHitTestCmd = new Command("hittest", "Find elements at a point") { hitTestXArg, hitTestYArg, windowOption }; - mauiHitTestCmd.SetHandler(async (host, port, x, y, window) => await MauiHitTestAsync(host, port, x, y, window), - agentHostOption, agentPortOption, hitTestXArg, hitTestYArg, windowOption); + mauiHitTestCmd.SetHandler(async (host, port, json, noJson, x, y, window) => await MauiHitTestAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), x, y, window), + agentHostOption, agentPortOption, jsonOption, noJsonOption, hitTestXArg, hitTestYArg, windowOption); mauiCommand.Add(mauiHitTestCmd); // MAUI tap var tapIdArg = new Argument("elementId", "Element ID to tap"); var mauiTapCmd = new Command("tap", "Tap element") { tapIdArg }; - mauiTapCmd.SetHandler(async (host, port, id) => await MauiTapAsync(host, port, id), agentHostOption, agentPortOption, tapIdArg); + mauiTapCmd.SetHandler(async (host, port, json, noJson, id) => await MauiTapAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), agentHostOption, agentPortOption, jsonOption, noJsonOption, tapIdArg); mauiCommand.Add(mauiTapCmd); // MAUI fill var fillIdArg = new Argument("elementId", "Element ID"); var fillTextArg2 = new Argument("text", "Text to fill"); var mauiFillCmd = new Command("fill", "Fill text into element") { fillIdArg, fillTextArg2 }; - mauiFillCmd.SetHandler(async (host, port, id, text) => await MauiFillAsync(host, port, id, text), agentHostOption, agentPortOption, fillIdArg, fillTextArg2); + mauiFillCmd.SetHandler(async (host, port, json, noJson, id, text) => await MauiFillAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id, text), agentHostOption, agentPortOption, jsonOption, noJsonOption, fillIdArg, fillTextArg2); mauiCommand.Add(mauiFillCmd); // MAUI clear var clearIdArg = new Argument("elementId", "Element ID to clear"); var mauiClearCmd = new Command("clear", "Clear text from element") { clearIdArg }; - mauiClearCmd.SetHandler(async (host, port, id) => await MauiClearAsync(host, port, id), agentHostOption, agentPortOption, clearIdArg); + mauiClearCmd.SetHandler(async (host, port, json, noJson, id) => await MauiClearAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), agentHostOption, agentPortOption, jsonOption, noJsonOption, clearIdArg); mauiCommand.Add(mauiClearCmd); // MAUI screenshot @@ -213,7 +241,7 @@ await MauiQueryAsync(host, port, type, autoId, text, selector), var screenshotIdOption = new Option("--id", "Element ID to capture"); var screenshotSelectorOption = new Option("--selector", "CSS selector to capture (first match)"); var mauiScreenshotCmd = new Command("screenshot", "Take screenshot") { screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption }; - mauiScreenshotCmd.SetHandler(async (host, port, output, window, id, selector) => await MauiScreenshotAsync(host, port, output, window, id, selector), agentHostOption, agentPortOption, screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption); + mauiScreenshotCmd.SetHandler(async (host, port, json, noJson, output, window, id, selector) => await MauiScreenshotAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), output, window, id, selector), agentHostOption, agentPortOption, jsonOption, noJsonOption, screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption); mauiCommand.Add(mauiScreenshotCmd); // MAUI recording subcommands @@ -243,7 +271,7 @@ await RecordingStopAsync(host, port, platform), var propIdArg = new Argument("elementId", "Element ID"); var propNameArg = new Argument("propertyName", "Property name"); var mauiPropertyCmd = new Command("property", "Get element property") { propIdArg, propNameArg }; - mauiPropertyCmd.SetHandler(async (host, port, id, name) => await MauiPropertyAsync(host, port, id, name), agentHostOption, agentPortOption, propIdArg, propNameArg); + mauiPropertyCmd.SetHandler(async (host, port, json, noJson, id, name) => await MauiPropertyAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id, name), agentHostOption, agentPortOption, jsonOption, noJsonOption, propIdArg, propNameArg); mauiCommand.Add(mauiPropertyCmd); // MAUI set-property @@ -251,19 +279,19 @@ await RecordingStopAsync(host, port, platform), var setPropNameArg = new Argument("propertyName", "Property name"); var setPropValueArg = new Argument("value", "Value to set"); var mauiSetPropertyCmd = new Command("set-property", "Set element property (live editing)") { setPropIdArg, setPropNameArg, setPropValueArg }; - mauiSetPropertyCmd.SetHandler(async (host, port, id, name, value) => await MauiSetPropertyAsync(host, port, id, name, value), agentHostOption, agentPortOption, setPropIdArg, setPropNameArg, setPropValueArg); + mauiSetPropertyCmd.SetHandler(async (host, port, json, noJson, id, name, value) => await MauiSetPropertyAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id, name, value), agentHostOption, agentPortOption, jsonOption, noJsonOption, setPropIdArg, setPropNameArg, setPropValueArg); mauiCommand.Add(mauiSetPropertyCmd); // MAUI element var elementIdArg = new Argument("elementId", "Element ID"); var mauiElementCmd = new Command("element", "Get element details") { elementIdArg }; - mauiElementCmd.SetHandler(async (host, port, id) => await MauiElementAsync(host, port, id), agentHostOption, agentPortOption, elementIdArg); + mauiElementCmd.SetHandler(async (host, port, json, noJson, id) => await MauiElementAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), agentHostOption, agentPortOption, jsonOption, noJsonOption, elementIdArg); mauiCommand.Add(mauiElementCmd); // MAUI navigate (Shell) var navRouteArg = new Argument("route", "Shell route (e.g. //blazor)"); var mauiNavigateCmd = new Command("navigate", "Navigate to Shell route") { navRouteArg }; - mauiNavigateCmd.SetHandler(async (host, port, route) => await MauiNavigateAsync(host, port, route), agentHostOption, agentPortOption, navRouteArg); + mauiNavigateCmd.SetHandler(async (host, port, json, noJson, route) => await MauiNavigateAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), route), agentHostOption, agentPortOption, jsonOption, noJsonOption, navRouteArg); mauiCommand.Add(mauiNavigateCmd); // MAUI scroll @@ -272,22 +300,31 @@ await RecordingStopAsync(host, port, platform), var scrollDeltaYOption = new Option("--dy", () => 0, "Vertical scroll delta"); var scrollAnimatedOption = new Option("--animated", () => true, "Animate the scroll"); var mauiScrollCmd = new Command("scroll", "Scroll content by delta or scroll element into view") { scrollElementIdOption, scrollDeltaXOption, scrollDeltaYOption, scrollAnimatedOption, windowOption }; - mauiScrollCmd.SetHandler(async (host, port, elementId, dx, dy, animated, window) => - await MauiScrollAsync(host, port, elementId, dx, dy, animated, window), - agentHostOption, agentPortOption, scrollElementIdOption, scrollDeltaXOption, scrollDeltaYOption, scrollAnimatedOption, windowOption); + mauiScrollCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + await MauiScrollAsync(host, port, isJson, + ctx.ParseResult.GetValueForOption(scrollElementIdOption), + ctx.ParseResult.GetValueForOption(scrollDeltaXOption), + ctx.ParseResult.GetValueForOption(scrollDeltaYOption), + ctx.ParseResult.GetValueForOption(scrollAnimatedOption), + ctx.ParseResult.GetValueForOption(windowOption)); + }); mauiCommand.Add(mauiScrollCmd); // MAUI focus var focusIdArg = new Argument("elementId", "Element ID to focus"); var mauiFocusCmd = new Command("focus", "Set focus to element") { focusIdArg }; - mauiFocusCmd.SetHandler(async (host, port, id) => await MauiFocusAsync(host, port, id), agentHostOption, agentPortOption, focusIdArg); + mauiFocusCmd.SetHandler(async (host, port, json, noJson, id) => await MauiFocusAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), agentHostOption, agentPortOption, jsonOption, noJsonOption, focusIdArg); mauiCommand.Add(mauiFocusCmd); // MAUI resize var resizeWidthArg = new Argument("width", "Window width"); var resizeHeightArg = new Argument("height", "Window height"); var mauiResizeCmd = new Command("resize", "Resize app window") { resizeWidthArg, resizeHeightArg, windowOption }; - mauiResizeCmd.SetHandler(async (host, port, w, h, window) => await MauiResizeAsync(host, port, w, h, window), agentHostOption, agentPortOption, resizeWidthArg, resizeHeightArg, windowOption); + mauiResizeCmd.SetHandler(async (host, port, json, noJson, w, h, window) => await MauiResizeAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), w, h, window), agentHostOption, agentPortOption, jsonOption, noJsonOption, resizeWidthArg, resizeHeightArg, windowOption); mauiCommand.Add(mauiResizeCmd); // MAUI alert subcommands — supports iOS simulator (apple CLI) and Mac Catalyst (macOS AX API) @@ -300,8 +337,8 @@ await MauiScrollAsync(host, port, elementId, dx, dy, animated, window), var detectHost = new Option("--agent-host", () => "localhost", "Agent HTTP host"); var detectPort = new Option("--agent-port", () => 9223, "Agent HTTP port"); var alertDetectCmd = new Command("detect", "Check if an alert/dialog is visible") { detectUdid, detectPid, detectPlatform, detectHost, detectPort }; - alertDetectCmd.SetHandler(async (udid, pid, platform, host, port) => - await AlertDetectAsync(udid, pid, platform, host, port), detectUdid, detectPid, detectPlatform, detectHost, detectPort); + alertDetectCmd.SetHandler(async (udid, pid, platform, host, port, json, noJson) => + await AlertDetectAsync(udid, pid, platform, host, port, OutputWriter.ResolveJsonMode(json, noJson)), detectUdid, detectPid, detectPlatform, detectHost, detectPort, jsonOption, noJsonOption); alertCommand.Add(alertDetectCmd); // dismiss @@ -312,8 +349,8 @@ await MauiScrollAsync(host, port, elementId, dx, dy, animated, window), var dismissPort = new Option("--agent-port", () => 9223, "Agent HTTP port"); var dismissButtonArg = new Argument("button", () => null, "Button label to tap (default: first accept-style button)"); var alertDismissCmd = new Command("dismiss", "Dismiss the current alert/dialog") { dismissButtonArg, dismissUdid, dismissPid, dismissPlatform, dismissHost, dismissPort }; - alertDismissCmd.SetHandler(async (udid, pid, platform, host, port, button) => - await AlertDismissAsync(udid, pid, platform, host, port, button), dismissUdid, dismissPid, dismissPlatform, dismissHost, dismissPort, dismissButtonArg); + alertDismissCmd.SetHandler(async (udid, pid, platform, host, port, button, json, noJson) => + await AlertDismissAsync(udid, pid, platform, host, port, button, OutputWriter.ResolveJsonMode(json, noJson)), dismissUdid, dismissPid, dismissPlatform, dismissHost, dismissPort, dismissButtonArg, jsonOption, noJsonOption); alertCommand.Add(alertDismissCmd); // tree @@ -323,8 +360,8 @@ await MauiScrollAsync(host, port, elementId, dx, dy, animated, window), var treeHost = new Option("--agent-host", () => "localhost", "Agent HTTP host"); var treePort = new Option("--agent-port", () => 9223, "Agent HTTP port"); var alertTreeCmd = new Command("tree", "Show raw accessibility tree") { treeUdid, treePid, treePlatform, treeHost, treePort }; - alertTreeCmd.SetHandler(async (udid, pid, platform, host, port) => - await AlertTreeAsync(udid, pid, platform, host, port), treeUdid, treePid, treePlatform, treeHost, treePort); + alertTreeCmd.SetHandler(async (udid, pid, platform, host, port, json, noJson) => + await AlertTreeAsync(udid, pid, platform, host, port, OutputWriter.ResolveJsonMode(json, noJson)), treeUdid, treePid, treePlatform, treeHost, treePort, jsonOption, noJsonOption); alertCommand.Add(alertTreeCmd); mauiCommand.Add(alertCommand); @@ -361,58 +398,58 @@ await MauiScrollAsync(host, port, elementId, dx, dy, animated, window), var logsSourceOption = new Option("--source", () => null, "Filter by log source: native, webview, or all (default: all)"); var logsFollowOption = new Option("--follow", () => false, "Stream logs in real-time (Ctrl+C to stop)"); logsFollowOption.AddAlias("-f"); - var logsJsonOption = new Option("--json", () => false, "Output as JSONL (machine-readable, use with --follow)"); var logsReplayOption = new Option("--replay", () => 100, "Number of recent entries to replay on connect (use with --follow, 0 to skip)"); - var mauiLogsCmd = new Command("logs", "Fetch application logs") { logsLimitOption, logsSkipOption, logsSourceOption, logsFollowOption, logsJsonOption, logsReplayOption }; - mauiLogsCmd.SetHandler(async (host, port, limit, skip, source, follow, json, replay) => + var mauiLogsCmd = new Command("logs", "Fetch application logs") { logsLimitOption, logsSkipOption, logsSourceOption, logsFollowOption, logsReplayOption }; + mauiLogsCmd.SetHandler(async (ctx) => { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + var follow = ctx.ParseResult.GetValueForOption(logsFollowOption); if (follow) - await MauiLogsFollowAsync(host, port, source, json, replay); + await MauiLogsFollowAsync(host, port, ctx.ParseResult.GetValueForOption(logsSourceOption), isJson, ctx.ParseResult.GetValueForOption(logsReplayOption)); else - await MauiLogsAsync(host, port, limit, skip, source); - }, - agentHostOption, agentPortOption, logsLimitOption, logsSkipOption, logsSourceOption, logsFollowOption, logsJsonOption, logsReplayOption); + await MauiLogsAsync(host, port, isJson, ctx.ParseResult.GetValueForOption(logsLimitOption), ctx.ParseResult.GetValueForOption(logsSkipOption), ctx.ParseResult.GetValueForOption(logsSourceOption)); + }); mauiCommand.Add(mauiLogsCmd); // ── Network monitoring command ── var networkCommand = new Command("network", "Monitor HTTP network requests"); - var networkJsonOption = new Option("--json", () => false, "Output as JSONL (machine-readable)"); var networkLimitOption = new Option("--limit", () => 100, "Maximum number of entries to show"); var networkHostOption = new Option("--host", () => null, "Filter by host"); var networkMethodOption = new Option("--method", () => null, "Filter by HTTP method"); - networkCommand.AddOption(networkJsonOption); networkCommand.AddOption(networkLimitOption); networkCommand.AddOption(networkHostOption); networkCommand.AddOption(networkMethodOption); - networkCommand.SetHandler(async (host, port, json, limit, filterHost, filterMethod) => + networkCommand.SetHandler(async (host, port, json, noJson, limit, filterHost, filterMethod) => { - if (json) - await MauiNetworkMonitorAsync(host, port, json, limit, filterHost, filterMethod); + var isJson = OutputWriter.ResolveJsonMode(json, noJson); + if (isJson) + await MauiNetworkMonitorAsync(host, port, isJson, limit, filterHost, filterMethod); else await MauiDevFlow.CLI.NetworkMonitorTui.RunAsync(host, port, filterHost, filterMethod); }, - agentHostOption, agentPortOption, networkJsonOption, networkLimitOption, networkHostOption, networkMethodOption); + agentHostOption, agentPortOption, jsonOption, noJsonOption, networkLimitOption, networkHostOption, networkMethodOption); var networkListCmd = new Command("list", "List recent network requests (one-shot)"); - networkListCmd.AddOption(networkJsonOption); networkListCmd.AddOption(networkLimitOption); networkListCmd.AddOption(networkHostOption); networkListCmd.AddOption(networkMethodOption); - networkListCmd.SetHandler(async (host, port, json, limit, filterHost, filterMethod) => - await MauiNetworkListAsync(host, port, json, limit, filterHost, filterMethod), - agentHostOption, agentPortOption, networkJsonOption, networkLimitOption, networkHostOption, networkMethodOption); + networkListCmd.SetHandler(async (host, port, json, noJson, limit, filterHost, filterMethod) => + await MauiNetworkListAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), limit, filterHost, filterMethod), + agentHostOption, agentPortOption, jsonOption, noJsonOption, networkLimitOption, networkHostOption, networkMethodOption); networkCommand.Add(networkListCmd); var networkDetailId = new Argument("id", "Request ID to show details for"); var networkDetailCmd = new Command("detail", "Show full request/response details") { networkDetailId }; - networkDetailCmd.SetHandler(async (host, port, id) => - await MauiNetworkDetailAsync(host, port, id), - agentHostOption, agentPortOption, networkDetailId); + networkDetailCmd.SetHandler(async (host, port, json, noJson, id) => + await MauiNetworkDetailAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), + agentHostOption, agentPortOption, jsonOption, noJsonOption, networkDetailId); networkCommand.Add(networkDetailCmd); var networkClearCmd = new Command("clear", "Clear the network request buffer"); - networkClearCmd.SetHandler(async (host, port) => await MauiNetworkClearAsync(host, port), - agentHostOption, agentPortOption); + networkClearCmd.SetHandler(async (host, port, json, noJson) => await MauiNetworkClearAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson)), + agentHostOption, agentPortOption, jsonOption, noJsonOption); networkCommand.Add(networkClearCmd); mauiCommand.Add(networkCommand); @@ -465,7 +502,7 @@ await MauiNetworkDetailAsync(host, port, id), brokerCommand.Add(brokerStopCmd); var brokerStatusCmd = new Command("status", "Show broker daemon status"); - brokerStatusCmd.SetHandler(async () => await BrokerStatusAsync()); + brokerStatusCmd.SetHandler(async (json, noJson) => await BrokerStatusAsync(OutputWriter.ResolveJsonMode(json, noJson)), jsonOption, noJsonOption); brokerCommand.Add(brokerStatusCmd); var brokerLogCmd = new Command("log", "Show broker log"); @@ -476,7 +513,7 @@ await MauiNetworkDetailAsync(host, port, id), // ===== list command (agent discovery) ===== var listCmd = new Command("list", "List all connected agents"); - listCmd.SetHandler(async () => await ListAgentsCommandAsync()); + listCmd.SetHandler(async (json, noJson) => await ListAgentsCommandAsync(OutputWriter.ResolveJsonMode(json, noJson)), jsonOption, noJsonOption); rootCommand.Add(listCmd); // ===== wait command (wait for agent to connect) ===== @@ -492,17 +529,13 @@ await MauiNetworkDetailAsync(host, port, id), ["--wait-platform"], () => null, "Filter by platform (e.g., macOS, iOS, Android)"); - var waitJsonOption = new Option( - "--json", - () => false, - "Output agent info as JSON instead of human-readable text"); var waitCmd = new Command("wait", "Wait for an agent to connect to the broker") { - waitTimeoutOption, waitProjectOption, waitPlatformOption, waitJsonOption + waitTimeoutOption, waitProjectOption, waitPlatformOption }; - waitCmd.SetHandler(async (timeout, project, waitPlatform, json) => - await WaitForAgentCommandAsync(timeout, project, waitPlatform, json), - waitTimeoutOption, waitProjectOption, waitPlatformOption, waitJsonOption); + waitCmd.SetHandler(async (timeout, project, waitPlatform, json, noJson) => + await WaitForAgentCommandAsync(timeout, project, waitPlatform, OutputWriter.ResolveJsonMode(json, noJson)), + waitTimeoutOption, waitProjectOption, waitPlatformOption, jsonOption, noJsonOption); rootCommand.Add(waitCmd); // ===== batch command (interactive stdin/stdout) ===== @@ -1152,7 +1185,7 @@ private static async Task ListGitHubDirectoryAsync(HttpClient http, string baseP // ===== MAUI Agent Commands ===== - private static async Task MauiStatusAsync(string host, int port, int? window) + private static async Task MauiStatusAsync(string host, int port, bool json, int? window) { try { @@ -1160,40 +1193,105 @@ private static async Task MauiStatusAsync(string host, int port, int? window) var status = await client.GetStatusAsync(window); if (status == null) { - WriteError($"Cannot connect to agent at {host}:{port}"); + OutputWriter.WriteError($"Cannot connect to agent at {host}:{port}", json); + _errorOccurred = true; return; } - Console.WriteLine($"Agent: {status.Agent} v{status.Version}"); - Console.WriteLine($"Platform: {status.Platform}"); - Console.WriteLine($"Device: {status.DeviceType} ({status.Idiom})"); - Console.WriteLine($"App: {status.AppName}"); + OutputWriter.WriteResult(status, json, s => + { + Console.WriteLine($"Agent: {s.Agent} v{s.Version}"); + Console.WriteLine($"Platform: {s.Platform}"); + Console.WriteLine($"Device: {s.DeviceType} ({s.Idiom})"); + Console.WriteLine($"App: {s.AppName}"); + }); } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiTreeAsync(string host, int port, int depth, int? window) + private static async Task MauiTreeAsync(string host, int port, bool json, int depth, int? window, string? fields, string? format) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var tree = await client.GetTreeAsync(depth, window); - PrintTree(tree, 0); + if (json) + { + var projected = ProjectElements(tree, fields, format); + Console.WriteLine(JsonSerializer.Serialize(projected, new JsonSerializerOptions { WriteIndented = true, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull })); + } + else + { + PrintTree(tree, 0); + } } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiQueryAsync(string host, int port, string? type, string? autoId, string? text, string? selector) + private static async Task MauiQueryAsync(string host, int port, bool json, string? type, string? autoId, string? text, string? selector, string? fields, string? format, string? waitUntil, int timeout) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); - List results; + + if (!string.IsNullOrWhiteSpace(waitUntil)) + { + var condition = waitUntil.ToLowerInvariant(); + if (condition != "exists" && condition != "gone") + { + OutputWriter.WriteError("--wait-until must be 'exists' or 'gone'", json, "InvocationError"); + _errorOccurred = true; + return; + } + + var deadline = DateTime.UtcNow.AddSeconds(timeout); + var pollInterval = TimeSpan.FromMilliseconds(250); + List results; + + while (true) + { + results = !string.IsNullOrWhiteSpace(selector) + ? await client.QueryCssAsync(selector) + : await client.QueryAsync(type, autoId, text); + + if (condition == "exists" && results.Count > 0) break; + if (condition == "gone" && results.Count == 0) break; + if (DateTime.UtcNow >= deadline) + { + OutputWriter.WriteError( + $"Timeout after {timeout}s: condition '{waitUntil}' not met", + json, "RuntimeError", retryable: true, + suggestions: new[] { "Increase --timeout", "Check element identifiers with 'MAUI tree'" }); + _errorOccurred = true; + return; + } + await Task.Delay(pollInterval); + } + + WriteQueryResults(results, json, fields, format); + return; + } + + List queryResults; if (!string.IsNullOrWhiteSpace(selector)) - results = await client.QueryCssAsync(selector); + queryResults = await client.QueryCssAsync(selector); else - results = await client.QueryAsync(type, autoId, text); + queryResults = await client.QueryAsync(type, autoId, text); + WriteQueryResults(queryResults, json, fields, format); + } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } + } + + private static void WriteQueryResults(List results, bool json, string? fields, string? format) + { + if (json) + { + var projected = ProjectElements(results, fields, format); + Console.WriteLine(JsonSerializer.Serialize(projected, new JsonSerializerOptions { WriteIndented = true, DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull })); + } + else + { if (results.Count == 0) { Console.WriteLine("No elements found"); @@ -1209,54 +1307,62 @@ private static async Task MauiQueryAsync(string host, int port, string? type, st (el.IsEnabled ? "" : " [disabled]")); } } - catch (Exception ex) { WriteError(ex.Message); } } - private static async Task MauiHitTestAsync(string host, int port, double x, double y, int? window) + private static async Task MauiHitTestAsync(string host, int port, bool json, double x, double y, int? window) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); - var json = await client.HitTestAsync(x, y, window); - Console.WriteLine(json); + var result = await client.HitTestAsync(x, y, window); + if (json) + Console.WriteLine(result); + else + Console.WriteLine(result); } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiTapAsync(string host, int port, string elementId) + private static async Task MauiTapAsync(string host, int port, bool json, string elementId) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var success = await client.TapAsync(elementId); - Console.WriteLine(success ? $"Tapped: {elementId}" : $"Failed to tap: {elementId}"); + OutputWriter.WriteActionResult(success, "Tapped", elementId, json, + success ? $"Tapped: {elementId}" : $"Failed to tap: {elementId}"); + if (!success) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json, suggestions: new[] { "Run 'MAUI tree' to refresh element IDs" }); _errorOccurred = true; } } - private static async Task MauiFillAsync(string host, int port, string elementId, string text) + private static async Task MauiFillAsync(string host, int port, bool json, string elementId, string text) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var success = await client.FillAsync(elementId, text); - Console.WriteLine(success ? $"Filled: {elementId}" : $"Failed to fill: {elementId}"); + OutputWriter.WriteActionResult(success, "Filled", elementId, json, + success ? $"Filled: {elementId}" : $"Failed to fill: {elementId}"); + if (!success) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json, suggestions: new[] { "Run 'MAUI tree' to refresh element IDs" }); _errorOccurred = true; } } - private static async Task MauiClearAsync(string host, int port, string elementId) + private static async Task MauiClearAsync(string host, int port, bool json, string elementId) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var success = await client.ClearAsync(elementId); - Console.WriteLine(success ? $"Cleared: {elementId}" : $"Failed to clear: {elementId}"); + OutputWriter.WriteActionResult(success, "Cleared", elementId, json, + success ? $"Cleared: {elementId}" : $"Failed to clear: {elementId}"); + if (!success) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json, suggestions: new[] { "Run 'MAUI tree' to refresh element IDs" }); _errorOccurred = true; } } - private static async Task MauiScreenshotAsync(string host, int port, string? output, int? window, string? id, string? selector) + private static async Task MauiScreenshotAsync(string host, int port, bool json, string? output, int? window, string? id, string? selector) { try { @@ -1264,15 +1370,24 @@ private static async Task MauiScreenshotAsync(string host, int port, string? out var data = await client.ScreenshotAsync(window, id, selector); if (data == null) { - WriteError("Failed to capture screenshot"); + OutputWriter.WriteError("Failed to capture screenshot", json); + _errorOccurred = true; return; } var filename = output ?? $"screenshot_{DateTime.Now:yyyyMMdd_HHmmss}.png"; await File.WriteAllBytesAsync(filename, data); - var target = id != null ? $" (element: {id})" : selector != null ? $" (selector: {selector})" : ""; - Console.WriteLine($"Screenshot saved: {Path.GetFullPath(filename)} ({data.Length} bytes){target}"); + var fullPath = Path.GetFullPath(filename); + if (json) + { + OutputWriter.WriteResult(new { path = fullPath, size = data.Length }, json); + } + else + { + var target = id != null ? $" (element: {id})" : selector != null ? $" (selector: {selector})" : ""; + Console.WriteLine($"Screenshot saved: {fullPath} ({data.Length} bytes){target}"); + } } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } private static async Task RecordingStartAsync(string host, int port, string platform, string? output, int timeout) @@ -1317,37 +1432,50 @@ private static void RecordingStatusAsync() Console.WriteLine($" PID: {state.RecordingPid}"); } - private static async Task MauiPropertyAsync(string host, int port, string elementId, string propertyName) + private static async Task MauiPropertyAsync(string host, int port, bool json, string elementId, string propertyName) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var value = await client.GetPropertyAsync(elementId, propertyName); - Console.WriteLine(value != null ? $"{propertyName}: {value}" : $"Property '{propertyName}' not found"); + if (json) + { + OutputWriter.WriteResult(new { property = propertyName, value }, json); + } + else + { + Console.WriteLine(value != null ? $"{propertyName}: {value}" : $"Property '{propertyName}' not found"); + } } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiSetPropertyAsync(string host, int port, string elementId, string propertyName, string value) + private static async Task MauiSetPropertyAsync(string host, int port, bool json, string elementId, string propertyName, string value) { try { using var http = new HttpClient(); http.Timeout = TimeSpan.FromSeconds(10); - var json = JsonSerializer.Serialize(new { value }); - var content = new StringContent(json, System.Text.Encoding.UTF8, "application/json"); + var body = JsonSerializer.Serialize(new { value }); + var content = new StringContent(body, System.Text.Encoding.UTF8, "application/json"); var response = await http.PostAsync($"http://{host}:{port}/api/property/{elementId}/{propertyName}", content); - var body = await response.Content.ReadAsStringAsync(); + var responseBody = await response.Content.ReadAsStringAsync(); if (response.IsSuccessStatusCode) - Console.WriteLine($"Set {propertyName} = {value}"); + { + OutputWriter.WriteActionResult(true, "SetProperty", elementId, json, + $"Set {propertyName} = {value}"); + } else - WriteError($"Failed: {body}"); + { + OutputWriter.WriteError($"Failed: {responseBody}", json); + _errorOccurred = true; + } } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiElementAsync(string host, int port, string elementId) + private static async Task MauiElementAsync(string host, int port, bool json, string elementId) { try { @@ -1355,62 +1483,80 @@ private static async Task MauiElementAsync(string host, int port, string element var el = await client.GetElementAsync(elementId); if (el == null) { - WriteError($"Element '{elementId}' not found"); + OutputWriter.WriteError($"Element '{elementId}' not found", json, + suggestions: new[] { "Run 'MAUI tree' to refresh element IDs", "Element IDs are ephemeral — re-query after navigation" }); + _errorOccurred = true; return; } - Console.WriteLine(JsonSerializer.Serialize(el, new JsonSerializerOptions { WriteIndented = true })); + OutputWriter.WriteResult(el, json); } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiNavigateAsync(string host, int port, string route) + private static async Task MauiNavigateAsync(string host, int port, bool json, string route) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var success = await client.NavigateAsync(route); - Console.WriteLine(success ? $"Navigated to: {route}" : $"Failed to navigate to: {route}"); + OutputWriter.WriteActionResult(success, "Navigated", route, json, + success ? $"Navigated to: {route}" : $"Failed to navigate to: {route}"); + if (!success) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiScrollAsync(string host, int port, string? elementId, double dx, double dy, bool animated, int? window) + private static async Task MauiScrollAsync(string host, int port, bool json, string? elementId, double dx, double dy, bool animated, int? window) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var success = await client.ScrollAsync(elementId, dx, dy, animated, window); - if (elementId != null) - Console.WriteLine(success ? $"Scrolled to element: {elementId}" : $"Failed to scroll to element: {elementId}"); + if (json) + { + OutputWriter.WriteActionResult(success, "Scrolled", elementId, json); + } else - Console.WriteLine(success ? $"Scrolled by dx={dx}, dy={dy}" : "Failed to scroll"); + { + if (elementId != null) + Console.WriteLine(success ? $"Scrolled to element: {elementId}" : $"Failed to scroll to element: {elementId}"); + else + Console.WriteLine(success ? $"Scrolled by dx={dx}, dy={dy}" : "Failed to scroll"); + } + if (!success) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiFocusAsync(string host, int port, string elementId) + private static async Task MauiFocusAsync(string host, int port, bool json, string elementId) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var success = await client.FocusAsync(elementId); - Console.WriteLine(success ? $"Focused: {elementId}" : $"Failed to focus: {elementId}"); + OutputWriter.WriteActionResult(success, "Focused", elementId, json, + success ? $"Focused: {elementId}" : $"Failed to focus: {elementId}"); + if (!success) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiResizeAsync(string host, int port, int width, int height, int? window) + private static async Task MauiResizeAsync(string host, int port, bool json, int width, int height, int? window) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var success = await client.ResizeAsync(width, height, window); - Console.WriteLine(success ? $"Resized to: {width}x{height}" : $"Failed to resize"); + if (json) + OutputWriter.WriteActionResult(success, "Resized", $"{width}x{height}", json); + else + Console.WriteLine(success ? $"Resized to: {width}x{height}" : $"Failed to resize"); + if (!success) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiLogsAsync(string host, int port, int limit, int skip, string? source) + private static async Task MauiLogsAsync(string host, int port, bool json, int limit, int skip, string? source) { try { @@ -1420,18 +1566,25 @@ private static async Task MauiLogsAsync(string host, int port, int limit, int sk if (!string.IsNullOrEmpty(source)) url += $"&source={Uri.EscapeDataString(source)}"; var response = await http.GetAsync(url); - var json = await response.Content.ReadAsStringAsync(); + var body = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) { - WriteError($"Failed to fetch logs: {response.StatusCode} {json}"); + OutputWriter.WriteError($"Failed to fetch logs: {response.StatusCode} {body}", json); + _errorOccurred = true; + return; + } + + if (json) + { + Console.WriteLine(body); return; } - using var doc = JsonDocument.Parse(json); + using var doc = JsonDocument.Parse(body); if (doc.RootElement.ValueKind != JsonValueKind.Array) { - Console.WriteLine(json); + Console.WriteLine(body); return; } @@ -1440,7 +1593,7 @@ private static async Task MauiLogsAsync(string host, int port, int limit, int sk PrintLogEntry(entry); } } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } private static async Task MauiLogsFollowAsync(string host, int port, string? source, bool json, int replay) @@ -1711,7 +1864,7 @@ private static async Task MauiNetworkListAsync(string host, int port, bool json, catch (Exception ex) { WriteError(ex.Message); } } - private static async Task MauiNetworkDetailAsync(string host, int port, string id) + private static async Task MauiNetworkDetailAsync(string host, int port, bool json, string id) { try { @@ -1720,7 +1873,14 @@ private static async Task MauiNetworkDetailAsync(string host, int port, string i if (req == null) { - WriteError($"Network request '{id}' not found."); + OutputWriter.WriteError($"Network request '{id}' not found.", json); + _errorOccurred = true; + return; + } + + if (json) + { + OutputWriter.WriteResult(req, json); return; } @@ -1758,18 +1918,20 @@ private static async Task MauiNetworkDetailAsync(string host, int port, string i PrintBody(req.ResponseBody, req.ResponseBodyEncoding); } } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiNetworkClearAsync(string host, int port) + private static async Task MauiNetworkClearAsync(string host, int port, bool json) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); var result = await client.ClearNetworkRequestsAsync(); - Console.WriteLine(result ? "Network request buffer cleared." : "Failed to clear."); + OutputWriter.WriteActionResult(result, "NetworkCleared", null, json, + result ? "Network request buffer cleared." : "Failed to clear."); + if (!result) _errorOccurred = true; } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } // ── Network display helpers ── @@ -1897,6 +2059,57 @@ private static void PrintTree(List elements, int } } + /// + /// Project elements to a subset of fields for reduced token usage. + /// --format compact: only id, type, text, automationId, bounds, children + /// --fields "id,type,text": only specified fields + /// + private static object ProjectElements(List elements, string? fields, string? format) + { + var isCompact = string.Equals(format, "compact", StringComparison.OrdinalIgnoreCase); + HashSet? fieldSet = null; + + if (!string.IsNullOrWhiteSpace(fields)) + { + fieldSet = new HashSet( + fields.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries), + StringComparer.OrdinalIgnoreCase); + } + else if (isCompact) + { + fieldSet = new HashSet(StringComparer.OrdinalIgnoreCase) + { "id", "type", "text", "automationId", "bounds", "children", "isVisible", "isEnabled" }; + } + + if (fieldSet == null) + return elements; + + return elements.Select(el => ProjectElement(el, fieldSet)).ToList(); + } + + private static Dictionary ProjectElement(MauiDevFlow.Driver.ElementInfo el, HashSet fields) + { + var dict = new Dictionary(); + if (fields.Contains("id")) dict["id"] = el.Id; + if (fields.Contains("parentId")) dict["parentId"] = el.ParentId; + if (fields.Contains("type")) dict["type"] = el.Type; + if (fields.Contains("fullType")) dict["fullType"] = el.FullType; + if (fields.Contains("automationId") && el.AutomationId != null) dict["automationId"] = el.AutomationId; + if (fields.Contains("text") && el.Text != null) dict["text"] = el.Text; + if (fields.Contains("isVisible")) dict["isVisible"] = el.IsVisible; + if (fields.Contains("isEnabled")) dict["isEnabled"] = el.IsEnabled; + if (fields.Contains("isFocused")) dict["isFocused"] = el.IsFocused; + if (fields.Contains("opacity")) dict["opacity"] = el.Opacity; + if (fields.Contains("bounds") && el.Bounds != null) + dict["bounds"] = new { el.Bounds.X, el.Bounds.Y, el.Bounds.Width, el.Bounds.Height }; + if (fields.Contains("gestures") && el.Gestures != null) dict["gestures"] = el.Gestures; + if (fields.Contains("nativeType")) dict["nativeType"] = el.NativeType; + if (fields.Contains("nativeProperties") && el.NativeProperties != null) dict["nativeProperties"] = el.NativeProperties; + if (fields.Contains("children") && el.Children != null && el.Children.Count > 0) + dict["children"] = el.Children.Select(c => ProjectElement(c, fields)).ToList(); + return dict; + } + // ===== Alert & Permission Commands (iOS Simulator) ===== private static async Task ResolveUdidAsync(string? udid) @@ -2023,132 +2236,142 @@ private static async Task ResolveWindowsPidAsync(int? pid, string host, int throw new InvalidOperationException("Cannot determine Windows app PID. Specify --pid."); } - private static async Task AlertDetectAsync(string? udid, int? pid, string platform, string host, int port) + private static async Task AlertDetectAsync(string? udid, int? pid, string platform, string host, int port, bool json) { try { var plat = await ResolveAlertPlatformAsync(platform, host, port); + MauiDevFlow.Driver.AlertInfo? alert = null; if (plat == "maccatalyst") { var resolvedPid = await ResolveMacCatalystPidAsync(pid, host, port); var driver = new MauiDevFlow.Driver.MacCatalystAppDriver { ProcessId = resolvedPid }; - var alert = await driver.DetectAlertAsync(); - if (alert is null) { Console.WriteLine("No alert detected"); return; } - Console.WriteLine($"Alert: {alert.Title ?? "(no title)"}"); - foreach (var btn in alert.Buttons) - Console.WriteLine($" Button: \"{btn.Label}\""); + alert = await driver.DetectAlertAsync(); } else if (plat == "android") { var driver = new MauiDevFlow.Driver.AndroidAppDriver { Serial = udid }; - var alert = await driver.DetectAlertAsync(); - if (alert is null) { Console.WriteLine("No alert detected"); return; } - Console.WriteLine($"Alert: {alert.Title ?? "(no title)"}"); - foreach (var btn in alert.Buttons) - Console.WriteLine($" Button: \"{btn.Label}\" at ({btn.CenterX}, {btn.CenterY})"); + alert = await driver.DetectAlertAsync(); } else if (plat == "windows") { var resolvedPid = await ResolveWindowsPidAsync(pid, host, port); var driver = new MauiDevFlow.Driver.WindowsAppDriver { ProcessId = resolvedPid }; - var alert = await driver.DetectAlertAsync(); - if (alert is null) { Console.WriteLine("No alert detected"); return; } - Console.WriteLine($"Alert: {alert.Title ?? "(no title)"}"); - foreach (var btn in alert.Buttons) - Console.WriteLine($" Button: \"{btn.Label}\""); + alert = await driver.DetectAlertAsync(); } else { var resolved = await ResolveUdidAsync(udid); var driver = new MauiDevFlow.Driver.iOSSimulatorAppDriver { DeviceUdid = resolved }; - var alert = await driver.DetectAlertAsync(); - if (alert is null) { Console.WriteLine("No alert detected"); return; } + alert = await driver.DetectAlertAsync(); + } + + if (alert is null) + { + OutputWriter.WriteResult(new { detected = false }, json, _ => Console.WriteLine("No alert detected")); + return; + } + + OutputWriter.WriteResult(new { detected = true, title = alert.Title, buttons = alert.Buttons.Select(b => new { label = b.Label, centerX = b.CenterX, centerY = b.CenterY }) }, json, _ => + { Console.WriteLine($"Alert: {alert.Title ?? "(no title)"}"); foreach (var btn in alert.Buttons) - Console.WriteLine($" Button: \"{btn.Label}\" at ({btn.CenterX}, {btn.CenterY})"); - } + Console.WriteLine($" Button: \"{btn.Label}\""); + }); } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task AlertDismissAsync(string? udid, int? pid, string platform, string host, int port, string? buttonLabel) + private static async Task AlertDismissAsync(string? udid, int? pid, string platform, string host, int port, string? buttonLabel, bool json) { try { var plat = await ResolveAlertPlatformAsync(platform, host, port); + MauiDevFlow.Driver.AlertInfo? alert = null; if (plat == "maccatalyst") { var resolvedPid = await ResolveMacCatalystPidAsync(pid, host, port); var driver = new MauiDevFlow.Driver.MacCatalystAppDriver { ProcessId = resolvedPid }; - var alert = await driver.HandleAlertIfPresentAsync(buttonLabel); - if (alert is null) Console.WriteLine("No alert to dismiss"); - else Console.WriteLine($"Dismissed: {alert.Title ?? "(alert)"}"); + alert = await driver.HandleAlertIfPresentAsync(buttonLabel); } else if (plat == "android") { var driver = new MauiDevFlow.Driver.AndroidAppDriver { Serial = udid }; - var alert = await driver.HandleAlertIfPresentAsync(buttonLabel); - if (alert is null) Console.WriteLine("No alert to dismiss"); - else Console.WriteLine($"Dismissed: {alert.Title ?? "(alert)"}"); + alert = await driver.HandleAlertIfPresentAsync(buttonLabel); } else if (plat == "windows") { var resolvedPid = await ResolveWindowsPidAsync(pid, host, port); var driver = new MauiDevFlow.Driver.WindowsAppDriver { ProcessId = resolvedPid }; - var alert = await driver.HandleAlertIfPresentAsync(buttonLabel); - if (alert is null) Console.WriteLine("No alert to dismiss"); - else Console.WriteLine($"Dismissed: {alert.Title ?? "(alert)"}"); + alert = await driver.HandleAlertIfPresentAsync(buttonLabel); } else { var resolved = await ResolveUdidAsync(udid); var driver = new MauiDevFlow.Driver.iOSSimulatorAppDriver { DeviceUdid = resolved }; - var alert = await driver.HandleAlertIfPresentAsync(buttonLabel); - if (alert is null) Console.WriteLine("No alert to dismiss"); - else Console.WriteLine($"Dismissed: {alert.Title ?? "(alert)"}"); + alert = await driver.HandleAlertIfPresentAsync(buttonLabel); } + + if (alert is null) + OutputWriter.WriteResult(new { dismissed = false }, json, _ => Console.WriteLine("No alert to dismiss")); + else + OutputWriter.WriteResult(new { dismissed = true, title = alert.Title }, json, _ => Console.WriteLine($"Dismissed: {alert.Title ?? "(alert)"}")); } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task AlertTreeAsync(string? udid, int? pid, string platform, string host, int port) + private static async Task AlertTreeAsync(string? udid, int? pid, string platform, string host, int port, bool json) { try { var plat = await ResolveAlertPlatformAsync(platform, host, port); + string treeResult; if (plat == "maccatalyst") { var resolvedPid = await ResolveMacCatalystPidAsync(pid, host, port); var driver = new MauiDevFlow.Driver.MacCatalystAppDriver { ProcessId = resolvedPid }; - var tree = await driver.GetAccessibilityTreeAsync(); - Console.WriteLine(tree); + treeResult = await driver.GetAccessibilityTreeAsync(); } else if (plat == "android") { var driver = new MauiDevFlow.Driver.AndroidAppDriver { Serial = udid }; - var tree = await driver.GetAccessibilityTreeAsync(); - Console.WriteLine(tree); + treeResult = await driver.GetAccessibilityTreeAsync(); } else if (plat == "windows") { var resolvedPid = await ResolveWindowsPidAsync(pid, host, port); var driver = new MauiDevFlow.Driver.WindowsAppDriver { ProcessId = resolvedPid }; - var tree = await driver.GetAccessibilityTreeAsync(); - Console.WriteLine(tree); + treeResult = await driver.GetAccessibilityTreeAsync(); } else { var resolved = await ResolveUdidAsync(udid); var driver = new MauiDevFlow.Driver.iOSSimulatorAppDriver { DeviceUdid = resolved }; - var json = await driver.GetAccessibilityTreeAsync(); - using var doc = JsonDocument.Parse(json); - Console.WriteLine(JsonSerializer.Serialize(doc, new JsonSerializerOptions { WriteIndented = true })); + treeResult = await driver.GetAccessibilityTreeAsync(); + } + + if (json) + { + // Try to parse as JSON and output directly; if not valid JSON, wrap as string + try + { + using var doc = JsonDocument.Parse(treeResult); + Console.WriteLine(JsonSerializer.Serialize(doc, new JsonSerializerOptions { WriteIndented = true })); + } + catch (JsonException) + { + OutputWriter.WriteResult(new { tree = treeResult }, json); + } + } + else + { + Console.WriteLine(treeResult); } } - catch (Exception ex) { WriteError(ex.Message); } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } private static async Task PermissionAsync(string action, string? udid, string? bundleId, string service) @@ -2305,7 +2528,7 @@ private static async Task BrokerStopAsync() Console.WriteLine(success ? "Broker shutdown requested" : "Broker is not running"); } - private static async Task BrokerStatusAsync() + private static async Task BrokerStatusAsync(bool json) { var port = Broker.BrokerClient.ReadBrokerPortPublic(); @@ -2318,12 +2541,14 @@ private static async Task BrokerStatusAsync() var response = await http.GetStringAsync($"http://localhost:{Broker.BrokerServer.DefaultPort}/api/health"); var doc = JsonDocument.Parse(response); var agents = doc.RootElement.GetProperty("agents").GetInt32(); - Console.WriteLine($"Broker: running on port {Broker.BrokerServer.DefaultPort} ({agents} agent(s) connected) [no state file]"); + OutputWriter.WriteResult(new { running = true, port = Broker.BrokerServer.DefaultPort, agents, stateFile = false }, json, + _ => Console.WriteLine($"Broker: running on port {Broker.BrokerServer.DefaultPort} ({agents} agent(s) connected) [no state file]")); return; } catch { } - Console.WriteLine("Broker: not running"); + OutputWriter.WriteResult(new { running = false }, json, + _ => Console.WriteLine("Broker: not running")); return; } @@ -2333,11 +2558,13 @@ private static async Task BrokerStatusAsync() var response = await http.GetStringAsync($"http://localhost:{port}/api/health"); var doc = JsonDocument.Parse(response); var agents = doc.RootElement.GetProperty("agents").GetInt32(); - Console.WriteLine($"Broker: running on port {port} ({agents} agent(s) connected)"); + OutputWriter.WriteResult(new { running = true, port, agents }, json, + _ => Console.WriteLine($"Broker: running on port {port} ({agents} agent(s) connected)")); } catch { - Console.WriteLine($"Broker: not responding on port {port} (stale state file?)"); + OutputWriter.WriteResult(new { running = false, port, stale = true }, json, + _ => Console.WriteLine($"Broker: not responding on port {port} (stale state file?)")); } } @@ -2357,32 +2584,40 @@ private static void BrokerLogAsync() Console.WriteLine(lines[i]); } - private static async Task ListAgentsCommandAsync() + private static async Task ListAgentsCommandAsync(bool json) { var port = await Broker.BrokerClient.EnsureBrokerRunningAsync(); if (port == null) { - Console.WriteLine("Broker unavailable"); + OutputWriter.WriteError("Broker unavailable", json); + _errorOccurred = true; return; } var agents = await Broker.BrokerClient.ListAgentsAsync(port.Value); if (agents == null || agents.Length == 0) { - Console.WriteLine("No agents connected"); + OutputWriter.WriteResult(Array.Empty(), json, _ => Console.WriteLine("No agents connected")); return; } - Console.WriteLine($"{"ID",-14} {"App",-20} {"Platform",-14} {"TFM",-24} {"Port",-6} {"Version",-12} {"Uptime"}"); - Console.WriteLine(new string('-', 102)); - foreach (var a in agents) + if (json) { - var uptime = DateTime.UtcNow - a.ConnectedAt; - var uptimeStr = uptime.TotalHours >= 1 - ? $"{uptime.Hours}h {uptime.Minutes}m" - : $"{uptime.Minutes}m {uptime.Seconds}s"; - var version = a.Version ?? "-"; - Console.WriteLine($"{a.Id,-14} {a.AppName,-20} {a.Platform,-14} {a.Tfm,-24} {a.Port,-6} {version,-12} {uptimeStr}"); + OutputWriter.WriteResult(agents, json); + } + else + { + Console.WriteLine($"{"ID",-14} {"App",-20} {"Platform",-14} {"TFM",-24} {"Port",-6} {"Version",-12} {"Uptime"}"); + Console.WriteLine(new string('-', 102)); + foreach (var a in agents) + { + var uptime = DateTime.UtcNow - a.ConnectedAt; + var uptimeStr = uptime.TotalHours >= 1 + ? $"{uptime.Hours}h {uptime.Minutes}m" + : $"{uptime.Minutes}m {uptime.Seconds}s"; + var version = a.Version ?? "-"; + Console.WriteLine($"{a.Id,-14} {a.AppName,-20} {a.Platform,-14} {a.Tfm,-24} {a.Port,-6} {version,-12} {uptimeStr}"); + } } } From 4631c810d60410391f9943ddb9198ecace226eaf Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 15:50:40 -0500 Subject: [PATCH 02/12] feat: add implicit resolution, assertions, schema discovery, and agent guidance (Issue #28, Phases 3-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 — Reduce Agent Round-Trips: - Implicit element resolution: tap/fill/clear/focus accept --automationId, --type, --text, --index to resolve and act in one command - Post-action flags: --and-screenshot, --and-tree, --and-tree-depth on tap/fill/clear to verify actions without extra round-trips - MAUI assert command: quick property assertions with pass/fail result Phase 4 — Input Hardening: - Validate element IDs: reject control chars (<0x20), ? and # (hallucinated query params), warn on % (double-encoding) - Screenshot --overwrite flag: default fail-on-existing to prevent clobbering Phase 5 — Schema Discovery: - commands subcommand: lists all ~50 commands with descriptions and mutating flag as JSON for runtime schema discovery Phase 6 — Enhanced Skill Files: - Agent best practices section in SKILL.md: output format, token reduction, round-trip elimination, element ID lifecycle - Canonical workflow recipes: login flow, element inspection, state verification Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/maui-ai-debugging/SKILL.md | 69 +++++ src/MauiDevFlow.CLI/Program.cs | 360 +++++++++++++++++++++- 2 files changed, 413 insertions(+), 16 deletions(-) diff --git a/.claude/skills/maui-ai-debugging/SKILL.md b/.claude/skills/maui-ai-debugging/SKILL.md index 6f17995..21ceea6 100644 --- a/.claude/skills/maui-ai-debugging/SKILL.md +++ b/.claude/skills/maui-ai-debugging/SKILL.md @@ -426,3 +426,72 @@ their input. - Use `AutomationId` on important MAUI controls for stable element references. - For Blazor Hybrid, `cdp snapshot` is the most AI-friendly way to read page state. - Port discovery, multi-project setup, and custom ports: see [references/setup.md](references/setup.md#3b-port-configuration). + +## AI Agent Best Practices + +### Output Format +- **Always use `--json`** or rely on TTY auto-detection (JSON is auto-enabled when stdout is piped/redirected). +- Set `MAUIDEVFLOW_OUTPUT=json` in your environment for consistent machine-readable output. +- Use `--no-json` only when you specifically need human-readable output in a pipe. +- Errors go to stderr as structured JSON: `{"error": "...", "type": "RuntimeError", "retryable": false, "suggestions": [...]}`. +- Check exit codes: 0 = success, non-zero = failure. + +### Reducing Token Usage +- **Always use `--depth 3`** (or similar) for `MAUI tree` to avoid context overflow from full tree dumps. +- Use **`--fields "id,type,text,automationId"`** to project only the fields you need. +- Use **`--format compact`** for minimal tree output (id, type, text, automationId, bounds). +- **Prefer `MAUI query --automationId`** over full tree traversal — much smaller response. +- Use **element-level screenshots** (`--id `) when you only need to see one control. + +### Eliminating Round-Trips +- **Use implicit resolution** instead of query-then-act: + ```bash + # Instead of: query → get ID → tap + maui-devflow MAUI tap --automationId "LoginButton" + maui-devflow MAUI fill --automationId "Username" --text "admin" + maui-devflow MAUI tap --type Button --index 0 # first Button + ``` +- **Use `--wait-until`** instead of polling loops: + ```bash + maui-devflow MAUI query --automationId "ResultsList" --wait-until exists --timeout 10 + maui-devflow MAUI query --automationId "Spinner" --wait-until gone --timeout 30 + ``` +- **Use post-action flags** to verify in one call: + ```bash + maui-devflow MAUI tap abc123 --and-screenshot --and-tree --and-tree-depth 2 + ``` +- **Use `MAUI assert`** for quick state checks: + ```bash + maui-devflow MAUI assert --id abc123 Text "Welcome!" + maui-devflow MAUI assert --automationId "Counter" Text "5" + ``` + +### Element IDs +- Element IDs are **ephemeral** — re-query after navigation or state changes. +- Don't cache element IDs across multiple actions — refresh with `tree` or `query`. +- Prefer `--automationId` for stable references (set in XAML). +- Use `maui-devflow commands --json` to discover available commands at runtime. + +### Canonical Workflows + +**Login flow:** +```bash +maui-devflow MAUI query --automationId "LoginPage" --wait-until exists --timeout 15 +maui-devflow MAUI fill --automationId "UsernameField" "admin" +maui-devflow MAUI fill --automationId "PasswordField" "password" +maui-devflow MAUI tap --automationId "LoginButton" --and-screenshot +maui-devflow MAUI query --automationId "HomePage" --wait-until exists --timeout 10 +``` + +**Element inspection:** +```bash +maui-devflow MAUI query --automationId "MyControl" --json --fields "id,type,text,bounds" +maui-devflow MAUI element --json +maui-devflow MAUI property Text +``` + +**State verification:** +```bash +maui-devflow MAUI tap --automationId "IncrementButton" +maui-devflow MAUI assert --automationId "CounterLabel" Text "1" +``` diff --git a/src/MauiDevFlow.CLI/Program.cs b/src/MauiDevFlow.CLI/Program.cs index 5135ed4..42975cb 100644 --- a/src/MauiDevFlow.CLI/Program.cs +++ b/src/MauiDevFlow.CLI/Program.cs @@ -217,31 +217,109 @@ static async Task Main(string[] args) agentHostOption, agentPortOption, jsonOption, noJsonOption, hitTestXArg, hitTestYArg, windowOption); mauiCommand.Add(mauiHitTestCmd); + // Shared element resolution options (for tap, fill, clear, focus) + var resolveAutoIdOption = new Option("--automationId", "Resolve element by AutomationId (instead of element ID)"); + var resolveTypeOption = new Option("--type", "Resolve element by type (used with --automationId or alone)"); + var resolveTextOption = new Option("--text", "Resolve element by text content"); + var resolveIndexOption = new Option("--index", () => 0, "Index when multiple elements match (0-based, default: first)"); + var andScreenshotOption = new Option("--and-screenshot", "Take screenshot after action (optional: output path)"); + andScreenshotOption.SetDefaultValue(null); + andScreenshotOption.Arity = ArgumentArity.ZeroOrOne; + var andTreeOption = new Option("--and-tree", "Dump visual tree after action"); + var andTreeDepthOption = new Option("--and-tree-depth", () => 2, "Max depth for --and-tree"); + // MAUI tap - var tapIdArg = new Argument("elementId", "Element ID to tap"); - var mauiTapCmd = new Command("tap", "Tap element") { tapIdArg }; - mauiTapCmd.SetHandler(async (host, port, json, noJson, id) => await MauiTapAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), agentHostOption, agentPortOption, jsonOption, noJsonOption, tapIdArg); + var tapIdArg = new Argument("elementId", () => null, "Element ID to tap (optional if --automationId, --type, or --text is used)"); + var mauiTapCmd = new Command("tap", "Tap element") { tapIdArg, resolveAutoIdOption, resolveTypeOption, resolveTextOption, resolveIndexOption, andScreenshotOption, andTreeOption, andTreeDepthOption }; + mauiTapCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + var id = ctx.ParseResult.GetValueForArgument(tapIdArg); + var autoId = ctx.ParseResult.GetValueForOption(resolveAutoIdOption); + var type = ctx.ParseResult.GetValueForOption(resolveTypeOption); + var text = ctx.ParseResult.GetValueForOption(resolveTextOption); + var index = ctx.ParseResult.GetValueForOption(resolveIndexOption); + var andScreenshot = ctx.ParseResult.GetValueForOption(andScreenshotOption); + var hasAndScreenshot = ctx.ParseResult.FindResultFor(andScreenshotOption) != null; + var andTree = ctx.ParseResult.GetValueForOption(andTreeOption); + var andTreeDepth = ctx.ParseResult.GetValueForOption(andTreeDepthOption); + var resolvedId = await ResolveElementIdAsync(host, port, isJson, id, autoId, type, text, index); + if (resolvedId == null) return; + await MauiTapAsync(host, port, isJson, resolvedId); + await HandlePostActionFlags(host, port, isJson, hasAndScreenshot, andScreenshot, andTree, andTreeDepth); + }); mauiCommand.Add(mauiTapCmd); // MAUI fill - var fillIdArg = new Argument("elementId", "Element ID"); + var fillIdArg = new Argument("elementId", () => null, "Element ID (optional if --automationId, --type, or --text is used)"); var fillTextArg2 = new Argument("text", "Text to fill"); - var mauiFillCmd = new Command("fill", "Fill text into element") { fillIdArg, fillTextArg2 }; - mauiFillCmd.SetHandler(async (host, port, json, noJson, id, text) => await MauiFillAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id, text), agentHostOption, agentPortOption, jsonOption, noJsonOption, fillIdArg, fillTextArg2); + var mauiFillCmd = new Command("fill", "Fill text into element") { fillIdArg, fillTextArg2, resolveAutoIdOption, resolveTypeOption, resolveTextOption, resolveIndexOption, andScreenshotOption, andTreeOption, andTreeDepthOption }; + mauiFillCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + var id = ctx.ParseResult.GetValueForArgument(fillIdArg); + var fillText = ctx.ParseResult.GetValueForArgument(fillTextArg2); + var autoId = ctx.ParseResult.GetValueForOption(resolveAutoIdOption); + var type = ctx.ParseResult.GetValueForOption(resolveTypeOption); + var text = ctx.ParseResult.GetValueForOption(resolveTextOption); + var index = ctx.ParseResult.GetValueForOption(resolveIndexOption); + var andScreenshot = ctx.ParseResult.GetValueForOption(andScreenshotOption); + var hasAndScreenshot = ctx.ParseResult.FindResultFor(andScreenshotOption) != null; + var andTree = ctx.ParseResult.GetValueForOption(andTreeOption); + var andTreeDepth = ctx.ParseResult.GetValueForOption(andTreeDepthOption); + var resolvedId = await ResolveElementIdAsync(host, port, isJson, id, autoId, type, text, index); + if (resolvedId == null) return; + await MauiFillAsync(host, port, isJson, resolvedId, fillText); + await HandlePostActionFlags(host, port, isJson, hasAndScreenshot, andScreenshot, andTree, andTreeDepth); + }); mauiCommand.Add(mauiFillCmd); // MAUI clear - var clearIdArg = new Argument("elementId", "Element ID to clear"); - var mauiClearCmd = new Command("clear", "Clear text from element") { clearIdArg }; - mauiClearCmd.SetHandler(async (host, port, json, noJson, id) => await MauiClearAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), agentHostOption, agentPortOption, jsonOption, noJsonOption, clearIdArg); + var clearIdArg = new Argument("elementId", () => null, "Element ID to clear (optional if --automationId, --type, or --text is used)"); + var mauiClearCmd = new Command("clear", "Clear text from element") { clearIdArg, resolveAutoIdOption, resolveTypeOption, resolveTextOption, resolveIndexOption, andScreenshotOption, andTreeOption, andTreeDepthOption }; + mauiClearCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + var id = ctx.ParseResult.GetValueForArgument(clearIdArg); + var autoId = ctx.ParseResult.GetValueForOption(resolveAutoIdOption); + var type = ctx.ParseResult.GetValueForOption(resolveTypeOption); + var text = ctx.ParseResult.GetValueForOption(resolveTextOption); + var index = ctx.ParseResult.GetValueForOption(resolveIndexOption); + var andScreenshot = ctx.ParseResult.GetValueForOption(andScreenshotOption); + var hasAndScreenshot = ctx.ParseResult.FindResultFor(andScreenshotOption) != null; + var andTree = ctx.ParseResult.GetValueForOption(andTreeOption); + var andTreeDepth = ctx.ParseResult.GetValueForOption(andTreeDepthOption); + var resolvedId = await ResolveElementIdAsync(host, port, isJson, id, autoId, type, text, index); + if (resolvedId == null) return; + await MauiClearAsync(host, port, isJson, resolvedId); + await HandlePostActionFlags(host, port, isJson, hasAndScreenshot, andScreenshot, andTree, andTreeDepth); + }); mauiCommand.Add(mauiClearCmd); // MAUI screenshot var screenshotOutputOption = new Option("--output", "Output file path"); var screenshotIdOption = new Option("--id", "Element ID to capture"); var screenshotSelectorOption = new Option("--selector", "CSS selector to capture (first match)"); - var mauiScreenshotCmd = new Command("screenshot", "Take screenshot") { screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption }; - mauiScreenshotCmd.SetHandler(async (host, port, json, noJson, output, window, id, selector) => await MauiScreenshotAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), output, window, id, selector), agentHostOption, agentPortOption, jsonOption, noJsonOption, screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption); + var screenshotOverwriteOption = new Option("--overwrite", () => false, "Overwrite existing file (default: fail if exists)"); + var mauiScreenshotCmd = new Command("screenshot", "Take screenshot") { screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption, screenshotOverwriteOption }; + mauiScreenshotCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + await MauiScreenshotAsync(host, port, isJson, + ctx.ParseResult.GetValueForOption(screenshotOutputOption), + ctx.ParseResult.GetValueForOption(windowOption), + ctx.ParseResult.GetValueForOption(screenshotIdOption), + ctx.ParseResult.GetValueForOption(screenshotSelectorOption), + ctx.ParseResult.GetValueForOption(screenshotOverwriteOption)); + }); mauiCommand.Add(mauiScreenshotCmd); // MAUI recording subcommands @@ -315,9 +393,22 @@ await MauiScrollAsync(host, port, isJson, mauiCommand.Add(mauiScrollCmd); // MAUI focus - var focusIdArg = new Argument("elementId", "Element ID to focus"); - var mauiFocusCmd = new Command("focus", "Set focus to element") { focusIdArg }; - mauiFocusCmd.SetHandler(async (host, port, json, noJson, id) => await MauiFocusAsync(host, port, OutputWriter.ResolveJsonMode(json, noJson), id), agentHostOption, agentPortOption, jsonOption, noJsonOption, focusIdArg); + var focusIdArg = new Argument("elementId", () => null, "Element ID to focus (optional if --automationId, --type, or --text is used)"); + var mauiFocusCmd = new Command("focus", "Set focus to element") { focusIdArg, resolveAutoIdOption, resolveTypeOption, resolveTextOption, resolveIndexOption }; + mauiFocusCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + var id = ctx.ParseResult.GetValueForArgument(focusIdArg); + var autoId = ctx.ParseResult.GetValueForOption(resolveAutoIdOption); + var type = ctx.ParseResult.GetValueForOption(resolveTypeOption); + var text = ctx.ParseResult.GetValueForOption(resolveTextOption); + var index = ctx.ParseResult.GetValueForOption(resolveIndexOption); + var resolvedId = await ResolveElementIdAsync(host, port, isJson, id, autoId, type, text, index); + if (resolvedId == null) return; + await MauiFocusAsync(host, port, isJson, resolvedId); + }); mauiCommand.Add(mauiFocusCmd); // MAUI resize @@ -366,6 +457,25 @@ await MauiScrollAsync(host, port, isJson, mauiCommand.Add(alertCommand); + // MAUI assert + var assertIdOption = new Option("--id", "Element ID to assert on"); + var assertAutoIdOption = new Option("--automationId", "Resolve element by AutomationId"); + var assertPropertyArg = new Argument("propertyName", "Property to check"); + var assertEqualsArg = new Argument("expectedValue", "Expected value"); + var mauiAssertCmd = new Command("assert", "Assert element property value") { assertIdOption, assertAutoIdOption, assertPropertyArg, assertEqualsArg }; + mauiAssertCmd.SetHandler(async (ctx) => + { + var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; + var port = ctx.ParseResult.GetValueForOption(agentPortOption); + var isJson = OutputWriter.ResolveJsonMode(ctx.ParseResult.GetValueForOption(jsonOption), ctx.ParseResult.GetValueForOption(noJsonOption)); + var id = ctx.ParseResult.GetValueForOption(assertIdOption); + var autoId = ctx.ParseResult.GetValueForOption(assertAutoIdOption); + var prop = ctx.ParseResult.GetValueForArgument(assertPropertyArg); + var expected = ctx.ParseResult.GetValueForArgument(assertEqualsArg); + await MauiAssertAsync(host, port, isJson, id, autoId, prop, expected); + }); + mauiCommand.Add(mauiAssertCmd); + // MAUI permission subcommands (iOS simulator only — uses xcrun simctl privacy) var permissionCommand = new Command("permission", "Manage iOS simulator permissions"); @@ -561,6 +671,22 @@ await BatchAsync(host, port, delay, continueOnError, human), }); rootCommand.Add(versionCmd); + // ===== commands command (schema discovery) ===== + var commandsCmd = new Command("commands", "List all available commands (machine-readable schema discovery)"); + commandsCmd.SetHandler((json, noJson) => + { + var isJson = OutputWriter.ResolveJsonMode(json, noJson); + var cmds = GetCommandDescriptions(); + OutputWriter.WriteResult(cmds, isJson, list => + { + Console.WriteLine($"{"Command",-35} {"Mutating",-10} {"Description"}"); + Console.WriteLine(new string('-', 85)); + foreach (var c in list) + Console.WriteLine($"{c.Command,-35} {(c.Mutating ? "yes" : "no"),-10} {c.Description}"); + }); + }, jsonOption, noJsonOption); + rootCommand.Add(commandsCmd); + _parser = new CommandLineBuilder(rootCommand) .UseDefaults() .Build(); @@ -978,6 +1104,201 @@ private static string FormatJson(JsonElement element) return JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = true }); } + // ===== Element Resolution Helper ===== + + /// + /// Resolve an element ID from either a direct ID or query options (--automationId, --type, --text). + /// Returns null and writes error if resolution fails. + /// + private static async Task ResolveElementIdAsync(string host, int port, bool json, + string? elementId, string? automationId, string? type, string? text, int index) + { + // Direct ID takes priority + if (!string.IsNullOrWhiteSpace(elementId)) + { + ValidateElementId(elementId, json); + return elementId; + } + + // Need at least one resolution option + if (string.IsNullOrWhiteSpace(automationId) && string.IsNullOrWhiteSpace(type) && string.IsNullOrWhiteSpace(text)) + { + OutputWriter.WriteError("Provide an element ID or use --automationId, --type, or --text to resolve", json, "InvocationError"); + _errorOccurred = true; + return null; + } + + try + { + using var client = new MauiDevFlow.Driver.AgentClient(host, port); + var results = await client.QueryAsync(type, automationId, text); + + if (results.Count == 0) + { + var criteria = new List(); + if (automationId != null) criteria.Add($"automationId=\"{automationId}\""); + if (type != null) criteria.Add($"type=\"{type}\""); + if (text != null) criteria.Add($"text=\"{text}\""); + OutputWriter.WriteError($"No elements found matching {string.Join(", ", criteria)}", json, + suggestions: new[] { "Run 'MAUI tree' to see available elements", "Check automationId spelling" }); + _errorOccurred = true; + return null; + } + + if (index >= results.Count) + { + OutputWriter.WriteError($"Index {index} out of range (found {results.Count} element(s))", json, "RuntimeError", + suggestions: new[] { $"Use --index 0 through {results.Count - 1}" }); + _errorOccurred = true; + return null; + } + + return results[index].Id; + } + catch (Exception ex) + { + OutputWriter.WriteError(ex.Message, json); + _errorOccurred = true; + return null; + } + } + + /// + /// Validate element ID for common agent mistakes (control chars, embedded query params). + /// + private static void ValidateElementId(string id, bool json) + { + if (id.Any(c => c < 0x20)) + { + OutputWriter.WriteError($"Element ID contains control characters: '{id}'", json, "InvocationError", + suggestions: new[] { "Element IDs should not contain control characters", "Run 'MAUI tree' to get valid IDs" }); + _errorOccurred = true; + return; + } + if (id.Contains('?') || id.Contains('#')) + { + OutputWriter.WriteError($"Element ID contains '?' or '#': '{id}' — this looks like a URL fragment, not an element ID", json, "InvocationError", + suggestions: new[] { "Run 'MAUI tree' to get valid element IDs" }); + _errorOccurred = true; + return; + } + if (id.Contains('%')) + { + Console.Error.WriteLine($"Warning: Element ID contains '%': '{id}' — possible double-encoding"); + } + } + + /// + /// Handle post-action flags (--and-screenshot, --and-tree) after a mutating command. + /// + private static async Task HandlePostActionFlags(string host, int port, bool json, + bool hasAndScreenshot, string? andScreenshotPath, bool andTree, int andTreeDepth) + { + if (hasAndScreenshot) + { + await MauiScreenshotAsync(host, port, json, andScreenshotPath, null, null, null); + } + if (andTree) + { + await MauiTreeAsync(host, port, json, andTreeDepth, null, null, null); + } + } + + // ===== Assert Command ===== + + private static async Task MauiAssertAsync(string host, int port, bool json, string? elementId, string? automationId, string propertyName, string expectedValue) + { + try + { + var resolvedId = await ResolveElementIdAsync(host, port, json, elementId, automationId, null, null, 0); + if (resolvedId == null) return; + + using var client = new MauiDevFlow.Driver.AgentClient(host, port); + var actualValue = await client.GetPropertyAsync(resolvedId, propertyName); + + var passed = string.Equals(actualValue, expectedValue, StringComparison.Ordinal); + if (json) + { + OutputWriter.WriteResult(new { passed, property = propertyName, expected = expectedValue, actual = actualValue, elementId = resolvedId }, json); + } + else + { + if (passed) + Console.WriteLine($"PASS: {propertyName} == \"{expectedValue}\""); + else + { + Console.WriteLine($"FAIL: {propertyName} expected \"{expectedValue}\" but got \"{actualValue ?? "(null)"}\""); + _errorOccurred = true; + } + } + if (!passed) _errorOccurred = true; + } + catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } + } + + // ===== Command Descriptions (Schema Discovery) ===== + + private record CommandDescription(string Command, string Description, bool Mutating); + + private static List GetCommandDescriptions() => new() + { + new("MAUI status", "Check agent connection and app info", false), + new("MAUI tree", "Dump visual element tree", false), + new("MAUI query", "Find elements by type, automationId, text, or CSS selector", false), + new("MAUI element", "Get detailed element info by ID", false), + new("MAUI hittest", "Find elements at screen coordinates", false), + new("MAUI tap", "Tap a UI element", true), + new("MAUI fill", "Fill text into an input element", true), + new("MAUI clear", "Clear text from an input element", true), + new("MAUI focus", "Set focus to an element", true), + new("MAUI navigate", "Navigate to a Shell route", true), + new("MAUI scroll", "Scroll content or scroll element into view", true), + new("MAUI resize", "Resize app window", true), + new("MAUI property", "Get element property value", false), + new("MAUI set-property", "Set element property value", true), + new("MAUI screenshot", "Take screenshot of app or element", false), + new("MAUI assert", "Assert element property equals expected value", false), + new("MAUI recording start", "Start screen recording", true), + new("MAUI recording stop", "Stop screen recording", true), + new("MAUI recording status", "Check recording status", false), + new("MAUI alert detect", "Check if a system dialog is visible", false), + new("MAUI alert dismiss", "Dismiss a system dialog", true), + new("MAUI alert tree", "Show accessibility tree for dialog detection", false), + new("MAUI permission grant", "Grant iOS simulator permission", true), + new("MAUI permission revoke", "Revoke iOS simulator permission", true), + new("MAUI permission reset", "Reset iOS simulator permission", true), + new("MAUI logs", "Fetch or stream application logs", false), + new("MAUI network", "Monitor HTTP network requests (live)", false), + new("MAUI network list", "List recent network requests", false), + new("MAUI network detail", "Show full network request details", false), + new("MAUI network clear", "Clear network request buffer", true), + new("cdp webviews", "List available CDP WebViews", false), + new("cdp status", "Check CDP connection status", false), + new("cdp Browser getVersion", "Get browser version", false), + new("cdp Runtime evaluate", "Evaluate JavaScript expression", false), + new("cdp DOM getDocument", "Get DOM document tree", false), + new("cdp DOM querySelector", "Find element by CSS selector", false), + new("cdp DOM querySelectorAll", "Find all elements by CSS selector", false), + new("cdp DOM getOuterHTML", "Get element outer HTML", false), + new("cdp Input click", "Click element by CSS selector", true), + new("cdp Input insertText", "Insert text at cursor", true), + new("cdp Input fill", "Fill form field by CSS selector", true), + new("cdp Page navigate", "Navigate WebView to URL", true), + new("cdp Page reload", "Reload WebView page", true), + new("cdp Page captureScreenshot", "Take WebView screenshot", false), + new("cdp snapshot", "Get simplified DOM snapshot", false), + new("cdp source", "Get page HTML source", false), + new("list", "List all connected agents", false), + new("wait", "Wait for an agent to connect", false), + new("batch", "Execute commands from stdin", true), + new("broker start", "Start the broker daemon", true), + new("broker stop", "Stop the broker daemon", true), + new("broker status", "Show broker status", false), + new("broker log", "Show broker log", false), + new("commands", "List all available commands", false), + new("version", "Show CLI version", false), + }; + // ===== Update Skill Command ===== private const string SkillRepo = "Redth/MauiDevFlow"; @@ -1362,10 +1683,18 @@ private static async Task MauiClearAsync(string host, int port, bool json, strin catch (Exception ex) { OutputWriter.WriteError(ex.Message, json, suggestions: new[] { "Run 'MAUI tree' to refresh element IDs" }); _errorOccurred = true; } } - private static async Task MauiScreenshotAsync(string host, int port, bool json, string? output, int? window, string? id, string? selector) + private static async Task MauiScreenshotAsync(string host, int port, bool json, string? output, int? window, string? id, string? selector, bool overwrite = false) { try { + var filename = output ?? $"screenshot_{DateTime.Now:yyyyMMdd_HHmmss}.png"; + if (!overwrite && File.Exists(filename)) + { + OutputWriter.WriteError($"File already exists: {Path.GetFullPath(filename)} (use --overwrite to replace)", json, "InvocationError"); + _errorOccurred = true; + return; + } + using var client = new MauiDevFlow.Driver.AgentClient(host, port); var data = await client.ScreenshotAsync(window, id, selector); if (data == null) @@ -1374,7 +1703,6 @@ private static async Task MauiScreenshotAsync(string host, int port, bool json, _errorOccurred = true; return; } - var filename = output ?? $"screenshot_{DateTime.Now:yyyyMMdd_HHmmss}.png"; await File.WriteAllBytesAsync(filename, data); var fullPath = Path.GetFullPath(filename); if (json) From 084484c2035a16b13ed8589fd9c890343258d7fa Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 16:07:27 -0500 Subject: [PATCH 03/12] Improve SKILL.md: accurate command reference, fix syntax errors - Update command reference table with all new CLI options: --json/--no-json global flags, --fields, --format compact, --wait-until, --overwrite, implicit resolution (--automationId, --type, --text, --index), post-action flags (--and-screenshot, --and-tree) - Add missing commands: assert, commands --json - Fix incorrect --text flag in fill examples (text is positional) - Document implicit element resolution and post-action flags prominently above command table - Update typical inspection/interaction flows with new features - Add element ID lifecycle guidance to command reference - Remove per-command --json options (now global) Eval results: improved skill 100% pass rate vs 93.3% old skill across 3 test scenarios (16 assertions). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/maui-ai-debugging/SKILL.md | 67 ++++++++++++++--------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/.claude/skills/maui-ai-debugging/SKILL.md b/.claude/skills/maui-ai-debugging/SKILL.md index 21ceea6..ae8c7ad 100644 --- a/.claude/skills/maui-ai-debugging/SKILL.md +++ b/.claude/skills/maui-ai-debugging/SKILL.md @@ -153,14 +153,15 @@ if the build fails. ### 4. Inspect and Interact **Typical inspection flow:** -1. `maui-devflow MAUI tree` — see the full visual tree with element IDs, types, text, bounds +1. `maui-devflow MAUI tree --depth 3 --fields "id,type,text,automationId"` — shallow tree with key fields only 2. `maui-devflow MAUI tree --window 1` — filter to a specific window (0-based index) 3. `maui-devflow MAUI query --automationId "MyButton"` — find specific elements -4. `maui-devflow MAUI element ` — get full details (type, bounds, visibility, children) -5. `maui-devflow MAUI property Text` — read any property by name -6. `maui-devflow MAUI screenshot --output screen.png` — visual verification -7. `maui-devflow MAUI screenshot --id --output el.png` — element-only screenshot -8. `maui-devflow MAUI screenshot --selector "Button" --output btn.png` — screenshot by CSS selector +4. `maui-devflow MAUI query --type Entry --fields "id,text,automationId"` — all Entry fields with specific fields +5. `maui-devflow MAUI element ` — get full details (type, bounds, visibility, children) +6. `maui-devflow MAUI property Text` — read any property by name +7. `maui-devflow MAUI screenshot --output screen.png` — visual verification +8. `maui-devflow MAUI screenshot --id --output el.png` — element-only screenshot +9. `maui-devflow MAUI screenshot --selector "Button" --output btn.png` — screenshot by CSS selector **Property inspection** is more reliable than screenshots for verifying exact runtime values: ```bash @@ -177,10 +178,11 @@ Supports: string, bool, int, double, Color (named/hex), Thickness, enums. Change until the app restarts — safe for experimentation. **Typical interaction flow:** -1. `maui-devflow MAUI fill "text"` — type into Entry/Editor fields -2. `maui-devflow MAUI tap ` — tap buttons, checkboxes, list items -3. `maui-devflow MAUI clear ` — clear text fields -4. Take screenshot to verify result +1. `maui-devflow MAUI fill --automationId "MyEntry" "text"` — type into Entry/Editor fields (no query needed) +2. `maui-devflow MAUI tap --automationId "MyButton"` — tap buttons, checkboxes, list items +3. `maui-devflow MAUI clear --automationId "MyEntry"` — clear text fields +4. Or use element IDs from tree/query: `maui-devflow MAUI tap ` +5. Take screenshot to verify result, or use `--and-screenshot` on the action **Blazor WebView (if applicable):** 1. `maui-devflow cdp snapshot` — DOM tree as accessible text (best for AI) @@ -298,41 +300,56 @@ Connecting clients receive a replay of buffered history, then live entries as th ### maui-devflow MAUI (Native Agent) -Global options: `--agent-host` (default localhost), `--agent-port` (auto-discovered via broker), `--platform`. +Global options (work on any subcommand): +- `--agent-host` (default localhost), `--agent-port` (auto-discovered via broker), `--platform` +- `--json` — force JSON output. Auto-enabled when stdout is piped/redirected (TTY auto-detection). +- `--no-json` — force human-readable output even when piped. +- Env var: `MAUIDEVFLOW_OUTPUT=json` for persistent JSON mode. -These options work on any subcommand position: `maui-devflow MAUI status --agent-port 10224` -or `maui-devflow --agent-port 10224 MAUI status` — both are valid. +**Implicit element resolution:** Commands that take an `` (tap, fill, clear, focus) +also accept `--automationId`, `--type`, `--text`, `--index` to resolve the element in a single +call. This eliminates the query→act round-trip. The `` argument is optional when +resolution options are provided. + +**Post-action flags:** tap, fill, clear accept `--and-screenshot [path]`, `--and-tree`, +`--and-tree-depth N` to return verification data alongside the action result. | Command | Description | |---------|-------------| | `MAUI status [--window W]` | Agent connection status, platform, app name, window count | -| `MAUI tree [--depth N] [--window W]` | Visual tree (IDs, types, text, bounds). Depth 0=unlimited. Window is 0-based index; omit for all windows | -| `MAUI query --type T --automationId A --text T` | Find elements (any/all filters) | +| `MAUI tree [--depth N] [--window W] [--fields F] [--format compact]` | Visual tree. `--fields "id,type,text"` projects specific fields. `--format compact` returns only id, type, text, automationId, bounds | +| `MAUI query [--type T] [--automationId A] [--text T] [--selector S] [--fields F] [--format compact] [--wait-until exists\|gone] [--timeout N]` | Find elements. `--wait-until` polls until condition met (default 30s timeout). `--fields` and `--format` same as tree | | `MAUI hittest [--window W]` | Find elements at a point (deepest first). Returns IDs, types, bounds | -| `MAUI tap ` | Tap an element | -| `MAUI fill ` | Fill text into Entry/Editor | -| `MAUI clear ` | Clear text from element | -| `MAUI screenshot [--output path.png] [--window W] [--id ID] [--selector SEL]` | PNG screenshot. Capture full window or a specific element by ID/selector. Window is 0-based index; default first window | +| `MAUI tap [elementId] [--automationId A] [--type T] [--text T] [--index N] [--and-screenshot [path]] [--and-tree] [--and-tree-depth N]` | Tap element by ID or implicit resolution | +| `MAUI fill [elementId] [--automationId A] [--type T] [--text T] [--index N] [--and-screenshot [path]] [--and-tree]` | Fill text into Entry/Editor. elementId optional when using resolution options | +| `MAUI clear [elementId] [--automationId A] [--type T] [--text T] [--index N] [--and-screenshot [path]] [--and-tree]` | Clear text. elementId optional when using resolution options | +| `MAUI focus [elementId] [--automationId A] [--type T] [--text T] [--index N]` | Set focus. elementId optional when using resolution options | +| `MAUI assert [--id ID] [--automationId A] ` | Assert element property value. Exit 0 if match, 1 if mismatch. Ideal for verification without screenshots | +| `MAUI screenshot [--output path.png] [--window W] [--id ID] [--selector SEL] [--overwrite]` | PNG screenshot. `--overwrite` replaces existing file (default: fail if exists) | | `MAUI property ` | Read property (Text, IsVisible, FontSize, etc.) | | `MAUI set-property ` | Set property (live editing — colors, text, sizes, etc.) | | `MAUI element ` | Full element JSON (type, bounds, children, etc.) | | `MAUI navigate ` | Shell navigation (e.g. `//native`, `//blazor`) | | `MAUI scroll [--element id] [--dx N] [--dy N] [--window W]` | Scroll by delta or scroll element into view | -| `MAUI focus ` | Set focus to element | | `MAUI resize [--window W]` | Resize app window. Window is 0-based index; default first window | -| `MAUI logs [--limit N] [--skip N] [--source S] [--follow] [--json]` | Fetch or stream application logs. `--follow` / `-f` streams in real-time via WebSocket (Ctrl+C to stop). `--json` outputs JSONL. Source: native, webview, or omit for all | -| `MAUI recording start [--output path] [--timeout 30]` | Start screen recording. Default timeout 30s. Uses platform-native tools (adb screenrecord, xcrun simctl, screencapture, ffmpeg) | +| `MAUI logs [--limit N] [--skip N] [--source S] [--follow]` | Fetch or stream application logs. `--follow` / `-f` streams in real-time (Ctrl+C to stop). Source: native, webview, or omit for all | +| `MAUI recording start [--output path] [--timeout 30]` | Start screen recording. Default timeout 30s | | `MAUI recording stop` | Stop active recording and save the video file | | `MAUI recording status` | Check if a recording is currently in progress | -| `MAUI network` | Live network monitor — streams HTTP requests in real-time (Ctrl+C to stop). Use `--json` for JSONL output | -| `MAUI network list [--host H] [--method M] [--json]` | One-shot: dump recent captured HTTP requests as table or JSONL | +| `MAUI network` | Live network monitor — streams HTTP requests in real-time (Ctrl+C to stop) | +| `MAUI network list [--host H] [--method M]` | One-shot: dump recent captured HTTP requests | | `MAUI network detail ` | Full request/response details: headers, body, timing | | `MAUI network clear` | Clear the captured request buffer | +| `commands [--json]` | List all available commands with descriptions. `--json` returns machine-readable schema with command names, descriptions, and whether they mutate state | Element IDs come from `MAUI tree` or `MAUI query`. AutomationId-based elements use their AutomationId directly. Others use generated hex IDs. When multiple elements share the same AutomationId, suffixes are appended: `TodoCheckBox`, `TodoCheckBox_1`, `TodoCheckBox_2`, etc. +**Element ID lifecycle:** IDs are ephemeral — they're regenerated on each tree walk. After +navigation, page changes, or significant UI updates, re-query to get fresh IDs. AutomationIds +are stable across rebuilds (they come from XAML), so prefer `--automationId` for scripted flows. + ### maui-devflow cdp (Blazor WebView CDP) Global options: `--agent-host` (default localhost), `--agent-port` (auto-discovered via broker). @@ -448,7 +465,7 @@ their input. ```bash # Instead of: query → get ID → tap maui-devflow MAUI tap --automationId "LoginButton" - maui-devflow MAUI fill --automationId "Username" --text "admin" + maui-devflow MAUI fill --automationId "Username" "admin" maui-devflow MAUI tap --type Button --index 0 # first Button ``` - **Use `--wait-until`** instead of polling loops: From c6727cf1f9e777a1df6f4a51011c9a095effbdbf Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 16:10:48 -0500 Subject: [PATCH 04/12] Fix tree depth guidance: recommend --depth 15+, add adaptive learning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MAUI visual trees are deeply nested — a simple control is typically at depth 10-15, not 3. Updated all depth recommendations from 3 to 15, and added an 'Adaptive Depth Learning' section encouraging agents to observe where controls appear in their first tree dump and reuse that depth for subsequent calls. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/maui-ai-debugging/SKILL.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.claude/skills/maui-ai-debugging/SKILL.md b/.claude/skills/maui-ai-debugging/SKILL.md index ae8c7ad..ee13478 100644 --- a/.claude/skills/maui-ai-debugging/SKILL.md +++ b/.claude/skills/maui-ai-debugging/SKILL.md @@ -153,7 +153,7 @@ if the build fails. ### 4. Inspect and Interact **Typical inspection flow:** -1. `maui-devflow MAUI tree --depth 3 --fields "id,type,text,automationId"` — shallow tree with key fields only +1. `maui-devflow MAUI tree --depth 15 --fields "id,type,text,automationId"` — tree with key fields only (depth 15 reaches most controls) 2. `maui-devflow MAUI tree --window 1` — filter to a specific window (0-based index) 3. `maui-devflow MAUI query --automationId "MyButton"` — find specific elements 4. `maui-devflow MAUI query --type Entry --fields "id,text,automationId"` — all Entry fields with specific fields @@ -454,12 +454,23 @@ their input. - Check exit codes: 0 = success, non-zero = failure. ### Reducing Token Usage -- **Always use `--depth 3`** (or similar) for `MAUI tree` to avoid context overflow from full tree dumps. +- **Use `--depth 15`** (or higher) for `MAUI tree` — MAUI visual trees are deeply nested (a simple + control is often at depth 10-15). Start with `--depth 15`; if you see truncated children, increase. + After your first successful tree dump, note the depth where meaningful controls appear and reuse + that depth for subsequent calls. If the tree is still too large, combine with `--fields` to reduce width. - Use **`--fields "id,type,text,automationId"`** to project only the fields you need. - Use **`--format compact`** for minimal tree output (id, type, text, automationId, bounds). - **Prefer `MAUI query --automationId`** over full tree traversal — much smaller response. - Use **element-level screenshots** (`--id `) when you only need to see one control. +### Adaptive Depth Learning +MAUI app trees vary in depth — a simple app might have controls at depth 8, while a complex app +with Shell + NavigationPage + nested layouts might need depth 20+. After your first `MAUI tree` +call, look at where the leaf-level controls (Button, Entry, Label) appear and remember that depth. +Use it for all subsequent tree calls in the same session. If you navigate to a new page that seems +deeper, bump the depth up. This avoids both truncating useful content and wasting tokens on +excessively deep dumps. + ### Eliminating Round-Trips - **Use implicit resolution** instead of query-then-act: ```bash @@ -475,7 +486,7 @@ their input. ``` - **Use post-action flags** to verify in one call: ```bash - maui-devflow MAUI tap abc123 --and-screenshot --and-tree --and-tree-depth 2 + maui-devflow MAUI tap abc123 --and-screenshot --and-tree --and-tree-depth 5 ``` - **Use `MAUI assert`** for quick state checks: ```bash From 48ac471d83d3097279e817c35715edc26bccd966 Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 16:18:52 -0500 Subject: [PATCH 05/12] feat: add --max-width to screenshot for HiDPI resize HiDPI displays (2x, 3x) produce screenshots 2-3x larger than needed for AI grounding. Add --max-width option that resizes the captured PNG server-side (in the agent via SkiaSharp) before transfer. - Agent: parse maxWidth query param, resize via SKBitmap.Resize() - Driver: pass maxWidth through AgentClient.ScreenshotAsync() - CLI: add --max-width option to screenshot command - SKILL.md: recommend --max-width 800 for AI agents, add 'Screenshot Size Reduction' guidance section Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/maui-ai-debugging/SKILL.md | 20 +++++++-- .gitignore | 1 + .../DevFlowAgentService.cs | 42 +++++++++++++++++-- .../MauiDevFlow.Agent.Core.csproj | 1 + src/MauiDevFlow.CLI/Program.cs | 17 ++++---- src/MauiDevFlow.Driver/AgentClient.cs | 3 +- 6 files changed, 69 insertions(+), 15 deletions(-) diff --git a/.claude/skills/maui-ai-debugging/SKILL.md b/.claude/skills/maui-ai-debugging/SKILL.md index ee13478..b70eb13 100644 --- a/.claude/skills/maui-ai-debugging/SKILL.md +++ b/.claude/skills/maui-ai-debugging/SKILL.md @@ -159,8 +159,8 @@ if the build fails. 4. `maui-devflow MAUI query --type Entry --fields "id,text,automationId"` — all Entry fields with specific fields 5. `maui-devflow MAUI element ` — get full details (type, bounds, visibility, children) 6. `maui-devflow MAUI property Text` — read any property by name -7. `maui-devflow MAUI screenshot --output screen.png` — visual verification -8. `maui-devflow MAUI screenshot --id --output el.png` — element-only screenshot +7. `maui-devflow MAUI screenshot --output screen.png --max-width 800` — visual verification (resized for AI) +8. `maui-devflow MAUI screenshot --id --output el.png --max-width 800` — element-only screenshot 9. `maui-devflow MAUI screenshot --selector "Button" --output btn.png` — screenshot by CSS selector **Property inspection** is more reliable than screenshots for verifying exact runtime values: @@ -325,7 +325,7 @@ resolution options are provided. | `MAUI clear [elementId] [--automationId A] [--type T] [--text T] [--index N] [--and-screenshot [path]] [--and-tree]` | Clear text. elementId optional when using resolution options | | `MAUI focus [elementId] [--automationId A] [--type T] [--text T] [--index N]` | Set focus. elementId optional when using resolution options | | `MAUI assert [--id ID] [--automationId A] ` | Assert element property value. Exit 0 if match, 1 if mismatch. Ideal for verification without screenshots | -| `MAUI screenshot [--output path.png] [--window W] [--id ID] [--selector SEL] [--overwrite]` | PNG screenshot. `--overwrite` replaces existing file (default: fail if exists) | +| `MAUI screenshot [--output path.png] [--window W] [--id ID] [--selector SEL] [--overwrite] [--max-width N]` | PNG screenshot. `--max-width 800` resizes for AI agents (HiDPI screens produce 2-3x larger images than needed). `--overwrite` replaces existing file | | `MAUI property ` | Read property (Text, IsVisible, FontSize, etc.) | | `MAUI set-property ` | Set property (live editing — colors, text, sizes, etc.) | | `MAUI element ` | Full element JSON (type, bounds, children, etc.) | @@ -471,6 +471,20 @@ Use it for all subsequent tree calls in the same session. If you navigate to a n deeper, bump the depth up. This avoids both truncating useful content and wasting tokens on excessively deep dumps. +### Screenshot Size Reduction +Modern devices have HiDPI displays (2x, 3x scale factors) that produce screenshots 2-3x larger +than needed for visual grounding. A full-screen iPhone screenshot can be 1290×2796 pixels (~3MB). +For AI understanding, 800px wide is usually plenty. + +- **Always use `--max-width 800`** on screenshots to reduce image size and token cost. + ```bash + maui-devflow MAUI screenshot --output screen.png --max-width 800 + maui-devflow MAUI screenshot --id --output el.png --max-width 800 + ``` +- The resize happens server-side (in the agent) before transfer, so it also reduces network time. +- For detailed pixel-level inspection (e.g., verifying exact colors or alignment), omit `--max-width` + to get the full-resolution capture. + ### Eliminating Round-Trips - **Use implicit resolution** instead of query-then-act: ```bash diff --git a/.gitignore b/.gitignore index 153e2ab..a3b8d50 100644 --- a/.gitignore +++ b/.gitignore @@ -434,3 +434,4 @@ FodyWeavers.xsd # Local skill version tracking (written by maui-devflow update-skill) .skill-version +maui-ai-debugging-workspace/ diff --git a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs index 541edc3..54e49ae 100644 --- a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs @@ -547,6 +547,10 @@ protected virtual async Task HandleScreenshot(HttpRequest request) { if (_app == null) return HttpResponse.Error("Agent not bound to app"); + int? maxWidth = null; + if (request.QueryParams.TryGetValue("maxWidth", out var mwStr) && int.TryParse(mwStr, out var mw) && mw > 0) + maxWidth = mw; + // Check for fullscreen mode (captures all windows including dialogs) if (request.QueryParams.TryGetValue("fullscreen", out var fs) && fs.Equals("true", StringComparison.OrdinalIgnoreCase)) @@ -555,7 +559,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) { var pngData = await CaptureFullScreenAsync(); if (pngData != null) - return HttpResponse.Png(pngData); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); return HttpResponse.Error("Full-screen capture not supported on this platform"); } catch (Exception ex) @@ -582,7 +586,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) if (pngData == null) return HttpResponse.Error($"Capture returned null for element '{elementId}'"); - return HttpResponse.Png(pngData); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); } catch (Exception ex) { @@ -617,7 +621,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) if (pngData == null) return HttpResponse.Error($"Capture returned null for element '{matchId}'"); - return HttpResponse.Png(pngData); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); } catch (FormatException ex) { @@ -673,7 +677,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) if (pngData == null) return HttpResponse.Error("Failed to capture screenshot"); - return HttpResponse.Png(pngData); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); } catch (Exception ex) { @@ -709,6 +713,36 @@ protected virtual async Task HandleScreenshot(HttpRequest request) return Task.FromResult(null); } + /// + /// Resizes a PNG image if it exceeds the specified max width, preserving aspect ratio. + /// HiDPI displays (2x, 3x) produce screenshots much larger than needed for AI grounding. + /// + private static byte[] ResizePngIfNeeded(byte[] pngData, int? maxWidth) + { + if (maxWidth == null || maxWidth <= 0) return pngData; + + try + { + using var original = SkiaSharp.SKBitmap.Decode(pngData); + if (original == null || original.Width <= maxWidth.Value) return pngData; + + var scale = (float)maxWidth.Value / original.Width; + var newHeight = (int)(original.Height * scale); + + using var resized = original.Resize(new SkiaSharp.SKImageInfo(maxWidth.Value, newHeight), SkiaSharp.SKFilterQuality.Medium); + if (resized == null) return pngData; + + using var image = SkiaSharp.SKImage.FromBitmap(resized); + using var encoded = image.Encode(SkiaSharp.SKEncodedImageFormat.Png, 100); + return encoded.ToArray(); + } + catch + { + // If resize fails for any reason, return original unchanged + return pngData; + } + } + private async Task HandleProperty(HttpRequest request) { if (_app == null) return HttpResponse.Error("Agent not bound to app"); diff --git a/src/MauiDevFlow.Agent.Core/MauiDevFlow.Agent.Core.csproj b/src/MauiDevFlow.Agent.Core/MauiDevFlow.Agent.Core.csproj index 2caf978..b42e01a 100644 --- a/src/MauiDevFlow.Agent.Core/MauiDevFlow.Agent.Core.csproj +++ b/src/MauiDevFlow.Agent.Core/MauiDevFlow.Agent.Core.csproj @@ -11,6 +11,7 @@ + diff --git a/src/MauiDevFlow.CLI/Program.cs b/src/MauiDevFlow.CLI/Program.cs index 42975cb..81bd297 100644 --- a/src/MauiDevFlow.CLI/Program.cs +++ b/src/MauiDevFlow.CLI/Program.cs @@ -307,7 +307,8 @@ static async Task Main(string[] args) var screenshotIdOption = new Option("--id", "Element ID to capture"); var screenshotSelectorOption = new Option("--selector", "CSS selector to capture (first match)"); var screenshotOverwriteOption = new Option("--overwrite", () => false, "Overwrite existing file (default: fail if exists)"); - var mauiScreenshotCmd = new Command("screenshot", "Take screenshot") { screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption, screenshotOverwriteOption }; + var screenshotMaxWidthOption = new Option("--max-width", "Resize screenshot to this max width (preserves aspect ratio). Reduces file size for AI agents on HiDPI displays"); + var mauiScreenshotCmd = new Command("screenshot", "Take screenshot") { screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption, screenshotOverwriteOption, screenshotMaxWidthOption }; mauiScreenshotCmd.SetHandler(async (ctx) => { var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; @@ -318,7 +319,8 @@ await MauiScreenshotAsync(host, port, isJson, ctx.ParseResult.GetValueForOption(windowOption), ctx.ParseResult.GetValueForOption(screenshotIdOption), ctx.ParseResult.GetValueForOption(screenshotSelectorOption), - ctx.ParseResult.GetValueForOption(screenshotOverwriteOption)); + ctx.ParseResult.GetValueForOption(screenshotOverwriteOption), + ctx.ParseResult.GetValueForOption(screenshotMaxWidthOption)); }); mauiCommand.Add(mauiScreenshotCmd); @@ -1196,7 +1198,7 @@ private static async Task HandlePostActionFlags(string host, int port, bool json { if (hasAndScreenshot) { - await MauiScreenshotAsync(host, port, json, andScreenshotPath, null, null, null); + await MauiScreenshotAsync(host, port, json, andScreenshotPath, null, null, null, false, null); } if (andTree) { @@ -1683,7 +1685,7 @@ private static async Task MauiClearAsync(string host, int port, bool json, strin catch (Exception ex) { OutputWriter.WriteError(ex.Message, json, suggestions: new[] { "Run 'MAUI tree' to refresh element IDs" }); _errorOccurred = true; } } - private static async Task MauiScreenshotAsync(string host, int port, bool json, string? output, int? window, string? id, string? selector, bool overwrite = false) + private static async Task MauiScreenshotAsync(string host, int port, bool json, string? output, int? window, string? id, string? selector, bool overwrite = false, int? maxWidth = null) { try { @@ -1696,7 +1698,7 @@ private static async Task MauiScreenshotAsync(string host, int port, bool json, } using var client = new MauiDevFlow.Driver.AgentClient(host, port); - var data = await client.ScreenshotAsync(window, id, selector); + var data = await client.ScreenshotAsync(window, id, selector, maxWidth); if (data == null) { OutputWriter.WriteError("Failed to capture screenshot", json); @@ -1707,12 +1709,13 @@ private static async Task MauiScreenshotAsync(string host, int port, bool json, var fullPath = Path.GetFullPath(filename); if (json) { - OutputWriter.WriteResult(new { path = fullPath, size = data.Length }, json); + OutputWriter.WriteResult(new { path = fullPath, size = data.Length, maxWidth = maxWidth }, json); } else { var target = id != null ? $" (element: {id})" : selector != null ? $" (selector: {selector})" : ""; - Console.WriteLine($"Screenshot saved: {fullPath} ({data.Length} bytes){target}"); + var resized = maxWidth != null ? $" (max-width: {maxWidth}px)" : ""; + Console.WriteLine($"Screenshot saved: {fullPath} ({data.Length} bytes){target}{resized}"); } } catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } diff --git a/src/MauiDevFlow.Driver/AgentClient.cs b/src/MauiDevFlow.Driver/AgentClient.cs index e06dcee..1084351 100644 --- a/src/MauiDevFlow.Driver/AgentClient.cs +++ b/src/MauiDevFlow.Driver/AgentClient.cs @@ -146,7 +146,7 @@ public async Task ResizeAsync(int width, int height, int? window = null) /// Take a screenshot (returns PNG bytes). /// Optionally target a specific element by ID or CSS selector. /// - public async Task ScreenshotAsync(int? window = null, string? elementId = null, string? selector = null) + public async Task ScreenshotAsync(int? window = null, string? elementId = null, string? selector = null, int? maxWidth = null) { try { @@ -154,6 +154,7 @@ public async Task ResizeAsync(int width, int height, int? window = null) if (window != null) queryParams.Add($"window={window}"); if (elementId != null) queryParams.Add($"id={Uri.EscapeDataString(elementId)}"); if (selector != null) queryParams.Add($"selector={Uri.EscapeDataString(selector)}"); + if (maxWidth != null) queryParams.Add($"maxWidth={maxWidth}"); var url = queryParams.Count > 0 ? $"{_baseUrl}/api/screenshot?{string.Join("&", queryParams)}" From e118c17cae371c4cfde635dd683549e8165f0553 Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 16:54:32 -0500 Subject: [PATCH 06/12] Auto-scale screenshots to 1x based on per-window display density Screenshots are now automatically scaled to 1x logical resolution by default. The agent detects each window's display density via platform-specific APIs: - iOS/Mac Catalyst: UIWindow.Screen.Scale - Android: Activity.Resources.DisplayMetrics.Density - Windows: XamlRoot.RasterizationScale - macOS AppKit: NSWindow.BackingScaleFactor - GTK: Widget.GetScaleFactor() This reduces a 3x iPhone screenshot from 1320x2868 (277KB) to 440x956 (52KB) without any CLI flags needed. Use --scale native for full resolution. Also adds displayDensity to the /api/status response and updates SKILL.md to remove the --max-width 800 recommendation (no longer needed). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/maui-ai-debugging/SKILL.md | 29 ++++---- .../DevFlowAgentService.cs | 68 +++++++++++++++---- src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs | 29 ++++++++ src/MauiDevFlow.Agent/DevFlowAgentService.cs | 36 ++++++++++ src/MauiDevFlow.CLI/Program.cs | 19 +++--- src/MauiDevFlow.Driver/AgentClient.cs | 3 +- 6 files changed, 149 insertions(+), 35 deletions(-) diff --git a/.claude/skills/maui-ai-debugging/SKILL.md b/.claude/skills/maui-ai-debugging/SKILL.md index b70eb13..a9c6b06 100644 --- a/.claude/skills/maui-ai-debugging/SKILL.md +++ b/.claude/skills/maui-ai-debugging/SKILL.md @@ -159,8 +159,8 @@ if the build fails. 4. `maui-devflow MAUI query --type Entry --fields "id,text,automationId"` — all Entry fields with specific fields 5. `maui-devflow MAUI element ` — get full details (type, bounds, visibility, children) 6. `maui-devflow MAUI property Text` — read any property by name -7. `maui-devflow MAUI screenshot --output screen.png --max-width 800` — visual verification (resized for AI) -8. `maui-devflow MAUI screenshot --id --output el.png --max-width 800` — element-only screenshot +7. `maui-devflow MAUI screenshot --output screen.png` — visual verification (auto-scaled to 1x on HiDPI) +8. `maui-devflow MAUI screenshot --id --output el.png` — element-only screenshot 9. `maui-devflow MAUI screenshot --selector "Button" --output btn.png` — screenshot by CSS selector **Property inspection** is more reliable than screenshots for verifying exact runtime values: @@ -325,7 +325,7 @@ resolution options are provided. | `MAUI clear [elementId] [--automationId A] [--type T] [--text T] [--index N] [--and-screenshot [path]] [--and-tree]` | Clear text. elementId optional when using resolution options | | `MAUI focus [elementId] [--automationId A] [--type T] [--text T] [--index N]` | Set focus. elementId optional when using resolution options | | `MAUI assert [--id ID] [--automationId A] ` | Assert element property value. Exit 0 if match, 1 if mismatch. Ideal for verification without screenshots | -| `MAUI screenshot [--output path.png] [--window W] [--id ID] [--selector SEL] [--overwrite] [--max-width N]` | PNG screenshot. `--max-width 800` resizes for AI agents (HiDPI screens produce 2-3x larger images than needed). `--overwrite` replaces existing file | +| `MAUI screenshot [--output path.png] [--window W] [--id ID] [--selector SEL] [--overwrite] [--max-width N] [--scale native]` | PNG screenshot. Auto-scales to 1x logical resolution on HiDPI displays (2x, 3x). Use `--scale native` for full resolution. `--max-width N` overrides auto-scaling with explicit width. `--overwrite` replaces existing file | | `MAUI property ` | Read property (Text, IsVisible, FontSize, etc.) | | `MAUI set-property ` | Set property (live editing — colors, text, sizes, etc.) | | `MAUI element ` | Full element JSON (type, bounds, children, etc.) | @@ -471,19 +471,22 @@ Use it for all subsequent tree calls in the same session. If you navigate to a n deeper, bump the depth up. This avoids both truncating useful content and wasting tokens on excessively deep dumps. -### Screenshot Size Reduction -Modern devices have HiDPI displays (2x, 3x scale factors) that produce screenshots 2-3x larger -than needed for visual grounding. A full-screen iPhone screenshot can be 1290×2796 pixels (~3MB). -For AI understanding, 800px wide is usually plenty. +### Screenshot Auto-Scaling (HiDPI) +Screenshots are **automatically scaled to 1x logical resolution** by default. The agent detects +the device's display density (2x on Retina, 3x on iPhone Pro Max, 1x on desktop) and divides +the screenshot dimensions accordingly. This happens server-side before transfer. -- **Always use `--max-width 800`** on screenshots to reduce image size and token cost. +- **No action needed** — just use `maui-devflow MAUI screenshot --output screen.png` and the + image will be appropriately sized for AI understanding. +- **Full resolution:** Use `--scale native` when you need pixel-perfect images (e.g., verifying + exact colors, alignment, or anti-aliasing). ```bash - maui-devflow MAUI screenshot --output screen.png --max-width 800 - maui-devflow MAUI screenshot --id --output el.png --max-width 800 + maui-devflow MAUI screenshot --output full-res.png --scale native + ``` +- **Explicit max width:** Use `--max-width N` to override auto-scaling with a specific pixel width. + ```bash + maui-devflow MAUI screenshot --output screen.png --max-width 600 ``` -- The resize happens server-side (in the agent) before transfer, so it also reduces network time. -- For detailed pixel-level inspection (e.g., verifying exact colors or alignment), omit `--max-width` - to get the full-resolution capture. ### Eliminating Round-Trips - **Use implicit resolution** instead of query-then-act: diff --git a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs index 54e49ae..1b72049 100644 --- a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs @@ -177,6 +177,19 @@ public DevFlowAgentService(AgentOptions? options = null) /// Device idiom for status reporting. Override for platforms without DeviceInfo. protected virtual string IdiomName => DeviceInfo.Current.Idiom.ToString(); + /// + /// Gets the display density (scale factor) for a specific window. Returns 1.0 for standard, + /// 2.0 for @2x (Retina), 3.0 for @3x (iPhone Pro Max), etc. + /// Used to auto-scale screenshots to 1x logical resolution. + /// Override in platform-specific agents to query the native window's actual screen density, + /// which may vary across displays in multi-monitor setups. + /// + protected virtual double GetWindowDisplayDensity(IWindow? window) + { + try { return DeviceDisplay.MainDisplayInfo.Density; } + catch { return 1.0; } + } + /// Gets native window dimensions when MAUI reports 0. Override for platform-specific access. protected virtual (double width, double height) GetNativeWindowSize(IWindow window) => (0, 0); @@ -292,6 +305,7 @@ private async Task HandleStatus(HttpRequest request) platform = PlatformName, deviceType = DeviceTypeName, idiom = IdiomName, + displayDensity = GetWindowDisplayDensity(window), appName = _app?.GetType().Assembly.GetName().Name ?? "unknown", running = _app != null, cdpReady = _cdpWebViews.Any(w => w.IsReady), @@ -551,6 +565,22 @@ protected virtual async Task HandleScreenshot(HttpRequest request) if (request.QueryParams.TryGetValue("maxWidth", out var mwStr) && int.TryParse(mwStr, out var mw) && mw > 0) maxWidth = mw; + // Auto-scale to 1x by default on HiDPI displays. Override with scale=native to keep full resolution. + bool autoScale = true; + if (request.QueryParams.TryGetValue("scale", out var scaleParam)) + { + autoScale = !scaleParam.Equals("native", StringComparison.OrdinalIgnoreCase) + && !scaleParam.Equals("full", StringComparison.OrdinalIgnoreCase); + } + + // Resolve the target window and its display density on the UI thread + var windowIndex = ParseWindowIndex(request); + var density = await DispatchAsync(() => + { + var w = GetWindow(windowIndex); + return GetWindowDisplayDensity(w); + }); + // Check for fullscreen mode (captures all windows including dialogs) if (request.QueryParams.TryGetValue("fullscreen", out var fs) && fs.Equals("true", StringComparison.OrdinalIgnoreCase)) @@ -559,7 +589,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) { var pngData = await CaptureFullScreenAsync(); if (pngData != null) - return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale)); return HttpResponse.Error("Full-screen capture not supported on this platform"); } catch (Exception ex) @@ -586,7 +616,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) if (pngData == null) return HttpResponse.Error($"Capture returned null for element '{elementId}'"); - return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale)); } catch (Exception ex) { @@ -621,7 +651,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) if (pngData == null) return HttpResponse.Error($"Capture returned null for element '{matchId}'"); - return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale)); } catch (FormatException ex) { @@ -635,7 +665,6 @@ protected virtual async Task HandleScreenshot(HttpRequest request) try { - var windowIndex = ParseWindowIndex(request); var pngData = await DispatchAsync(async () => { var window = GetWindow(windowIndex); @@ -677,7 +706,7 @@ protected virtual async Task HandleScreenshot(HttpRequest request) if (pngData == null) return HttpResponse.Error("Failed to capture screenshot"); - return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth)); + return HttpResponse.Png(ResizePngIfNeeded(pngData, maxWidth, density, autoScale)); } catch (Exception ex) { @@ -714,22 +743,36 @@ protected virtual async Task HandleScreenshot(HttpRequest request) } /// - /// Resizes a PNG image if it exceeds the specified max width, preserving aspect ratio. - /// HiDPI displays (2x, 3x) produce screenshots much larger than needed for AI grounding. + /// Resizes a PNG image based on display density and/or max width constraint. + /// By default, HiDPI screenshots are scaled to 1x logical resolution (e.g., a 3x iPhone + /// screenshot of 1290px becomes 430px). An explicit maxWidth overrides density scaling. /// - private static byte[] ResizePngIfNeeded(byte[] pngData, int? maxWidth) + private static byte[] ResizePngIfNeeded(byte[] pngData, int? maxWidth, double density = 1.0, bool autoScale = true) { - if (maxWidth == null || maxWidth <= 0) return pngData; + // Determine target width: explicit maxWidth takes priority, then auto-scale by density + int? targetWidth = maxWidth; + if (targetWidth == null && autoScale && density > 1.0) + { + try + { + using var probe = SkiaSharp.SKBitmap.Decode(pngData); + if (probe != null) + targetWidth = (int)(probe.Width / density); + } + catch { return pngData; } + } + + if (targetWidth == null || targetWidth <= 0) return pngData; try { using var original = SkiaSharp.SKBitmap.Decode(pngData); - if (original == null || original.Width <= maxWidth.Value) return pngData; + if (original == null || original.Width <= targetWidth.Value) return pngData; - var scale = (float)maxWidth.Value / original.Width; + var scale = (float)targetWidth.Value / original.Width; var newHeight = (int)(original.Height * scale); - using var resized = original.Resize(new SkiaSharp.SKImageInfo(maxWidth.Value, newHeight), SkiaSharp.SKFilterQuality.Medium); + using var resized = original.Resize(new SkiaSharp.SKImageInfo(targetWidth.Value, newHeight), SkiaSharp.SKFilterQuality.Medium); if (resized == null) return pngData; using var image = SkiaSharp.SKImage.FromBitmap(resized); @@ -738,7 +781,6 @@ private static byte[] ResizePngIfNeeded(byte[] pngData, int? maxWidth) } catch { - // If resize fails for any reason, return original unchanged return pngData; } } diff --git a/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs b/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs index 447d812..34d2377 100644 --- a/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs +++ b/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs @@ -17,6 +17,35 @@ public GtkAgentService(AgentOptions? options = null) : base(options) { } protected override string DeviceTypeName => "Virtual"; protected override string IdiomName => "Desktop"; + protected override double GetWindowDisplayDensity(IWindow? window) + { + try + { + // GTK4: get the scale factor from the native Gtk.Window's display/surface + if (window?.Handler?.PlatformView is global::Gtk.Window gtkWindow) + { + var surface = gtkWindow.GetSurface(); + if (surface != null) + return surface.GetScaleFactor(); + } + + // Fallback: walk widget hierarchy to find the Gtk.Window + if (window is Microsoft.Maui.Controls.Window mauiWindow) + { + if (mauiWindow.Page is Shell shell && shell.CurrentPage?.Handler?.PlatformView is global::Gtk.Widget cpWidget) + { + var root = cpWidget.GetRoot(); + if (root is global::Gtk.Widget rootWidget) + return rootWidget.GetScaleFactor(); + } + if (mauiWindow.Page?.Handler?.PlatformView is global::Gtk.Widget pageWidget) + return pageWidget.GetScaleFactor(); + } + } + catch { } + return 1.0; + } + protected override (double width, double height) GetNativeWindowSize(IWindow window) { try diff --git a/src/MauiDevFlow.Agent/DevFlowAgentService.cs b/src/MauiDevFlow.Agent/DevFlowAgentService.cs index ca0a88e..a0ce3d6 100644 --- a/src/MauiDevFlow.Agent/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent/DevFlowAgentService.cs @@ -17,6 +17,42 @@ public PlatformAgentService(AgentOptions? options = null) : base(options) { } protected override VisualTreeWalker CreateTreeWalker() => new PlatformVisualTreeWalker(); + protected override double GetWindowDisplayDensity(IWindow? window) + { + try + { +#if IOS || MACCATALYST + if (window?.Handler?.PlatformView is UIKit.UIWindow uiWindow) + return uiWindow.Screen.Scale; + return UIKit.UIScreen.MainScreen.Scale; +#elif ANDROID + if (window?.Handler?.PlatformView is Android.App.Activity activity) + return activity.Resources?.DisplayMetrics?.Density ?? 1.0; + if (Android.App.Application.Context.Resources?.DisplayMetrics is Android.Util.DisplayMetrics dm) + return dm.Density; + return 1.0; +#elif WINDOWS + if (window?.Handler?.PlatformView is Microsoft.UI.Xaml.Window winuiWindow) + { + var xamlRoot = winuiWindow.Content?.XamlRoot; + if (xamlRoot != null) + return xamlRoot.RasterizationScale; + } + return 1.0; +#elif MACOS + if (window?.Handler?.PlatformView is AppKit.NSWindow nsWindow) + return nsWindow.BackingScaleFactor; + return AppKit.NSScreen.MainScreen?.BackingScaleFactor ?? 2.0; +#else + return base.GetWindowDisplayDensity(window); +#endif + } + catch + { + return base.GetWindowDisplayDensity(window); + } + } + protected override bool TryNativeTap(VisualElement ve) { try diff --git a/src/MauiDevFlow.CLI/Program.cs b/src/MauiDevFlow.CLI/Program.cs index 81bd297..bdad00f 100644 --- a/src/MauiDevFlow.CLI/Program.cs +++ b/src/MauiDevFlow.CLI/Program.cs @@ -307,8 +307,9 @@ static async Task Main(string[] args) var screenshotIdOption = new Option("--id", "Element ID to capture"); var screenshotSelectorOption = new Option("--selector", "CSS selector to capture (first match)"); var screenshotOverwriteOption = new Option("--overwrite", () => false, "Overwrite existing file (default: fail if exists)"); - var screenshotMaxWidthOption = new Option("--max-width", "Resize screenshot to this max width (preserves aspect ratio). Reduces file size for AI agents on HiDPI displays"); - var mauiScreenshotCmd = new Command("screenshot", "Take screenshot") { screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption, screenshotOverwriteOption, screenshotMaxWidthOption }; + var screenshotMaxWidthOption = new Option("--max-width", "Resize screenshot to this max width (overrides auto-scaling)"); + var screenshotScaleOption = new Option("--scale", "Scale mode: 'native' keeps full HiDPI resolution, default auto-scales to 1x logical pixels"); + var mauiScreenshotCmd = new Command("screenshot", "Take screenshot") { screenshotOutputOption, windowOption, screenshotIdOption, screenshotSelectorOption, screenshotOverwriteOption, screenshotMaxWidthOption, screenshotScaleOption }; mauiScreenshotCmd.SetHandler(async (ctx) => { var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; @@ -320,7 +321,8 @@ await MauiScreenshotAsync(host, port, isJson, ctx.ParseResult.GetValueForOption(screenshotIdOption), ctx.ParseResult.GetValueForOption(screenshotSelectorOption), ctx.ParseResult.GetValueForOption(screenshotOverwriteOption), - ctx.ParseResult.GetValueForOption(screenshotMaxWidthOption)); + ctx.ParseResult.GetValueForOption(screenshotMaxWidthOption), + ctx.ParseResult.GetValueForOption(screenshotScaleOption)); }); mauiCommand.Add(mauiScreenshotCmd); @@ -1685,7 +1687,7 @@ private static async Task MauiClearAsync(string host, int port, bool json, strin catch (Exception ex) { OutputWriter.WriteError(ex.Message, json, suggestions: new[] { "Run 'MAUI tree' to refresh element IDs" }); _errorOccurred = true; } } - private static async Task MauiScreenshotAsync(string host, int port, bool json, string? output, int? window, string? id, string? selector, bool overwrite = false, int? maxWidth = null) + private static async Task MauiScreenshotAsync(string host, int port, bool json, string? output, int? window, string? id, string? selector, bool overwrite = false, int? maxWidth = null, string? scale = null) { try { @@ -1698,7 +1700,7 @@ private static async Task MauiScreenshotAsync(string host, int port, bool json, } using var client = new MauiDevFlow.Driver.AgentClient(host, port); - var data = await client.ScreenshotAsync(window, id, selector, maxWidth); + var data = await client.ScreenshotAsync(window, id, selector, maxWidth, scale); if (data == null) { OutputWriter.WriteError("Failed to capture screenshot", json); @@ -1709,13 +1711,14 @@ private static async Task MauiScreenshotAsync(string host, int port, bool json, var fullPath = Path.GetFullPath(filename); if (json) { - OutputWriter.WriteResult(new { path = fullPath, size = data.Length, maxWidth = maxWidth }, json); + OutputWriter.WriteResult(new { path = fullPath, size = data.Length, maxWidth = maxWidth, scale = scale ?? "auto" }, json); } else { var target = id != null ? $" (element: {id})" : selector != null ? $" (selector: {selector})" : ""; - var resized = maxWidth != null ? $" (max-width: {maxWidth}px)" : ""; - Console.WriteLine($"Screenshot saved: {fullPath} ({data.Length} bytes){target}{resized}"); + var scaleInfo = scale?.Equals("native", StringComparison.OrdinalIgnoreCase) == true ? " (native resolution)" : + maxWidth != null ? $" (max-width: {maxWidth}px)" : " (auto-scaled to 1x)"; + Console.WriteLine($"Screenshot saved: {fullPath} ({data.Length} bytes){target}{scaleInfo}"); } } catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } diff --git a/src/MauiDevFlow.Driver/AgentClient.cs b/src/MauiDevFlow.Driver/AgentClient.cs index 1084351..c6bf650 100644 --- a/src/MauiDevFlow.Driver/AgentClient.cs +++ b/src/MauiDevFlow.Driver/AgentClient.cs @@ -146,7 +146,7 @@ public async Task ResizeAsync(int width, int height, int? window = null) /// Take a screenshot (returns PNG bytes). /// Optionally target a specific element by ID or CSS selector. /// - public async Task ScreenshotAsync(int? window = null, string? elementId = null, string? selector = null, int? maxWidth = null) + public async Task ScreenshotAsync(int? window = null, string? elementId = null, string? selector = null, int? maxWidth = null, string? scale = null) { try { @@ -155,6 +155,7 @@ public async Task ResizeAsync(int width, int height, int? window = null) if (elementId != null) queryParams.Add($"id={Uri.EscapeDataString(elementId)}"); if (selector != null) queryParams.Add($"selector={Uri.EscapeDataString(selector)}"); if (maxWidth != null) queryParams.Add($"maxWidth={maxWidth}"); + if (scale != null) queryParams.Add($"scale={Uri.EscapeDataString(scale)}"); var url = queryParams.Count > 0 ? $"{_baseUrl}/api/screenshot?{string.Join("&", queryParams)}" From eda696d7fcdb4816177b55ac4980271790d7319c Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 17:02:00 -0500 Subject: [PATCH 07/12] Fix --and-screenshot triggering on every tap/fill/clear Removed SetDefaultValue(null) from andScreenshotOption. With the default value set, System.CommandLine's FindResultFor() always returned non-null, causing HandlePostActionFlags to take a screenshot after every action regardless of whether --and-screenshot was specified. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/MauiDevFlow.CLI/Program.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MauiDevFlow.CLI/Program.cs b/src/MauiDevFlow.CLI/Program.cs index bdad00f..6873375 100644 --- a/src/MauiDevFlow.CLI/Program.cs +++ b/src/MauiDevFlow.CLI/Program.cs @@ -223,7 +223,6 @@ static async Task Main(string[] args) var resolveTextOption = new Option("--text", "Resolve element by text content"); var resolveIndexOption = new Option("--index", () => 0, "Index when multiple elements match (0-based, default: first)"); var andScreenshotOption = new Option("--and-screenshot", "Take screenshot after action (optional: output path)"); - andScreenshotOption.SetDefaultValue(null); andScreenshotOption.Arity = ArgumentArity.ZeroOrOne; var andTreeOption = new Option("--and-tree", "Dump visual tree after action"); var andTreeDepthOption = new Option("--and-tree-depth", () => 2, "Max depth for --and-tree"); From 0a9805a78ec3a3d539f3f183f2d1fb2758884680 Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 17:03:01 -0500 Subject: [PATCH 08/12] Add Shell navigation, CollectionView, and implicit resolution guidance to SKILL.md Based on ControlGallery testing hurdles (H5-H8, H12-H13): - Shell routes are case-sensitive, discovered via AppShell.xaml - Flyout items use generated IDs, dismissal via FlyoutIsPresented - CollectionView items must be tapped via container, not inner elements - scroll command doesn't work with CollectionView (native scrolling) - --text resolution searches entire tree including hidden pages - Added Shell navigation canonical workflow Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/maui-ai-debugging/SKILL.md | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.claude/skills/maui-ai-debugging/SKILL.md b/.claude/skills/maui-ai-debugging/SKILL.md index a9c6b06..faf15cd 100644 --- a/.claude/skills/maui-ai-debugging/SKILL.md +++ b/.claude/skills/maui-ai-debugging/SKILL.md @@ -443,6 +443,10 @@ their input. - Use `AutomationId` on important MAUI controls for stable element references. - For Blazor Hybrid, `cdp snapshot` is the most AI-friendly way to read page state. - Port discovery, multi-project setup, and custom ports: see [references/setup.md](references/setup.md#3b-port-configuration). +- **Shell apps:** Read `AppShell.xaml` to discover routes before navigating. Routes are + case-sensitive and often lowercase. +- **CollectionView items:** Tap the container Grid/StackLayout, not inner Labels/Images. +- **Ambiguous `--text`:** When text appears on multiple pages, use explicit IDs from `tree`. ## AI Agent Best Practices @@ -517,6 +521,35 @@ the screenshot dimensions accordingly. This happens server-side before transfer. - Prefer `--automationId` for stable references (set in XAML). - Use `maui-devflow commands --json` to discover available commands at runtime. +### Shell Navigation +- **Routes are case-sensitive** and come from `ShellContent Route=""` in XAML, not from + `FlyoutItem Title`. Discover routes by reading `AppShell.xaml`: + ```bash + grep -i 'Route=' AppShell.xaml + ``` +- **Flyout menu items** use generated IDs like `FlyoutItem_D_FAULT_FlyoutItem0`. Find them + at the top level of the tree output. Don't try to tap Labels inside flyout items. +- **Flyout dismissal:** After tapping a flyout item, the flyout may stay open. Dismiss with: + ```bash + maui-devflow MAUI set-property FlyoutIsPresented "false" + ``` + +### CollectionView / ListView +- **Tapping items:** Always tap the item's container (Grid/StackLayout), not inner elements + (Label/Image). The item template's root element handles selection. +- **Scrolling:** `MAUI scroll` uses MAUI's `ScrollView.ScrollToAsync` and does **not** work + with `CollectionView` or `ListView` (which use native platform scrolling). Items off-screen + appear in the tree with `0x0` or `-1x-1` bounds — tap them by ID regardless (the platform + will scroll to make them visible on tap if the app handles `SelectionChanged`). + +### Implicit Resolution Gotchas +- **`--text` searches the entire visual tree**, including hidden pages (other Shell tabs). + If the text is ambiguous (e.g., `"+"`, `"OK"`, `"Cancel"`), it may match a wrong element + on a different page. +- **Prefer `--automationId`** for reliable targeting. Fall back to explicit element IDs from + `tree`/`query` for elements without AutomationIds. +- **Use `--type` + `--text` together** to narrow matches when text alone is ambiguous. + ### Canonical Workflows **Login flow:** @@ -528,6 +561,17 @@ maui-devflow MAUI tap --automationId "LoginButton" --and-screenshot maui-devflow MAUI query --automationId "HomePage" --wait-until exists --timeout 10 ``` +**Shell navigation:** +```bash +# Discover routes from XAML +grep -i 'Route=' AppShell.xaml # find route names +maui-devflow MAUI navigate "//home" # navigate to a route +maui-devflow MAUI tap FlyoutButton # open flyout +maui-devflow MAUI tree --depth 3 --fields "id,type,text" # find flyout items +maui-devflow MAUI tap # tap item +maui-devflow MAUI set-property FlyoutIsPresented "false" # dismiss flyout +``` + **Element inspection:** ```bash maui-devflow MAUI query --automationId "MyControl" --json --fields "id,type,text,bounds" From 883ab88ac6b13a83bfde1eaca2e4a5c255ace98a Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 18:06:59 -0500 Subject: [PATCH 09/12] Add comprehensive scroll support for CollectionView/ListView Completely rewrites HandleScroll with multi-tier resolution: 1. Item-index scrolling: --item-index N scrolls to a specific item in CollectionView/ListView, even virtualized off-screen items 2. Smart scroll-into-view: elements inside an ItemsView find their data item via BindingContext and scroll to the correct index 3. Platform-native delta scroll: TryNativeScroll() virtual method with overrides for iOS (UIScrollView), Android (RecyclerView), Windows (ScrollViewer), and GTK (ScrolledWindow) 4. ScrollView fallback: existing behavior preserved Also adds: - itemCount in tree metadata for ItemsView elements (via NativeProperties) - --item-index, --group-index, --position CLI options - ScrollToPosition support (MakeVisible/Start/Center/End) - Updated SKILL.md with CollectionView scrolling guidance Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .claude/skills/maui-ai-debugging/SKILL.md | 26 ++- .../DevFlowAgentService.cs | 187 +++++++++++++++--- .../VisualTreeWalker.cs | 19 ++ src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs | 33 ++++ src/MauiDevFlow.Agent/DevFlowAgentService.cs | 109 ++++++++++ src/MauiDevFlow.CLI/Program.cs | 24 ++- src/MauiDevFlow.Driver/AgentClient.cs | 6 +- 7 files changed, 364 insertions(+), 40 deletions(-) diff --git a/.claude/skills/maui-ai-debugging/SKILL.md b/.claude/skills/maui-ai-debugging/SKILL.md index faf15cd..4a720ca 100644 --- a/.claude/skills/maui-ai-debugging/SKILL.md +++ b/.claude/skills/maui-ai-debugging/SKILL.md @@ -330,7 +330,7 @@ resolution options are provided. | `MAUI set-property ` | Set property (live editing — colors, text, sizes, etc.) | | `MAUI element ` | Full element JSON (type, bounds, children, etc.) | | `MAUI navigate ` | Shell navigation (e.g. `//native`, `//blazor`) | -| `MAUI scroll [--element id] [--dx N] [--dy N] [--window W]` | Scroll by delta or scroll element into view | +| `MAUI scroll [--element id] [--dx N] [--dy N] [--item-index N] [--group-index N] [--position P] [--window W]` | Scroll by delta, item index, or scroll element into view. `--item-index` scrolls to a specific item in CollectionView/ListView (works even for virtualized off-screen items). `--position`: MakeVisible (default), Start, Center, End. Delta scroll (`--dy -500`) uses native platform scroll for CollectionView | | `MAUI resize [--window W]` | Resize app window. Window is 0-based index; default first window | | `MAUI logs [--limit N] [--skip N] [--source S] [--follow]` | Fetch or stream application logs. `--follow` / `-f` streams in real-time (Ctrl+C to stop). Source: native, webview, or omit for all | | `MAUI recording start [--output path] [--timeout 30]` | Start screen recording. Default timeout 30s | @@ -446,6 +446,7 @@ their input. - **Shell apps:** Read `AppShell.xaml` to discover routes before navigating. Routes are case-sensitive and often lowercase. - **CollectionView items:** Tap the container Grid/StackLayout, not inner Labels/Images. + Use `--item-index` to scroll to off-screen items. - **Ambiguous `--text`:** When text appears on multiple pages, use explicit IDs from `tree`. ## AI Agent Best Practices @@ -537,10 +538,25 @@ the screenshot dimensions accordingly. This happens server-side before transfer. ### CollectionView / ListView - **Tapping items:** Always tap the item's container (Grid/StackLayout), not inner elements (Label/Image). The item template's root element handles selection. -- **Scrolling:** `MAUI scroll` uses MAUI's `ScrollView.ScrollToAsync` and does **not** work - with `CollectionView` or `ListView` (which use native platform scrolling). Items off-screen - appear in the tree with `0x0` or `-1x-1` bounds — tap them by ID regardless (the platform - will scroll to make them visible on tap if the app handles `SelectionChanged`). +- **Virtualization:** CollectionView/ListView use item virtualization — only visible items + (plus a small buffer) exist in the visual tree. Off-screen items have NO visual element. + The tree shows `itemCount` in the CollectionView's properties so you know total items. +- **Scrolling by item index** (best for reaching off-screen items): + ```bash + maui-devflow MAUI scroll --element --item-index 20 --position Center + ``` + This works even for items not in the tree yet — the platform scrolls to materialize them. +- **Scrolling by pixel delta** (for fine-grained scrolling): + ```bash + maui-devflow MAUI scroll --element --dy -500 + ``` + Uses native platform scroll (UIScrollView/RecyclerView) — works on CollectionView. +- **Workflow:** Get tree → note `itemCount` → scroll by index → re-query tree → interact: + ```bash + maui-devflow MAUI tree --depth 15 # CollectionView shows itemCount: 25 + maui-devflow MAUI scroll --item-index 20 + maui-devflow MAUI tree --depth 15 # items around index 20 now visible + ``` ### Implicit Resolution Gotchas - **`--text` searches the entire visual tree**, including hidden pages (other Shell tabs). diff --git a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs index 1b72049..38fdd41 100644 --- a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs @@ -1306,9 +1306,43 @@ private async Task HandleScroll(HttpRequest request) if (body == null) return HttpResponse.Error("Request body is required"); + var position = ParseScrollToPosition(body.ScrollToPosition); + var result = await DispatchAsync(async () => { - // If elementId is given, find the nearest ScrollView ancestor and scroll to that element + // Priority 1: Scroll by item index on a specific ItemsView + if (body.ItemIndex.HasValue) + { + object? targetObj = null; + if (!string.IsNullOrEmpty(body.ElementId)) + { + targetObj = _treeWalker.GetElementById(body.ElementId, _app); + if (targetObj == null) return "Element not found"; + } + + // Find the ItemsView — either the target itself or its ancestor + var itemsView = targetObj as ItemsView ?? (targetObj is VisualElement tve ? FindAncestor(tve) : null); + // Since ListView inherits from ItemsView in .NET 10+, ItemsView check covers both + if (itemsView == null && targetObj == null) + { + // No element specified — find first ItemsView on the page + var window = GetWindow(ParseWindowIndex(request)); + if (window?.Page != null) + itemsView = FindDescendant(window.Page); + } + + if (itemsView != null) + { + await ScrollWithTimeoutAsync( + () => { itemsView.ScrollTo(body.ItemIndex.Value, body.GroupIndex ?? -1, position, body.Animated); return Task.CompletedTask; }, + () => { itemsView.ScrollTo(body.ItemIndex.Value, body.GroupIndex ?? -1, position, false); return Task.CompletedTask; }); + return "ok"; + } + + return "No CollectionView or ListView found for item-index scroll"; + } + + // Priority 2: Scroll element into view if (!string.IsNullOrEmpty(body.ElementId)) { var el = _treeWalker.GetElementById(body.ElementId, _app); @@ -1316,49 +1350,151 @@ private async Task HandleScroll(HttpRequest request) if (el is VisualElement ve) { - // Find the nearest ScrollView ancestor + // 2a: Check for ItemsView ancestor — use BindingContext to find item index + var ancestorItemsView = FindAncestor(ve); + if (ancestorItemsView != null && ve.BindingContext != null) + { + var index = GetItemIndex(ancestorItemsView.ItemsSource, ve.BindingContext); + if (index >= 0) + { + await ScrollWithTimeoutAsync( + () => { ancestorItemsView.ScrollTo(index, position: position, animate: body.Animated); return Task.CompletedTask; }, + () => { ancestorItemsView.ScrollTo(index, position: position, animate: false); return Task.CompletedTask; }); + return "ok"; + } + } + + // 2b: Check for ScrollView ancestor (existing behavior) var scrollView = FindAncestor(ve); if (scrollView != null) { await ScrollWithTimeoutAsync( - () => scrollView.ScrollToAsync(ve, ScrollToPosition.MakeVisible, body.Animated), - () => scrollView.ScrollToAsync(ve, ScrollToPosition.MakeVisible, false)); + () => scrollView.ScrollToAsync(ve, (ScrollToPosition)position, body.Animated), + () => scrollView.ScrollToAsync(ve, (ScrollToPosition)position, false)); return "ok"; } - // Maybe the element itself is a ScrollView - if (el is ScrollView sv) + // 2d: Element is itself a scrollable view — apply delta + if (el is ScrollView sv && (body.DeltaX != 0 || body.DeltaY != 0)) { + var newX = Math.Max(0, sv.ScrollX + body.DeltaX); + var newY = Math.Max(0, sv.ScrollY + body.DeltaY); await ScrollWithTimeoutAsync( - () => sv.ScrollToAsync(body.DeltaX, body.DeltaY, body.Animated), - () => sv.ScrollToAsync(body.DeltaX, body.DeltaY, false)); + () => sv.ScrollToAsync(newX, newY, body.Animated), + () => sv.ScrollToAsync(newX, newY, false)); return "ok"; } + + // 2e: Element is an ItemsView — apply delta via native scroll + if (el is ItemsView && (body.DeltaX != 0 || body.DeltaY != 0)) + { + if (await TryNativeScroll(ve, body.DeltaX, body.DeltaY)) + return "ok"; + return $"Native scroll not supported on this platform for {el.GetType().Name}"; + } + + // 2f: Try native scroll as final fallback + if (body.DeltaX != 0 || body.DeltaY != 0) + { + if (await TryNativeScroll(ve, body.DeltaX, body.DeltaY)) + return "ok"; + } } - return $"No ScrollView ancestor found for element '{body.ElementId}'"; + return $"No scrollable ancestor found for element '{body.ElementId}'"; + } + + // Priority 3: Delta scroll with no element — find first scrollable on page + var pageWindow = GetWindow(ParseWindowIndex(request)); + if (pageWindow?.Page == null) return "No page available"; + + // 3a: Try ScrollView first + var targetScroll = FindDescendant(pageWindow.Page); + if (targetScroll != null) + { + var newX = targetScroll.ScrollX + body.DeltaX; + var newY = targetScroll.ScrollY + body.DeltaY; + var x = Math.Max(0, newX); + var y = Math.Max(0, newY); + await ScrollWithTimeoutAsync( + () => targetScroll.ScrollToAsync(x, y, body.Animated), + () => targetScroll.ScrollToAsync(x, y, false)); + return "ok"; + } + + // 3b: Try ItemsView via native scroll (covers CollectionView and ListView) + var targetItemsView = FindDescendant(pageWindow.Page); + if (targetItemsView is VisualElement ive) + { + if (await TryNativeScroll(ive, body.DeltaX, body.DeltaY)) + return "ok"; } - // Otherwise scroll by delta on the first ScrollView we find - var window = GetWindow(ParseWindowIndex(request)); - if (window?.Page == null) return "No page available"; - - var targetScroll = FindDescendant(window.Page); - if (targetScroll == null) return "No ScrollView found on page"; - - var newX = targetScroll.ScrollX + body.DeltaX; - var newY = targetScroll.ScrollY + body.DeltaY; - var x = Math.Max(0, newX); - var y = Math.Max(0, newY); - await ScrollWithTimeoutAsync( - () => targetScroll.ScrollToAsync(x, y, body.Animated), - () => targetScroll.ScrollToAsync(x, y, false)); - return "ok"; + return "No scrollable view found on page"; }); return result == "ok" ? HttpResponse.Ok("Scrolled") : HttpResponse.Error(result ?? "Scroll failed"); } + /// + /// Parse a ScrollToPosition string to the MAUI enum value. + /// + private static ScrollToPosition ParseScrollToPosition(string? value) + { + if (string.IsNullOrEmpty(value)) return ScrollToPosition.MakeVisible; + return value.ToLowerInvariant() switch + { + "start" => ScrollToPosition.Start, + "center" => ScrollToPosition.Center, + "end" => ScrollToPosition.End, + "makevisible" => ScrollToPosition.MakeVisible, + _ => ScrollToPosition.MakeVisible + }; + } + + /// + /// Get item from an IEnumerable by index. + /// + private static object? GetItemByIndex(System.Collections.IEnumerable? source, int index) + { + if (source == null) return null; + if (source is System.Collections.IList list && index >= 0 && index < list.Count) + return list[index]; + var i = 0; + foreach (var item in source) + { + if (i == index) return item; + i++; + } + return null; + } + + /// + /// Find the index of an item in an IEnumerable by reference or equality. + /// + private static int GetItemIndex(System.Collections.IEnumerable? source, object item) + { + if (source == null) return -1; + if (source is System.Collections.IList list) + return list.IndexOf(item); + var i = 0; + foreach (var obj in source) + { + if (ReferenceEquals(obj, item) || Equals(obj, item)) return i; + i++; + } + return -1; + } + + /// + /// Try to scroll a native view by pixel delta. Override in platform-specific subclasses. + /// Returns true if the scroll was handled natively. + /// + protected virtual Task TryNativeScroll(VisualElement element, double deltaX, double deltaY) + { + return Task.FromResult(false); + } + /// /// Animated ScrollToAsync can deadlock on iOS when dispatched. /// Fall back to non-animated scroll if the animated version doesn't complete in time. @@ -1787,4 +1923,7 @@ public class ScrollRequest public double DeltaX { get; set; } public double DeltaY { get; set; } public bool Animated { get; set; } = true; + public int? ItemIndex { get; set; } + public int? GroupIndex { get; set; } + public string? ScrollToPosition { get; set; } } diff --git a/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs b/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs index c7e93fc..edf0316 100644 --- a/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs +++ b/src/MauiDevFlow.Agent.Core/VisualTreeWalker.cs @@ -1278,6 +1278,25 @@ private ElementInfo CreateElementInfo(IVisualTreeElement element, string id, str info.Gestures = gestures; } + // Populate ItemsView metadata (item count for CollectionView/ListView/CarouselView) + if (element is ItemsView itemsView) + { + info.NativeProperties ??= new Dictionary(); + try + { + if (itemsView.ItemsSource != null) + { + var count = itemsView.ItemsSource switch + { + System.Collections.ICollection c => c.Count, + _ => itemsView.ItemsSource.Cast().Count() + }; + info.NativeProperties["itemCount"] = count.ToString(); + } + } + catch { /* ItemsSource may not support counting */ } + } + return info; } diff --git a/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs b/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs index 34d2377..5bf8d47 100644 --- a/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs +++ b/src/MauiDevFlow.Agent.Gtk/GtkAgentService.cs @@ -77,6 +77,39 @@ protected override (double width, double height) GetNativeWindowSize(IWindow win return base.GetNativeWindowSize(window); } + protected override Task TryNativeScroll(VisualElement element, double deltaX, double deltaY) + { + try + { + var target = element; + while (target != null) + { + if (target.Handler?.PlatformView is global::Gtk.Widget widget) + { + // Walk up GTK widget hierarchy looking for ScrolledWindow + var current = widget; + while (current != null) + { + if (current is global::Gtk.ScrolledWindow scrolledWindow) + { + var hAdj = scrolledWindow.GetHadjustment(); + var vAdj = scrolledWindow.GetVadjustment(); + if (hAdj != null && deltaX != 0) + hAdj.SetValue(Math.Max(hAdj.GetLower(), Math.Min(hAdj.GetValue() + deltaX, hAdj.GetUpper() - hAdj.GetPageSize()))); + if (vAdj != null && deltaY != 0) + vAdj.SetValue(Math.Max(vAdj.GetLower(), Math.Min(vAdj.GetValue() - deltaY, vAdj.GetUpper() - vAdj.GetPageSize()))); + return Task.FromResult(true); + } + current = current.GetParent() as global::Gtk.Widget; + } + } + target = target.Parent as VisualElement; + } + } + catch { } + return Task.FromResult(false); + } + protected override bool TryNativeTap(VisualElement ve) { try diff --git a/src/MauiDevFlow.Agent/DevFlowAgentService.cs b/src/MauiDevFlow.Agent/DevFlowAgentService.cs index a0ce3d6..10603e1 100644 --- a/src/MauiDevFlow.Agent/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent/DevFlowAgentService.cs @@ -53,6 +53,115 @@ protected override double GetWindowDisplayDensity(IWindow? window) } } + protected override Task TryNativeScroll(VisualElement element, double deltaX, double deltaY) + { + try + { + // Walk up from the element to find a native scrollable view + var target = element; + while (target != null) + { + var platformView = target.Handler?.PlatformView; + if (platformView != null) + { +#if IOS || MACCATALYST + // Find UIScrollView in the native hierarchy + var uiScrollView = FindNativeAncestor(platformView as UIKit.UIView); + if (uiScrollView != null) + { + var offset = uiScrollView.ContentOffset; + var newX = Math.Max(0, Math.Min(offset.X + deltaX, uiScrollView.ContentSize.Width - uiScrollView.Bounds.Width)); + var newY = Math.Max(0, Math.Min(offset.Y - deltaY, uiScrollView.ContentSize.Height - uiScrollView.Bounds.Height)); + uiScrollView.SetContentOffset(new CoreGraphics.CGPoint(newX, newY), animated: true); + return Task.FromResult(true); + } +#elif ANDROID + // Find RecyclerView or ScrollView in the native hierarchy + var recyclerView = FindNativeAncestorAndroid(platformView as Android.Views.View); + if (recyclerView != null) + { + recyclerView.ScrollBy((int)deltaX, (int)-deltaY); + return Task.FromResult(true); + } + var androidScrollView = FindNativeAncestorAndroid(platformView as Android.Views.View); + if (androidScrollView != null) + { + androidScrollView.ScrollBy((int)deltaX, (int)-deltaY); + return Task.FromResult(true); + } +#elif WINDOWS + // Find ScrollViewer in the WinUI XAML tree + var scrollViewer = FindWinUIScrollViewer(platformView as Microsoft.UI.Xaml.DependencyObject); + if (scrollViewer != null) + { + scrollViewer.ChangeView( + scrollViewer.HorizontalOffset + deltaX, + scrollViewer.VerticalOffset - deltaY, + null); + return Task.FromResult(true); + } +#endif + } + target = target.Parent as VisualElement; + } + } + catch { } + return Task.FromResult(false); + } + +#if IOS || MACCATALYST + private static T? FindNativeAncestor(UIKit.UIView? view) where T : UIKit.UIView + { + var current = view; + while (current != null) + { + if (current is T match) return match; + current = current.Superview; + } + return null; + } +#elif ANDROID + private static T? FindNativeAncestorAndroid(Android.Views.View? view) where T : Android.Views.View + { + var current = view; + while (current != null) + { + if (current is T match) return match; + current = current.Parent as Android.Views.View; + } + return null; + } +#elif WINDOWS + private static Microsoft.UI.Xaml.Controls.ScrollViewer? FindWinUIScrollViewer(Microsoft.UI.Xaml.DependencyObject? obj) + { + if (obj == null) return null; + if (obj is Microsoft.UI.Xaml.Controls.ScrollViewer sv) return sv; + // Walk up the visual tree + var parent = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetParent(obj); + while (parent != null) + { + if (parent is Microsoft.UI.Xaml.Controls.ScrollViewer scrollViewer) + return scrollViewer; + parent = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetParent(parent); + } + // Also search children (CollectionView wraps a ScrollViewer internally) + return FindWinUIScrollViewerInChildren(obj); + } + + private static Microsoft.UI.Xaml.Controls.ScrollViewer? FindWinUIScrollViewerInChildren(Microsoft.UI.Xaml.DependencyObject parent) + { + var count = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetChildrenCount(parent); + for (var i = 0; i < count; i++) + { + var child = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetChild(parent, i); + if (child is Microsoft.UI.Xaml.Controls.ScrollViewer sv) return sv; + var found = FindWinUIScrollViewerInChildren(child); + if (found != null) return found; + } + return null; + } +#endif + protected override bool TryNativeTap(VisualElement ve) { try diff --git a/src/MauiDevFlow.CLI/Program.cs b/src/MauiDevFlow.CLI/Program.cs index 6873375..4732f99 100644 --- a/src/MauiDevFlow.CLI/Program.cs +++ b/src/MauiDevFlow.CLI/Program.cs @@ -376,11 +376,14 @@ await RecordingStopAsync(host, port, platform), mauiCommand.Add(mauiNavigateCmd); // MAUI scroll - var scrollElementIdOption = new Option("--element", "Element ID to scroll into view"); - var scrollDeltaXOption = new Option("--dx", () => 0, "Horizontal scroll delta"); - var scrollDeltaYOption = new Option("--dy", () => 0, "Vertical scroll delta"); + var scrollElementIdOption = new Option("--element", "Element ID to scroll into view or to scroll within"); + var scrollDeltaXOption = new Option("--dx", () => 0, "Horizontal scroll delta (pixels)"); + var scrollDeltaYOption = new Option("--dy", () => 0, "Vertical scroll delta (pixels, negative = down)"); var scrollAnimatedOption = new Option("--animated", () => true, "Animate the scroll"); - var mauiScrollCmd = new Command("scroll", "Scroll content by delta or scroll element into view") { scrollElementIdOption, scrollDeltaXOption, scrollDeltaYOption, scrollAnimatedOption, windowOption }; + var scrollItemIndexOption = new Option("--item-index", "Item index to scroll to (for CollectionView/ListView)"); + var scrollGroupIndexOption = new Option("--group-index", "Group index for grouped CollectionView"); + var scrollPositionOption = new Option("--position", "Scroll position: MakeVisible (default), Start, Center, End"); + var mauiScrollCmd = new Command("scroll", "Scroll content by delta, item index, or scroll element into view") { scrollElementIdOption, scrollDeltaXOption, scrollDeltaYOption, scrollAnimatedOption, scrollItemIndexOption, scrollGroupIndexOption, scrollPositionOption, windowOption }; mauiScrollCmd.SetHandler(async (ctx) => { var host = ctx.ParseResult.GetValueForOption(agentHostOption)!; @@ -391,7 +394,10 @@ await MauiScrollAsync(host, port, isJson, ctx.ParseResult.GetValueForOption(scrollDeltaXOption), ctx.ParseResult.GetValueForOption(scrollDeltaYOption), ctx.ParseResult.GetValueForOption(scrollAnimatedOption), - ctx.ParseResult.GetValueForOption(windowOption)); + ctx.ParseResult.GetValueForOption(windowOption), + ctx.ParseResult.GetValueForOption(scrollItemIndexOption), + ctx.ParseResult.GetValueForOption(scrollGroupIndexOption), + ctx.ParseResult.GetValueForOption(scrollPositionOption)); }); mauiCommand.Add(mauiScrollCmd); @@ -1839,19 +1845,21 @@ private static async Task MauiNavigateAsync(string host, int port, bool json, st catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task MauiScrollAsync(string host, int port, bool json, string? elementId, double dx, double dy, bool animated, int? window) + private static async Task MauiScrollAsync(string host, int port, bool json, string? elementId, double dx, double dy, bool animated, int? window, int? itemIndex = null, int? groupIndex = null, string? scrollToPosition = null) { try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); - var success = await client.ScrollAsync(elementId, dx, dy, animated, window); + var success = await client.ScrollAsync(elementId, dx, dy, animated, window, itemIndex, groupIndex, scrollToPosition); if (json) { OutputWriter.WriteActionResult(success, "Scrolled", elementId, json); } else { - if (elementId != null) + if (itemIndex.HasValue) + Console.WriteLine(success ? $"Scrolled to item index {itemIndex.Value}" : $"Failed to scroll to item index {itemIndex.Value}"); + else if (elementId != null) Console.WriteLine(success ? $"Scrolled to element: {elementId}" : $"Failed to scroll to element: {elementId}"); else Console.WriteLine(success ? $"Scrolled by dx={dx}, dy={dy}" : "Failed to scroll"); diff --git a/src/MauiDevFlow.Driver/AgentClient.cs b/src/MauiDevFlow.Driver/AgentClient.cs index c6bf650..6f34918 100644 --- a/src/MauiDevFlow.Driver/AgentClient.cs +++ b/src/MauiDevFlow.Driver/AgentClient.cs @@ -123,13 +123,13 @@ public async Task NavigateAsync(string route) } /// - /// Scroll by delta or scroll an element into view. + /// Scroll by delta, item index, or scroll element into view. /// - public async Task ScrollAsync(string? elementId = null, double deltaX = 0, double deltaY = 0, bool animated = true, int? window = null) + public async Task ScrollAsync(string? elementId = null, double deltaX = 0, double deltaY = 0, bool animated = true, int? window = null, int? itemIndex = null, int? groupIndex = null, string? scrollToPosition = null) { var url = "/api/action/scroll"; if (window != null) url += $"?window={window}"; - return await PostActionAsync(url, new { elementId, deltaX, deltaY, animated }); + return await PostActionAsync(url, new { elementId, deltaX, deltaY, animated, itemIndex, groupIndex, scrollToPosition }); } /// From 0579c1757ce54f1032d341163a3422b4285fd5bb Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 18:13:32 -0500 Subject: [PATCH 10/12] Fix native scroll: search subviews not just ancestors CollectionView's handler wraps the actual UICollectionView (UIScrollView) inside a container UIView. The native scroll search must check the view itself, then search subviews (descendants), then ancestors. Applied the same fix to Android (RecyclerView may be a child of the handler's platform view) and Windows (already had descendant search via FindWinUIScrollViewerInChildren, now consolidated to generic FindWinUIDescendant). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/MauiDevFlow.Agent/DevFlowAgentService.cs | 72 ++++++++++++++++---- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/src/MauiDevFlow.Agent/DevFlowAgentService.cs b/src/MauiDevFlow.Agent/DevFlowAgentService.cs index 10603e1..c991b0b 100644 --- a/src/MauiDevFlow.Agent/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent/DevFlowAgentService.cs @@ -65,8 +65,13 @@ protected override Task TryNativeScroll(VisualElement element, double delt if (platformView != null) { #if IOS || MACCATALYST - // Find UIScrollView in the native hierarchy - var uiScrollView = FindNativeAncestor(platformView as UIKit.UIView); + // Check: view itself → subviews → ancestors + var uiView = platformView as UIKit.UIView; + UIKit.UIScrollView? uiScrollView = uiView as UIKit.UIScrollView; + if (uiScrollView == null) + uiScrollView = FindNativeDescendant(uiView); + if (uiScrollView == null) + uiScrollView = FindNativeAncestor(uiView); if (uiScrollView != null) { var offset = uiScrollView.ContentOffset; @@ -76,22 +81,36 @@ protected override Task TryNativeScroll(VisualElement element, double delt return Task.FromResult(true); } #elif ANDROID - // Find RecyclerView or ScrollView in the native hierarchy - var recyclerView = FindNativeAncestorAndroid(platformView as Android.Views.View); + // Check: view itself → descendants → ancestors + var androidView = platformView as Android.Views.View; + var recyclerView = androidView as AndroidX.RecyclerView.Widget.RecyclerView; + if (recyclerView == null) + recyclerView = FindNativeDescendantAndroid(androidView); + if (recyclerView == null) + recyclerView = FindNativeAncestorAndroid(androidView); if (recyclerView != null) { recyclerView.ScrollBy((int)deltaX, (int)-deltaY); return Task.FromResult(true); } - var androidScrollView = FindNativeAncestorAndroid(platformView as Android.Views.View); + var androidScrollView = androidView as Android.Widget.ScrollView; + if (androidScrollView == null) + androidScrollView = FindNativeDescendantAndroid(androidView); + if (androidScrollView == null) + androidScrollView = FindNativeAncestorAndroid(androidView); if (androidScrollView != null) { androidScrollView.ScrollBy((int)deltaX, (int)-deltaY); return Task.FromResult(true); } #elif WINDOWS - // Find ScrollViewer in the WinUI XAML tree - var scrollViewer = FindWinUIScrollViewer(platformView as Microsoft.UI.Xaml.DependencyObject); + // Check: view itself → descendants → ancestors + var winView = platformView as Microsoft.UI.Xaml.DependencyObject; + var scrollViewer = winView as Microsoft.UI.Xaml.Controls.ScrollViewer; + if (scrollViewer == null) + scrollViewer = FindWinUIDescendant(winView); + if (scrollViewer == null) + scrollViewer = FindWinUIScrollViewer(winView); if (scrollViewer != null) { scrollViewer.ChangeView( @@ -120,6 +139,18 @@ protected override Task TryNativeScroll(VisualElement element, double delt } return null; } + + private static T? FindNativeDescendant(UIKit.UIView? view) where T : UIKit.UIView + { + if (view == null) return null; + if (view is T match) return match; + foreach (var subview in view.Subviews) + { + var found = FindNativeDescendant(subview); + if (found != null) return found; + } + return null; + } #elif ANDROID private static T? FindNativeAncestorAndroid(Android.Views.View? view) where T : Android.Views.View { @@ -131,6 +162,21 @@ protected override Task TryNativeScroll(VisualElement element, double delt } return null; } + + private static T? FindNativeDescendantAndroid(Android.Views.View? view) where T : Android.Views.View + { + if (view == null) return null; + if (view is T match) return match; + if (view is Android.Views.ViewGroup vg) + { + for (var i = 0; i < vg.ChildCount; i++) + { + var found = FindNativeDescendantAndroid(vg.GetChildAt(i)); + if (found != null) return found; + } + } + return null; + } #elif WINDOWS private static Microsoft.UI.Xaml.Controls.ScrollViewer? FindWinUIScrollViewer(Microsoft.UI.Xaml.DependencyObject? obj) { @@ -145,18 +191,20 @@ protected override Task TryNativeScroll(VisualElement element, double delt parent = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetParent(parent); } // Also search children (CollectionView wraps a ScrollViewer internally) - return FindWinUIScrollViewerInChildren(obj); + return FindWinUIDescendant(obj); } - private static Microsoft.UI.Xaml.Controls.ScrollViewer? FindWinUIScrollViewerInChildren(Microsoft.UI.Xaml.DependencyObject parent) + private static T? FindWinUIDescendant(Microsoft.UI.Xaml.DependencyObject? parent) where T : Microsoft.UI.Xaml.DependencyObject { + if (parent == null) return null; + if (parent is T match) return match; var count = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetChildrenCount(parent); for (var i = 0; i < count; i++) { var child = Microsoft.UI.Xaml.Media.VisualTreeHelper.GetChild(parent, i); - if (child is Microsoft.UI.Xaml.Controls.ScrollViewer sv) return sv; - var found = FindWinUIScrollViewerInChildren(child); - if (found != null) return found; + if (child is T found) return found; + var descendant = FindWinUIDescendant(child); + if (descendant != null) return descendant; } return null; } From 03bcada797bc6b85ec30ceb477ac045714bcc564 Mon Sep 17 00:00:00 2001 From: redth Date: Thu, 5 Mar 2026 18:21:43 -0500 Subject: [PATCH 11/12] Fix no-element scroll: use current page, prioritize ItemsView When no element is specified for delta scroll, the handler searched the entire Shell page tree (all tabs) for scrollable views. This found ScrollViews on hidden pages (e.g., Home) instead of the CollectionView on the active page. Fix: Use Shell.CurrentPage to scope the search to the visible page, and check ItemsView before ScrollView since CollectionView/ListView are more common scroll targets in modern MAUI apps. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DevFlowAgentService.cs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs index 38fdd41..850d35e 100644 --- a/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs +++ b/src/MauiDevFlow.Agent.Core/DevFlowAgentService.cs @@ -1404,12 +1404,23 @@ await ScrollWithTimeoutAsync( return $"No scrollable ancestor found for element '{body.ElementId}'"; } - // Priority 3: Delta scroll with no element — find first scrollable on page + // Priority 3: Delta scroll with no element — find first scrollable on current page var pageWindow = GetWindow(ParseWindowIndex(request)); if (pageWindow?.Page == null) return "No page available"; - // 3a: Try ScrollView first - var targetScroll = FindDescendant(pageWindow.Page); + // Use the current visible page (Shell.CurrentPage or the window page) + var currentPage = (pageWindow.Page as Shell)?.CurrentPage ?? pageWindow.Page; + + // 3a: Try ItemsView via native scroll first (CollectionView/ListView are more common scroll targets) + var targetItemsView = FindDescendant(currentPage); + if (targetItemsView is VisualElement ive) + { + if (await TryNativeScroll(ive, body.DeltaX, body.DeltaY)) + return "ok"; + } + + // 3b: Try ScrollView on current page + var targetScroll = FindDescendant(currentPage); if (targetScroll != null) { var newX = targetScroll.ScrollX + body.DeltaX; @@ -1422,14 +1433,6 @@ await ScrollWithTimeoutAsync( return "ok"; } - // 3b: Try ItemsView via native scroll (covers CollectionView and ListView) - var targetItemsView = FindDescendant(pageWindow.Page); - if (targetItemsView is VisualElement ive) - { - if (await TryNativeScroll(ive, body.DeltaX, body.DeltaY)) - return "ok"; - } - return "No scrollable view found on page"; }); From 17b86ecc8a80338c38ddec87536908ed590d4de6 Mon Sep 17 00:00:00 2001 From: redth Date: Sat, 7 Mar 2026 17:23:45 -0500 Subject: [PATCH 12/12] Fix iOS dialog detection regression with multiple agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alert commands (detect, dismiss, tree) previously defined their own --agent-port and --platform options with hardcoded defaults (port 9223, platform 'auto'). With the broker-based port assignment, port 9223 could belong to a different agent (e.g. Mac Catalyst), causing platform auto-detection to return 'maccatalyst' instead of 'ios-simulator'. Changes: - Alert commands now use global --agent-host and --agent-port options (broker-aware port resolution) - ResolveAlertPlatformAsync no longer takes a platform parameter; instead it intelligently detects based on: udid → iOS simulator, pid → Mac Catalyst/Windows, agent status, booted simulator check - When the connected agent reports MacCatalyst but a booted iOS simulator exists, prefer iOS simulator (the common alert use case) - Removed duplicate local option definitions (15 options → 6) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/MauiDevFlow.CLI/Program.cs | 92 ++++++++++++------- src/SampleMauiApp.MacOS/MauiProgram.cs | 2 +- .../SampleMauiApp.MacOS.csproj | 2 + 3 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/MauiDevFlow.CLI/Program.cs b/src/MauiDevFlow.CLI/Program.cs index 4732f99..1e6e046 100644 --- a/src/MauiDevFlow.CLI/Program.cs +++ b/src/MauiDevFlow.CLI/Program.cs @@ -433,35 +433,26 @@ await MauiScrollAsync(host, port, isJson, // detect var detectUdid = new Option("--udid", "Simulator UDID (auto-detects booted simulator if omitted)"); var detectPid = new Option("--pid", "Mac Catalyst app PID (auto-detects if omitted)"); - var detectPlatform = new Option("--platform", () => "auto", "Platform: maccatalyst, ios, android, windows, or auto"); - var detectHost = new Option("--agent-host", () => "localhost", "Agent HTTP host"); - var detectPort = new Option("--agent-port", () => 9223, "Agent HTTP port"); - var alertDetectCmd = new Command("detect", "Check if an alert/dialog is visible") { detectUdid, detectPid, detectPlatform, detectHost, detectPort }; - alertDetectCmd.SetHandler(async (udid, pid, platform, host, port, json, noJson) => - await AlertDetectAsync(udid, pid, platform, host, port, OutputWriter.ResolveJsonMode(json, noJson)), detectUdid, detectPid, detectPlatform, detectHost, detectPort, jsonOption, noJsonOption); + var alertDetectCmd = new Command("detect", "Check if an alert/dialog is visible") { detectUdid, detectPid }; + alertDetectCmd.SetHandler(async (udid, pid, host, port, json, noJson) => + await AlertDetectAsync(udid, pid, host, port, OutputWriter.ResolveJsonMode(json, noJson)), detectUdid, detectPid, agentHostOption, agentPortOption, jsonOption, noJsonOption); alertCommand.Add(alertDetectCmd); // dismiss var dismissUdid = new Option("--udid", "Simulator UDID (auto-detects booted simulator if omitted)"); var dismissPid = new Option("--pid", "Mac Catalyst app PID (auto-detects if omitted)"); - var dismissPlatform = new Option("--platform", () => "auto", "Platform: maccatalyst, ios, android, windows, or auto"); - var dismissHost = new Option("--agent-host", () => "localhost", "Agent HTTP host"); - var dismissPort = new Option("--agent-port", () => 9223, "Agent HTTP port"); var dismissButtonArg = new Argument("button", () => null, "Button label to tap (default: first accept-style button)"); - var alertDismissCmd = new Command("dismiss", "Dismiss the current alert/dialog") { dismissButtonArg, dismissUdid, dismissPid, dismissPlatform, dismissHost, dismissPort }; - alertDismissCmd.SetHandler(async (udid, pid, platform, host, port, button, json, noJson) => - await AlertDismissAsync(udid, pid, platform, host, port, button, OutputWriter.ResolveJsonMode(json, noJson)), dismissUdid, dismissPid, dismissPlatform, dismissHost, dismissPort, dismissButtonArg, jsonOption, noJsonOption); + var alertDismissCmd = new Command("dismiss", "Dismiss the current alert/dialog") { dismissButtonArg, dismissUdid, dismissPid }; + alertDismissCmd.SetHandler(async (udid, pid, host, port, button, json, noJson) => + await AlertDismissAsync(udid, pid, host, port, button, OutputWriter.ResolveJsonMode(json, noJson)), dismissUdid, dismissPid, agentHostOption, agentPortOption, dismissButtonArg, jsonOption, noJsonOption); alertCommand.Add(alertDismissCmd); // tree var treeUdid = new Option("--udid", "Simulator UDID (auto-detects booted simulator if omitted)"); var treePid = new Option("--pid", "Mac Catalyst app PID (auto-detects if omitted)"); - var treePlatform = new Option("--platform", () => "auto", "Platform: maccatalyst, ios, android, windows, or auto"); - var treeHost = new Option("--agent-host", () => "localhost", "Agent HTTP host"); - var treePort = new Option("--agent-port", () => 9223, "Agent HTTP port"); - var alertTreeCmd = new Command("tree", "Show raw accessibility tree") { treeUdid, treePid, treePlatform, treeHost, treePort }; - alertTreeCmd.SetHandler(async (udid, pid, platform, host, port, json, noJson) => - await AlertTreeAsync(udid, pid, platform, host, port, OutputWriter.ResolveJsonMode(json, noJson)), treeUdid, treePid, treePlatform, treeHost, treePort, jsonOption, noJsonOption); + var alertTreeCmd = new Command("tree", "Show raw accessibility tree") { treeUdid, treePid }; + alertTreeCmd.SetHandler(async (udid, pid, host, port, json, noJson) => + await AlertTreeAsync(udid, pid, host, port, OutputWriter.ResolveJsonMode(json, noJson)), treeUdid, treePid, agentHostOption, agentPortOption, jsonOption, noJsonOption); alertCommand.Add(alertTreeCmd); mauiCommand.Add(alertCommand); @@ -2486,16 +2477,17 @@ private static async Task ResolveUdidAsync(string? udid) throw new InvalidOperationException("No booted simulator found. Specify --udid or boot a simulator."); } - private static async Task ResolveAlertPlatformAsync(string platform, string host, int port) + private static async Task ResolveAlertPlatformAsync(string? udid, int? pid, string host, int port) { - var p = platform.ToLowerInvariant(); - if (p.Contains("catalyst")) return "maccatalyst"; - if (p.Contains("ios") || p.Contains("simulator")) return "ios-simulator"; - if (p.Contains("android")) return "android"; - if (p.Contains("windows") || p.Contains("win")) return "windows"; - if (p.Contains("linux") || p.Contains("gtk")) return "linux"; + // If a UDID was explicitly provided, it's an iOS simulator + if (!string.IsNullOrEmpty(udid)) + return "ios-simulator"; - // Auto-detect from agent + // If a PID was explicitly provided, it's Mac Catalyst or Windows + if (pid.HasValue) + return OperatingSystem.IsWindows() ? "windows" : "maccatalyst"; + + // Auto-detect from connected agent try { using var client = new MauiDevFlow.Driver.AgentClient(host, port); @@ -2503,15 +2495,47 @@ private static async Task ResolveAlertPlatformAsync(string platform, str if (status?.Platform != null) { var sp = status.Platform.ToLowerInvariant(); - if (sp.Contains("catalyst")) return "maccatalyst"; - if (sp.Contains("android")) return "android"; if (sp.Contains("ios")) return "ios-simulator"; + if (sp.Contains("android")) return "android"; if (sp.Contains("windows")) return "windows"; if (sp.Contains("linux") || sp.Contains("gtk")) return "linux"; + // For MacCatalyst, don't return immediately — check if there's a + // booted iOS simulator first, since it's more likely the user wants + // iOS dialog detection (Mac Catalyst dialogs are less common) } } catch { } + // Check if a booted iOS simulator exists + if (!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + { + try + { + var psi = new System.Diagnostics.ProcessStartInfo("xcrun", "simctl list devices booted -j") + { + RedirectStandardOutput = true, + UseShellExecute = false + }; + using var proc = System.Diagnostics.Process.Start(psi)!; + var output = await proc.StandardOutput.ReadToEndAsync(); + await proc.WaitForExitAsync(); + + using var doc = JsonDocument.Parse(output); + if (doc.RootElement.TryGetProperty("devices", out var devices)) + { + foreach (var runtime in devices.EnumerateObject()) + { + foreach (var device in runtime.Value.EnumerateArray()) + { + var state = device.TryGetProperty("state", out var s) ? s.GetString() : null; + if (state == "Booted") return "ios-simulator"; + } + } + } + } + catch { } + } + if (OperatingSystem.IsWindows()) return "windows"; if (OperatingSystem.IsLinux()) return "linux"; return "maccatalyst"; @@ -2577,11 +2601,11 @@ private static async Task ResolveWindowsPidAsync(int? pid, string host, int throw new InvalidOperationException("Cannot determine Windows app PID. Specify --pid."); } - private static async Task AlertDetectAsync(string? udid, int? pid, string platform, string host, int port, bool json) + private static async Task AlertDetectAsync(string? udid, int? pid, string host, int port, bool json) { try { - var plat = await ResolveAlertPlatformAsync(platform, host, port); + var plat = await ResolveAlertPlatformAsync(udid, pid, host, port); MauiDevFlow.Driver.AlertInfo? alert = null; if (plat == "maccatalyst") @@ -2624,11 +2648,11 @@ private static async Task AlertDetectAsync(string? udid, int? pid, string platfo catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task AlertDismissAsync(string? udid, int? pid, string platform, string host, int port, string? buttonLabel, bool json) + private static async Task AlertDismissAsync(string? udid, int? pid, string host, int port, string? buttonLabel, bool json) { try { - var plat = await ResolveAlertPlatformAsync(platform, host, port); + var plat = await ResolveAlertPlatformAsync(udid, pid, host, port); MauiDevFlow.Driver.AlertInfo? alert = null; if (plat == "maccatalyst") @@ -2663,11 +2687,11 @@ private static async Task AlertDismissAsync(string? udid, int? pid, string platf catch (Exception ex) { OutputWriter.WriteError(ex.Message, json); _errorOccurred = true; } } - private static async Task AlertTreeAsync(string? udid, int? pid, string platform, string host, int port, bool json) + private static async Task AlertTreeAsync(string? udid, int? pid, string host, int port, bool json) { try { - var plat = await ResolveAlertPlatformAsync(platform, host, port); + var plat = await ResolveAlertPlatformAsync(udid, pid, host, port); string treeResult; if (plat == "maccatalyst") diff --git a/src/SampleMauiApp.MacOS/MauiProgram.cs b/src/SampleMauiApp.MacOS/MauiProgram.cs index 2df4090..f45e695 100644 --- a/src/SampleMauiApp.MacOS/MauiProgram.cs +++ b/src/SampleMauiApp.MacOS/MauiProgram.cs @@ -32,7 +32,7 @@ public static MauiApp CreateMauiApp() #if DEBUG builder.Logging.AddDebug(); - builder.AddMauiDevFlowAgent(options => { options.Port = 9223; }); + builder.AddMauiDevFlowAgent(); builder.AddMauiBlazorDevFlowTools(); #endif diff --git a/src/SampleMauiApp.MacOS/SampleMauiApp.MacOS.csproj b/src/SampleMauiApp.MacOS/SampleMauiApp.MacOS.csproj index 194a706..2f13f4a 100644 --- a/src/SampleMauiApp.MacOS/SampleMauiApp.MacOS.csproj +++ b/src/SampleMauiApp.MacOS/SampleMauiApp.MacOS.csproj @@ -39,6 +39,7 @@ + @@ -51,6 +52,7 @@ +