From 07f2864b8e824d5b00c2285b4a02b869d22bef59 Mon Sep 17 00:00:00 2001 From: Sam Edwards <264948+handstandsam@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:53:53 -0400 Subject: [PATCH] Upstream 2026.08.10 --- .trailblaze-sync | 2 +- docs/adding_a_model.md | 25 +- docs/configuration.md | 2 +- docs/generated/LLM_MODELS.md | 1 - docs/llm_configuration.md | 63 ++- docs/llms.md | 2 +- .../trailblaze/AndroidStandaloneServerTest.kt | 7 + .../android/AndroidLlmClientResolver.kt | 7 + .../accessibility/AccessibilityAction.kt | 6 + .../AccessibilityDeviceManager.kt | 61 ++- .../AccessibilityTrailblazeAgent.kt | 3 + .../accessibility/PlanActionClickRouteTest.kt | 144 +++++++ trailblaze-common/build.gradle.kts | 11 + .../tools/rememberNumberBySelector.tool.yaml | 2 + .../tools/rememberTextBySelector.tool.yaml | 2 + .../trailblaze/MaestroTrailblazeAgent.kt | 6 + .../trailblaze/maestro/AssertionLogger.kt | 1 + .../trailblaze/toolcalls/TrailblazeToolSet.kt | 4 + .../AssertVisibleBySelectorTrailblazeTool.kt | 13 +- .../commands/TapOnByElementSelector.kt | 13 + .../RememberNumberBySelectorTrailblazeTool.kt | 62 +++ .../RememberTextBySelectorTrailblazeTool.kt | 56 +++ .../commands/memory/SelectorTextCapture.kt | 156 ++++++++ .../trailblaze/maestro/AssertionLoggerTest.kt | 104 +++++ .../commands/AssertToolNodeSelectorTest.kt | 20 + .../RememberBySelectorTrailblazeToolTest.kt | 336 ++++++++++++++++ .../trailblaze/yaml/ToolSerializationTest.kt | 53 +++ .../yaml/TrailYamlValidationTest.kt | 373 ++++++++++++++++++ .../desktop/TrailblazeDesktopAppConfig.kt | 30 +- .../host/HostMaestroTrailblazeAgent.kt | 3 + .../host/HostOnDeviceRpcTrailblazeAgent.kt | 3 + .../trailblaze/host/axe/AxeDeviceManager.kt | 40 +- .../host/ios/IosDriverTrailblazeAgent.kt | 3 + .../block/trailblaze/host/ios/SimctlCli.kt | 94 +++++ .../TrailblazeHostDynamicLlmTokenProvider.kt | 12 + .../recording/RecordingScreenComposable.kt | 5 +- .../tabs/recording/RecordingToolRewrites.kt | 19 + .../ResolveSavedModelWithinProviderTest.kt | 96 +++++ .../host/axe/AxeClearStateModeTest.kt | 29 ++ .../recording/RecordingToolRewritesTest.kt | 73 ++++ .../api/android/trailblaze-models.api | 20 + .../api/jvm/trailblaze-models.api | 20 + .../llm/config/OllamaContextWindow.kt | 55 +++ .../trailblaze/model/TapRouteOverride.kt | 42 ++ .../trails/config/providers/google.yaml | 14 +- .../llm/config/OllamaContextWindowTest.kt | 44 +++ .../block/trailblaze/report/utils/LogsRepo.kt | 30 +- .../web/app/run-report-core.test.ts | 23 +- .../trailrunner/web/app/run-report-css.ts | 22 +- .../trailrunner/web/app/run-report-viewer.ts | 20 +- .../report/utils/LogsRepoDiskTruthTest.kt | 76 +++- .../logs/server/TrailblazeMcpServer.kt | 25 +- .../ComputeStaleToolNamesToRemoveTest.kt | 71 ++++ 53 files changed, 2312 insertions(+), 92 deletions(-) create mode 100644 trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberNumberBySelector.tool.yaml create mode 100644 trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberTextBySelector.tool.yaml create mode 100644 trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberNumberBySelectorTrailblazeTool.kt create mode 100644 trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberTextBySelectorTrailblazeTool.kt create mode 100644 trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/SelectorTextCapture.kt create mode 100644 trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/maestro/AssertionLoggerTest.kt create mode 100644 trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberBySelectorTrailblazeToolTest.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewrites.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/desktop/ResolveSavedModelWithinProviderTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/axe/AxeClearStateModeTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewritesTest.kt create mode 100644 trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindow.kt create mode 100644 trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TapRouteOverride.kt create mode 100644 trailblaze-models/src/jvmTest/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindowTest.kt create mode 100644 trailblaze-server/src/test/kotlin/xyz/block/trailblaze/logs/server/ComputeStaleToolNamesToRemoveTest.kt diff --git a/.trailblaze-sync b/.trailblaze-sync index 75856ef8d..96bfc948d 100644 --- a/.trailblaze-sync +++ b/.trailblaze-sync @@ -1 +1 @@ -8141286d5a206c6ee3b16169f1b7224ac2d881aa +9091a0cea8a1703fc9ed45ec128a4d4a00bf5206 diff --git a/docs/adding_a_model.md b/docs/adding_a_model.md index b32d04170..bfa93626b 100644 --- a/docs/adding_a_model.md +++ b/docs/adding_a_model.md @@ -38,8 +38,14 @@ Set the provider's API key (`GOOGLE_API_KEY` here — see the ## How your entry combines with the built-ins -Workspace config **adds to** the built-in registry — it does not replace it. Entries are -matched by `id`: +Declaring an `llm.providers` block **replaces** the built-in catalog: the models Trailblaze +offers become exactly the ones you list, under exactly the providers you declare. With no +`llm.providers` block at all you get the built-in catalog unchanged. So if you want your +model *alongside* the built-ins rather than instead of them, list the built-in models you +still want next to it. + +The built-in registry still does work for you on each entry — ids are matched against it, so +you only spell out what differs: | Your `id` | Result | |---|---| @@ -109,10 +115,10 @@ llm: model: "gemma4:31b" ``` -Quote Ollama ids — the `:` makes them look like YAML mappings otherwise. Trailblaze also -discovers whatever `ollama list` reports at runtime, so a model already pulled locally shows -up without any config; listing it explicitly is how you tell teammates which model the -project expects. Nothing is auto-downloaded — they run `ollama pull gemma4:31b`. +Quote Ollama ids — the `:` makes them look like YAML mappings otherwise. Trailblaze offers +the Ollama models in the built-in catalog plus whatever you list here; pulling a model +locally does not make it selectable on its own, so an id outside the built-in catalog has to +be declared. Nothing is auto-downloaded either — teammates run `ollama pull gemma4:31b`. ### A model behind your own gateway @@ -161,8 +167,11 @@ The workspace file wins over the user file; environment variables win over both. precedence table: [Configuration → Precedence](configuration.md#precedence). Committing the workspace file is the recommended shape for teams — everyone who clones the -repo gets a working model with no per-machine setup, and the project is pinned to models you -have actually validated rather than to whatever the current release happens to ship. +repo sees the same set of models, and the project is pinned to models you have actually +validated rather than to whatever the current release happens to ship. Each person still +picks their active model once with `trailblaze config llm` (that selection is persisted per +machine); `llm.defaults.model` is read by the Android on-device runner, not by the desktop +app or CLI. ## Verifying diff --git a/docs/configuration.md b/docs/configuration.md index 627efbba6..093067641 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -54,7 +54,7 @@ llm: | Key | Type | What it does | |---|---|---| | `defaults` | map | Workspace-wide defaults — see below | -| `targets` | list of ids | Target-trailmap ids this workspace opts into. **Omit to auto-discover** every target trailmap under `/trailmaps/`. Listing ids is how a workspace with many trailmaps loads only a subset. Each id must be a *target* trailmap (one with a `target:` block); library trailmaps reach scope through a target's `dependencies:`. | +| `targets` | list of ids | Target-trailmap ids this workspace opts into. **Omit to auto-discover** every target trailmap under `/trails/config/trailmaps/`. Listing ids is how a workspace with many trailmaps loads only a subset. Each id must be a *target* trailmap (one with a `target:` block); library trailmaps reach scope through a target's `dependencies:`. | | `toolsets` | list | Extra toolsets, either written inline or pulled in with `ref: path/to/toolset.yaml` | | `tools` | list | Extra tools, same inline-or-`ref:` shape | | `providers` | list | Reserved for standalone LLM provider files. Provider and model definitions are read from the `llm:` block today — put them there. | diff --git a/docs/generated/LLM_MODELS.md b/docs/generated/LLM_MODELS.md index fb79032e1..95252ea1f 100644 --- a/docs/generated/LLM_MODELS.md +++ b/docs/generated/LLM_MODELS.md @@ -22,7 +22,6 @@ Trailblaze ships with the following built-in models. When you reference a model | `gemini-3.5-flash` | 1M | 65K | $1.50 | $9.00 | $0.15 | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | | `gemini-3.5-flash-lite` | 1M | 65K | $0.30 | $2.50 | $0.03 | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | | `gemini-3.6-flash` | 1M | 65K | $1.50 | $7.50 | $0.15 | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | -| `gemma-4-12b-it` | 262K | 32K | free | free | free | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | | `gemma-4-26b-a4b-it` | 262K | 32K | free | free | free | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | | `gemma-4-31b-it` | 262K | 32K | free | free | free | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | diff --git a/docs/llm_configuration.md b/docs/llm_configuration.md index 995b5d597..ca42fc7d3 100644 --- a/docs/llm_configuration.md +++ b/docs/llm_configuration.md @@ -231,11 +231,14 @@ llm: models: - id: qwen3-vl:8b - id: qwen3.5:27b - - id: llama3.2:latest defaults: model: qwen3-vl:8b ``` +Both ids above are in the built-in registry, so they need no spec. A model the registry +doesn't know falls back to a generic context length — give those an explicit spec, as +[Custom model specs](#custom-model-specs) below shows. + Models that are not installed will show a warning in the desktop app UI indicating the model is not available via Ollama. Developers can install them with: ```bash @@ -254,6 +257,64 @@ For Ollama models not in the built-in registry, provide context length and outpu max_output_tokens: 8192 ``` +For a text-only model, also set `vision: false` — otherwise Trailblaze attaches screenshots +to requests (including AI-backed assertions) and Ollama rejects them with a 400. + +### Context window (`num_ctx`) + +Trailblaze requests a 64K context window (`num_ctx: 65536`) on every Ollama call, clamped +down to the model's declared `context_length` when that is lower. + +**Trailblaze now owns this setting.** A `num_ctx` on the request sits at the top of +Ollama's precedence chain — above a `PARAMETER num_ctx` in the model's Modelfile, and above +the server's `OLLAMA_CONTEXT_LENGTH` — so neither of those takes effect for Trailblaze's +requests any more. Use `TRAILBLAZE_OLLAMA_NUM_CTX` (below) rather than a server-side +setting, which will look like it is being ignored. + +The clamp only applies to a model whose `context_length` Trailblaze knows: one in the +built-in registry, or one you declared with an explicit spec. A model with neither falls +back to a generic context length, and the clamp does nothing for it — so if such a model's +real window is smaller than the requested value, declare its `context_length` (see +[Custom model specs](#custom-model-specs)) instead of relying on the clamp. + +When a request doesn't ask for a context window, Ollama picks one itself, sized to the +memory it has available rather than to what the model supports — so the same model gets a +large window on a workstation and a very small one on a laptop. A single Trailblaze agent +turn (screenshot + view hierarchy + tool definitions) is ~20K tokens on a content-heavy +screen, which is more than the low end of that range, and the turn fails with +`exceed_context_size_error`. Asking explicitly is what makes the context predictable +across machines instead of a property of the developer's hardware. + +64K is a deliberate middle: several multi-turn agent loops fit, and the KV cache still +fits a laptop. On a machine with a lot of memory Ollama's automatic choice can be larger +than 64K, and an explicit request replaces it — raise it with the override below if you +run very long loops on such a machine. + +Override the requested value with `TRAILBLAZE_OLLAMA_NUM_CTX`: + +```bash +TRAILBLAZE_OLLAMA_NUM_CTX=32768 trailblaze run --no-daemon login.trail.yaml +``` + +Lower it if your machine can't afford the 64K KV cache for a larger model; raise it for +very long agent loops. Malformed or non-positive values fall back to the default. + +The value is read from the process that builds the Ollama client. `trailblaze run` +normally hands the run to a background daemon, which inherits the environment it was +started with — so a one-shot prefix like the above only applies with `--no-daemon`. To +change it for daemon-backed runs, restart the daemon with the variable set: + +```bash +trailblaze stop +TRAILBLAZE_OLLAMA_NUM_CTX=32768 trailblaze app start +``` + +The override applies to host-side clients only. Android on-device runs always request the +64K default, because the instrumentation process has no host environment to read. If you +set the override *and* run on-device AI legs against the same Ollama server, the two ends +request different context lengths and Ollama reloads the model on every alternation — leave +it unset for those runs. + ## Environment Variables Standard environment variables for authentication: diff --git a/docs/llms.md b/docs/llms.md index 2d3a15a35..915ae368f 100644 --- a/docs/llms.md +++ b/docs/llms.md @@ -28,7 +28,7 @@ import xyz.block.trailblaze.agent.TrailblazeRunner import xyz.block.trailblaze.llm.TrailblazeLlmModel import xyz.block.trailblaze.llm.providers.OpenAITrailblazeLlmModelList -private val trailblazeLlmModel: TrailblazeLlmModel = OpenAITrailblazeLlmModelList.OPENAI_GPT_4_1 +private val trailblazeLlmModel: TrailblazeLlmModel = OpenAITrailblazeLlmModelList.OPENAI_DEFAULT private val llmClient: LLMClient = OpenAILLMClient("API_KEY_HERE") TrailblazeRunner( llmClient = llmClient, diff --git a/trailblaze-android-ondevice-mcp/src/androidTest/java/xyz/block/trailblaze/AndroidStandaloneServerTest.kt b/trailblaze-android-ondevice-mcp/src/androidTest/java/xyz/block/trailblaze/AndroidStandaloneServerTest.kt index bc1bf4cc6..c7ef3e5e8 100644 --- a/trailblaze-android-ondevice-mcp/src/androidTest/java/xyz/block/trailblaze/AndroidStandaloneServerTest.kt +++ b/trailblaze-android-ondevice-mcp/src/androidTest/java/xyz/block/trailblaze/AndroidStandaloneServerTest.kt @@ -5,6 +5,7 @@ import ai.koog.prompt.executor.clients.LLMClient import ai.koog.prompt.executor.clients.anthropic.AnthropicClientSettings import ai.koog.prompt.executor.clients.anthropic.AnthropicLLMClient import ai.koog.prompt.executor.clients.openai.OpenAILLMClient +import ai.koog.prompt.executor.ollama.client.ContextWindowStrategy import ai.koog.prompt.executor.ollama.client.OllamaClient import ai.koog.prompt.llm.LLMProvider import org.junit.Test @@ -20,6 +21,7 @@ import xyz.block.trailblaze.devices.TrailblazeDeviceClassifier import xyz.block.trailblaze.llm.TrailblazeLlmProvider import xyz.block.trailblaze.llm.config.BuiltInLlmModelRegistry import xyz.block.trailblaze.llm.config.LlmAuthResolver +import xyz.block.trailblaze.llm.config.OllamaContextWindow import xyz.block.trailblaze.http.DefaultDynamicLlmClient import xyz.block.trailblaze.http.DynamicLlmClient import xyz.block.trailblaze.http.NoOpLlmClient @@ -87,6 +89,11 @@ class AndroidStandaloneServerTest : BaseAndroidStandaloneServerTest() { LLMProvider.Ollama to OllamaClient( baseUrl = ollamaBaseUrl ?: "http://localhost:11434", httpClientFactory = httpClientFactory, + // Request num_ctx on every call — left to itself Ollama sizes the window to the + // memory it has available, which can reject real agent turns (~20K tokens) + // regardless of the model's declared context. + // No env override on-device; the host-side knob is TRAILBLAZE_OLLAMA_NUM_CTX. + contextWindowStrategy = ContextWindowStrategy.Companion.Fixed(OllamaContextWindow.DEFAULT_NUM_CTX), ), ) InstrumentationArgUtil.getInstrumentationArg(LlmAuthResolver.resolve(TrailblazeLlmProvider.OPENAI))?.let { openAiApiKey -> diff --git a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidLlmClientResolver.kt b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidLlmClientResolver.kt index d556d6fc7..ecb617787 100644 --- a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidLlmClientResolver.kt +++ b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidLlmClientResolver.kt @@ -7,6 +7,7 @@ import ai.koog.prompt.executor.clients.anthropic.AnthropicLLMClient import ai.koog.prompt.executor.clients.openai.OpenAIClientSettings import ai.koog.prompt.executor.clients.openai.OpenAILLMClient import ai.koog.prompt.executor.clients.openrouter.OpenRouterLLMClient +import ai.koog.prompt.executor.ollama.client.ContextWindowStrategy import ai.koog.prompt.executor.ollama.client.OllamaClient import ai.koog.prompt.llm.LLMProvider import xyz.block.trailblaze.android.openai.OpenAiInstrumentationArgUtil @@ -20,6 +21,7 @@ import com.charleskorn.kaml.YamlConfiguration import xyz.block.trailblaze.llm.config.BuiltInLlmModelRegistry import xyz.block.trailblaze.llm.config.TrailblazeConfigPaths import xyz.block.trailblaze.llm.config.LlmAuthResolver +import xyz.block.trailblaze.llm.config.OllamaContextWindow import xyz.block.trailblaze.llm.config.TrailblazeProjectYamlConfig import xyz.block.trailblaze.util.Console @@ -168,6 +170,11 @@ object AndroidLlmClientResolver { OllamaClient( baseUrl = ollamaBaseUrl ?: "http://localhost:11434", httpClientFactory = httpClientFactory, + // Request num_ctx on every call — left to itself Ollama sizes the window to the + // memory it has available, which can reject real agent turns (~20K tokens) + // regardless of the model's declared context. + // No env override on-device; the host-side knob is TRAILBLAZE_OLLAMA_NUM_CTX. + contextWindowStrategy = ContextWindowStrategy.Companion.Fixed(OllamaContextWindow.DEFAULT_NUM_CTX), ), ) diff --git a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityAction.kt b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityAction.kt index 2ea9a5979..a3a5f3829 100644 --- a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityAction.kt +++ b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityAction.kt @@ -2,6 +2,7 @@ package xyz.block.trailblaze.android.accessibility import maestro.KeyCode import xyz.block.trailblaze.api.TrailblazeNodeSelector +import xyz.block.trailblaze.model.TapRouteOverride /** * Default timeout for actions that poll the accessibility tree waiting for an element to appear @@ -127,6 +128,11 @@ sealed interface AccessibilityAction { * out, and the trail fails on what's supposed to be a best-effort step. */ val optional: Boolean = false, + /** + * Pins the dispatch route for this tap instead of letting [planActionClickRoute] choose it + * from the resolved node's fields. Carried from the recording; null means the gate decides. + */ + val tapRoute: TapRouteOverride? = null, ) : AccessibilityAction { override val description: String get() = "${if (longPress) "Long press" else "Tap"} on ${nodeSelector.description()}" diff --git a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt index 56d4ca071..5bf5f062e 100644 --- a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt +++ b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt @@ -21,6 +21,7 @@ import xyz.block.trailblaze.api.TrailblazeNode import xyz.block.trailblaze.api.TrailblazeNodeSelector import xyz.block.trailblaze.api.TrailblazeNodeSelectorResolver import xyz.block.trailblaze.devices.TrailblazeDeviceClassifier +import xyz.block.trailblaze.model.TapRouteOverride import xyz.block.trailblaze.util.Console /** @@ -875,7 +876,9 @@ class AccessibilityDeviceManager( unfilteredTree, result.node, center.first, center.second, action.nodeSelector.description(), ) val route = - tapOrLongPressOnResolvedNode(result.node, center.first, center.second, action.longPress) + tapOrLongPressOnResolvedNode( + result.node, center.first, center.second, action.longPress, action.tapRoute, + ) return ExecutionResult( resolvedX = center.first, resolvedY = center.second, @@ -894,7 +897,9 @@ class AccessibilityDeviceManager( unfilteredTree, chosen, center.first, center.second, action.nodeSelector.description(), ) val route = - tapOrLongPressOnResolvedNode(chosen, center.first, center.second, action.longPress) + tapOrLongPressOnResolvedNode( + chosen, center.first, center.second, action.longPress, action.tapRoute, + ) return ExecutionResult( resolvedX = center.first, resolvedY = center.second, @@ -970,21 +975,27 @@ class AccessibilityDeviceManager( centerX: Int, centerY: Int, longPress: Boolean, + tapRoute: TapRouteOverride?, ): TapDispatchRoute { if (actionClickRouteDisabled()) { Console.log("[tap-route] kill-switch set, using gesture at ($centerX,$centerY)") tapOrLongPress(centerX, centerY, longPress) return TapDispatchRoute.GESTURE } - val plan = planActionClickRoute(resolvedNode, longPress) + val plan = planActionClickRoute(resolvedNode, longPress, tapRoute) if (plan == null) { // Surface the gate-relevant fields of the resolved node so an oncall debugging // "why did this tap go via gesture?" can map each value to the matching condition // in `planActionClickRoute`'s kdoc without re-running the session. The 7-condition // gate makes a generic "declined" message uninformative. + val why = if (tapRoute == TapRouteOverride.GESTURE) { + "recording pinned gesture" + } else { + "gate declined ACTION_CLICK" + } Console.log( - "[tap-route] gesture at ($centerX,$centerY) — gate declined ACTION_CLICK " + - "(${describeNodeForRouteLog(resolvedNode, longPress)})", + "[tap-route] gesture at ($centerX,$centerY) — $why " + + "(${describeNodeForRouteLog(resolvedNode, longPress, tapRoute)})", ) tapOrLongPress(centerX, centerY, longPress) return TapDispatchRoute.GESTURE @@ -1194,23 +1205,37 @@ internal fun scrollToSwipeDirection(direction: AccessibilityAction.Direction): A * still taps correctly, while a stateless container wrongly sent to `ACTION_CLICK` changes * what the tap does. A misrouting node is diagnosable from the `[tap-route]` log and the * whole route is revertible with `TRAILBLAZE_DISABLE_ACTION_CLICK_ROUTE`. + * + * [tapRoute] lets a single recorded step pin its own route for the case this gate cannot decide: + * two rows that need opposite routes presenting identical fields (see [TapRouteOverride]). The pin + * reaches only the leaf-vs-container judgement above. Every check before it is a precondition for + * `ACTION_CLICK` being dispatchable at all, so a step that pins `ACTION_CLICK` on a long-press, an + * editable field, an invisible node, or a node that doesn't advertise the action still routes to + * gesture rather than dispatching an action the node can't answer. */ -internal fun planActionClickRoute(node: TrailblazeNode, longPress: Boolean): ActionClickPlan? { +internal fun planActionClickRoute( + node: TrailblazeNode, + longPress: Boolean, + tapRoute: TapRouteOverride? = null, +): ActionClickPlan? { if (longPress) return null + if (tapRoute == TapRouteOverride.GESTURE) return null val bounds = node.bounds ?: return null val detail = node.driverDetail as? DriverNodeDetail.AndroidAccessibility ?: return null if (ACTION_CLICK_NAME !in detail.actions) return null if (!detail.isEnabled) return null if (detail.isEditable) return null if (!detail.isVisibleToUser) return null - val publishesCheckedState = - detail.isCheckable && (detail.isChecked || !detail.stateDescription.isNullOrBlank()) - if ( - detail.text.isNullOrBlank() && - detail.contentDescription.isNullOrBlank() && - !publishesCheckedState - ) { - return null + if (tapRoute != TapRouteOverride.ACTION_CLICK) { + val publishesCheckedState = + detail.isCheckable && (detail.isChecked || !detail.stateDescription.isNullOrBlank()) + if ( + detail.text.isNullOrBlank() && + detail.contentDescription.isNullOrBlank() && + !publishesCheckedState + ) { + return null + } } return ActionClickPlan(bounds, detail.className, detail.resourceId) } @@ -1246,9 +1271,13 @@ internal fun pickPreferredMatch(nodes: List): TrailblazeNode = * function, kept top-level alongside [planActionClickRoute] for the same JVM-unit-testability * reasons. */ -internal fun describeNodeForRouteLog(node: TrailblazeNode, longPress: Boolean): String { +internal fun describeNodeForRouteLog( + node: TrailblazeNode, + longPress: Boolean, + tapRoute: TapRouteOverride? = null, +): String { val detail = node.driverDetail as? DriverNodeDetail.AndroidAccessibility - return "longPress=$longPress, hasBounds=${node.bounds != null}, " + + return "tapRoute=$tapRoute, longPress=$longPress, hasBounds=${node.bounds != null}, " + "className=${detail?.className}, text=${detail?.text}, contentDescription=${detail?.contentDescription}, " + "isEnabled=${detail?.isEnabled}, isEditable=${detail?.isEditable}, " + "isVisibleToUser=${detail?.isVisibleToUser}, isCheckable=${detail?.isCheckable}, " + diff --git a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityTrailblazeAgent.kt b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityTrailblazeAgent.kt index 59f937ef7..974cc5e30 100644 --- a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityTrailblazeAgent.kt +++ b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityTrailblazeAgent.kt @@ -19,6 +19,7 @@ import xyz.block.trailblaze.logs.client.TrailblazeLogger import xyz.block.trailblaze.logs.client.TrailblazeSessionProvider import xyz.block.trailblaze.logs.model.TraceId import xyz.block.trailblaze.model.ResolvedTarget +import xyz.block.trailblaze.model.TapRouteOverride import xyz.block.trailblaze.toolcalls.TrailblazeToolResult import xyz.block.trailblaze.util.Console @@ -234,11 +235,13 @@ class AccessibilityTrailblazeAgent( override suspend fun executeNodeSelectorTap( nodeSelector: TrailblazeNodeSelector, longPress: Boolean, + tapRoute: TapRouteOverride?, traceId: TraceId?, ): TrailblazeToolResult { val action = AccessibilityAction.TapOnElement( nodeSelector = nodeSelector, longPress = longPress, + tapRoute = tapRoute, ) return AccessibilityTrailRunner.runActions( diff --git a/trailblaze-android/src/test/java/xyz/block/trailblaze/android/accessibility/PlanActionClickRouteTest.kt b/trailblaze-android/src/test/java/xyz/block/trailblaze/android/accessibility/PlanActionClickRouteTest.kt index f5b71fd1f..a5b6ab50c 100644 --- a/trailblaze-android/src/test/java/xyz/block/trailblaze/android/accessibility/PlanActionClickRouteTest.kt +++ b/trailblaze-android/src/test/java/xyz/block/trailblaze/android/accessibility/PlanActionClickRouteTest.kt @@ -5,6 +5,7 @@ import kotlin.test.assertEquals import kotlin.test.assertNull import xyz.block.trailblaze.api.DriverNodeDetail import xyz.block.trailblaze.api.TrailblazeNode +import xyz.block.trailblaze.model.TapRouteOverride /** * Pure-function coverage of [planActionClickRoute] — the gate that decides whether a @@ -385,6 +386,149 @@ class PlanActionClickRouteTest { assertNull(plan, "Hidden background nodes must defer to the OS z-order via the gesture path.") } + @Test + fun `an ACTION_CLICK pin promotes a stateless checkable container the gate declines`() { + // The shape that needs the pin: an option row inside a dropdown sheet, textless because the + // label is on a child node, checkable, and publishing no state at all because the driver + // reads `stateDescription` only from API 30 (below that the platform field doesn't exist and + // Compose's backport lands in the node's extras, which the driver doesn't read). On such a + // device the row is field-identical to a stateless wrapper whose handler lives elsewhere, so + // no predicate over these fields can separate them — only the recording knows. + val row = node( + bounds = TrailblazeNode.Bounds(0, 1200, 1080, 1340), + detail = androidA11y( + className = "android.view.View", + actions = listOf(ACTION_CLICK_NAME), + isCheckable = true, + ), + ) + assertEquals( + ActionClickPlan( + bounds = TrailblazeNode.Bounds(0, 1200, 1080, 1340), + className = "android.view.View", + resourceId = null, + ), + planActionClickRoute(node = row, longPress = false, tapRoute = TapRouteOverride.ACTION_CLICK), + "An ACTION_CLICK pin must reach past the leaf-vs-container judgement.", + ) + assertNull( + planActionClickRoute(node = row, longPress = false, tapRoute = null), + "Negative control — the same node with no pin must still decline, so every unpinned tap " + + "in every other recording keeps the route it has today.", + ) + } + + @Test + fun `a GESTURE pin declines a node the gate would otherwise route semantically`() { + // The mirror-image shape: a row that publishes state (so the gate grants ACTION_CLICK) but + // whose semantic click performs a different action than a real touch — an accordion header + // whose ACTION_CLICK selects while only a touch expands the sub-options a later step taps. + // Such a row publishes "Collapsed"/"Expanded" natively from API 30, so the pin is how a + // recording holds its route when the same trail moves to a newer device. + val row = node( + bounds = TrailblazeNode.Bounds(0, 1100, 1080, 1240), + detail = androidA11y( + className = "android.view.View", + actions = listOf(ACTION_CLICK_NAME), + isCheckable = true, + stateDescription = "Collapsed", + ), + ) + assertNull( + planActionClickRoute(node = row, longPress = false, tapRoute = TapRouteOverride.GESTURE), + "A GESTURE pin must decline even a node that satisfies every gate condition.", + ) + assertEquals( + ActionClickPlan( + bounds = TrailblazeNode.Bounds(0, 1100, 1080, 1240), + className = "android.view.View", + resourceId = null, + ), + planActionClickRoute(node = row, longPress = false, tapRoute = null), + "Negative control — without the pin this node routes semantically, which is what makes " + + "the assertion above about the pin rather than about the node.", + ) + } + + @Test + fun `an ACTION_CLICK pin cannot dispatch an action the node cannot answer`() { + // The pin overrides one judgement — leaf-vs-container — not the conditions that make + // ACTION_CLICK dispatchable at all. A recording that pins a route the node can't honor gets + // the gesture path rather than a dispatch that silently reports success. + val pin = TapRouteOverride.ACTION_CLICK + assertNull( + planActionClickRoute(node = clickableNode(), longPress = true, tapRoute = pin), + "Long-press has no ACTION_CLICK to pin.", + ) + assertNull( + planActionClickRoute( + node = TrailblazeNode(bounds = null, driverDetail = androidA11y("android.view.View")), + longPress = false, + tapRoute = pin, + ), + "Without bounds there is no identity to find in the live tree.", + ) + assertNull( + planActionClickRoute( + node = node( + bounds = TrailblazeNode.Bounds(0, 0, 100, 100), + detail = androidA11y(className = "android.view.View", actions = emptyList()), + ), + longPress = false, + tapRoute = pin, + ), + "A node that doesn't advertise ACTION_CLICK has nothing to dispatch.", + ) + assertNull( + planActionClickRoute( + node = node( + bounds = TrailblazeNode.Bounds(0, 0, 100, 100), + detail = androidA11y( + className = "android.widget.EditText", + text = "user@example.com", + actions = listOf(ACTION_CLICK_NAME), + isEditable = true, + ), + ), + longPress = false, + tapRoute = pin, + ), + "Editable fields still need the touch offset for caret placement.", + ) + assertNull( + planActionClickRoute( + node = node( + bounds = TrailblazeNode.Bounds(0, 0, 100, 100), + detail = androidA11y( + className = "android.widget.Button", + text = "Submit", + actions = listOf(ACTION_CLICK_NAME), + isVisibleToUser = false, + ), + ), + longPress = false, + tapRoute = pin, + ), + "An occluded node must still defer to the OS z-order.", + ) + assertNull( + planActionClickRoute( + node = node( + bounds = TrailblazeNode.Bounds(0, 0, 100, 100), + detail = androidA11y( + className = "android.widget.Button", + text = "Submit", + actions = listOf(ACTION_CLICK_NAME), + isEnabled = false, + ), + ), + longPress = false, + tapRoute = pin, + ), + "A disabled node's performAction returns false; route to gesture so retry surfaces it.", + ) + } + // --- Test helpers --- /** diff --git a/trailblaze-common/build.gradle.kts b/trailblaze-common/build.gradle.kts index 76e2b89d1..cd11b8f5c 100644 --- a/trailblaze-common/build.gradle.kts +++ b/trailblaze-common/build.gradle.kts @@ -17,6 +17,7 @@ import org.gradle.api.tasks.OutputFiles import org.gradle.api.tasks.PathSensitive import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.testing.logging.TestExceptionFormat import org.jetbrains.kotlin.gradle.dsl.JvmTarget abstract class BundleFrameworkScriptedToolsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { @@ -434,6 +435,16 @@ dependencyGuard { } } +// Gradle's default console output for a failed test is `AssertionError at Foo.kt:42` with the message +// dropped, which hides exactly the part an assertion writes for the reader (see TrailYamlValidationTest, +// whose failures name the offending tool and file). +tasks.withType().configureEach { + testLogging { + events("failed") + exceptionFormat = TestExceptionFormat.FULL + } +} + // --- Framework scripted-tool QuickJS bundles --- // Pre-compiles every @trailblaze/scripting-based `.ts` tool in the framework `trailblaze` trailmap // into a self-contained `.bundle.js` that QuickJsToolHost evaluates directly (host daemon + diff --git a/trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberNumberBySelector.tool.yaml b/trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberNumberBySelector.tool.yaml new file mode 100644 index 000000000..77f6d3e30 --- /dev/null +++ b/trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberNumberBySelector.tool.yaml @@ -0,0 +1,2 @@ +id: rememberNumberBySelector +class: xyz.block.trailblaze.toolcalls.commands.memory.RememberNumberBySelectorTrailblazeTool diff --git a/trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberTextBySelector.tool.yaml b/trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberTextBySelector.tool.yaml new file mode 100644 index 000000000..406ffce64 --- /dev/null +++ b/trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/rememberTextBySelector.tool.yaml @@ -0,0 +1,2 @@ +id: rememberTextBySelector +class: xyz.block.trailblaze.toolcalls.commands.memory.RememberTextBySelectorTrailblazeTool diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/MaestroTrailblazeAgent.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/MaestroTrailblazeAgent.kt index 55534e48d..d69a7fc9b 100644 --- a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/MaestroTrailblazeAgent.kt +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/MaestroTrailblazeAgent.kt @@ -16,6 +16,7 @@ import xyz.block.trailblaze.logs.model.SessionId import xyz.block.trailblaze.logs.model.TraceId import xyz.block.trailblaze.model.NodeSelectorMode import xyz.block.trailblaze.model.ResolvedTarget +import xyz.block.trailblaze.model.TapRouteOverride import xyz.block.trailblaze.toolcalls.DelegatingTrailblazeTool import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeTool @@ -140,6 +141,11 @@ abstract class MaestroTrailblazeAgent( open suspend fun executeNodeSelectorTap( nodeSelector: TrailblazeNodeSelector, longPress: Boolean, + /** + * Route pin carried by the recording. Only the accessibility driver chooses between dispatch + * routes; drivers that always dispatch a gesture ignore this. + */ + tapRoute: TapRouteOverride? = null, traceId: TraceId?, ): TrailblazeToolResult? = null diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/maestro/AssertionLogger.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/maestro/AssertionLogger.kt index bd5b20828..b4088258b 100644 --- a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/maestro/AssertionLogger.kt +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/maestro/AssertionLogger.kt @@ -131,6 +131,7 @@ class AssertionLogger( durationMs = 0, timestamp = kotlinx.datetime.Clock.System.now(), session = session.sessionId, + captureCoverage = screenState.captureCoverage, deviceWidth = screenState.deviceWidth, deviceHeight = screenState.deviceHeight, ) diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/TrailblazeToolSet.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/TrailblazeToolSet.kt index 89d31ae92..780807b6b 100644 --- a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/TrailblazeToolSet.kt +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/TrailblazeToolSet.kt @@ -16,6 +16,8 @@ import xyz.block.trailblaze.toolcalls.commands.SwipeWithRelativeCoordinatesTool import xyz.block.trailblaze.toolcalls.commands.TapOnByElementSelector import xyz.block.trailblaze.toolcalls.commands.TapOnElementWithAccessiblityTextTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.TapOnElementWithTextTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.memory.RememberNumberBySelectorTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.memory.RememberTextBySelectorTrailblazeTool import kotlin.reflect.KClass @Suppress("ktlint:standard:property-naming") @@ -53,6 +55,8 @@ abstract class TrailblazeToolSet( AssertVisibleBySelectorTrailblazeTool::class, AssertNotVisibleBySelectorTrailblazeTool::class, AssertMatchCountTrailblazeTool::class, + RememberTextBySelectorTrailblazeTool::class, + RememberNumberBySelectorTrailblazeTool::class, TapOnByElementSelector::class, SwipeWithRelativeCoordinatesTool::class, AndroidSystemUiDemoModeTrailblazeTool::class, diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertVisibleBySelectorTrailblazeTool.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertVisibleBySelectorTrailblazeTool.kt index 5c2b019be..76086b681 100644 --- a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertVisibleBySelectorTrailblazeTool.kt +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertVisibleBySelectorTrailblazeTool.kt @@ -45,8 +45,10 @@ data class AssertVisibleBySelectorTrailblazeTool( * settle (e.g. an "Authorizing" overlay clearing) before the target text renders. * * When `null` the call is unopinionated about timeout and each agent applies its own - * idle/wait policy (per-driver default). The Maestro fallback path ignores this field - * entirely — Maestro's own assert timeout is always used there. + * idle/wait policy (per-driver default). Forwarded to the Maestro fallback path too — + * without that, a driver that resolves the selector but does not itself poll (the iOS host + * agent returns null when the element is not on screen yet, by design) silently got + * Maestro's default budget instead of the one the author asked for. */ val timeoutMs: Long? = null, /** @@ -88,7 +90,12 @@ data class AssertVisibleBySelectorTrailblazeTool( val maestroElement = maestroSelector.toMaestroElementSelector().let { base -> if (expectedText != null) base.copy(textRegex = maestroTextRegexFor()) else base } - return listOf(AssertConditionCommand(condition = Condition(visible = maestroElement))) + return listOf( + AssertConditionCommand( + condition = Condition(visible = maestroElement), + timeout = timeoutMs?.toString(), + ), + ) } override suspend fun execute( diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/TapOnByElementSelector.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/TapOnByElementSelector.kt index 23eb3c471..356a53375 100644 --- a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/TapOnByElementSelector.kt +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/TapOnByElementSelector.kt @@ -6,6 +6,7 @@ import maestro.orchestra.Command import maestro.orchestra.TapOnElementCommand import xyz.block.trailblaze.api.TrailblazeNodeSelector import xyz.block.trailblaze.model.NodeSelectorMode +import xyz.block.trailblaze.model.TapRouteOverride import xyz.block.trailblaze.toolcalls.MapsToMaestroCommands import xyz.block.trailblaze.toolcalls.TrailblazeToolClass import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext @@ -33,6 +34,15 @@ import xyz.block.trailblaze.util.Console data class TapOnByElementSelector( val reason: String? = null, val longPress: Boolean = false, + /** + * Pins how this tap is dispatched, overriding the Android accessibility driver's automatic + * route choice for this one step. Unset (the default) leaves that choice to the driver, which + * is right for essentially every tap — see [TapRouteOverride] for when a step needs the pin. + * + * Only the Android accessibility driver routes taps; every other driver dispatches a gesture + * unconditionally and ignores this. + */ + val tapRoute: TapRouteOverride? = null, /** * Rich driver-native selector generated from [TrailblazeNode] trees. * @@ -87,6 +97,7 @@ data class TapOnByElementSelector( val accessibilityResult = agent.executeNodeSelectorTap( nodeSelector = nodeSelector, longPress = longPress, + tapRoute = tapRoute, traceId = toolExecutionContext.traceId, ) if (accessibilityResult != null) return accessibilityResult @@ -116,6 +127,7 @@ data class TapOnByElementSelector( val result = agent.executeNodeSelectorTap( nodeSelector = nodeSelector, longPress = longPress, + tapRoute = tapRoute, traceId = toolExecutionContext.traceId, ) if (result != null) return result @@ -127,6 +139,7 @@ data class TapOnByElementSelector( val result = agent.executeNodeSelectorTap( nodeSelector = nodeSelector, longPress = longPress, + tapRoute = tapRoute, traceId = toolExecutionContext.traceId, ) if (result != null) return result diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberNumberBySelectorTrailblazeTool.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberNumberBySelectorTrailblazeTool.kt new file mode 100644 index 000000000..ef24edbb8 --- /dev/null +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberNumberBySelectorTrailblazeTool.kt @@ -0,0 +1,62 @@ +package xyz.block.trailblaze.toolcalls.commands.memory + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.api.TrailblazeNodeSelector +import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool +import xyz.block.trailblaze.toolcalls.ReadOnlyTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult +import xyz.block.trailblaze.utils.parseNumberString + +@Serializable +@TrailblazeToolClass( + name = "rememberNumberBySelector", + surfaceToLlm = false, +) +@LLMDescription( + "Captures the number in the text of the element matching a selector into a memory variable, " + + "with no LLM call.", +) +/** + * ----- DO NOT GIVE THIS TOOL TO THE LLM ----- + * + * The deterministic counterpart to [RememberNumberTrailblazeTool], and the numeric sibling of + * [RememberTextBySelectorTrailblazeTool]: it captures the same selector-resolved text, then keeps + * only the first number in it (via [parseNumberString], the same extraction `rememberNumber` uses), + * so `$42.50` stores as `42.50` and stays comparable by `assertMath` / `assertEquals`. + */ +data class RememberNumberBySelectorTrailblazeTool( + val reason: String? = null, + /** Selector for the element whose number is captured. Required — [execute] enforces non-null. */ + val nodeSelector: TrailblazeNodeSelector? = null, + /** Memory variable name the parsed number is stored under. */ + val variable: String, +) : ExecutableTrailblazeTool, ReadOnlyTrailblazeTool { + + override suspend fun execute( + toolExecutionContext: TrailblazeToolExecutionContext, + ): TrailblazeToolResult = when ( + val capture = captureSelectorText(TOOL_NAME, toolExecutionContext, nodeSelector) + ) { + is SelectorTextCapture.Failed -> capture.error + is SelectorTextCapture.Captured -> { + val number = parseNumberString(capture.text) + if (number == null) { + TrailblazeToolResult.Error.ExceptionThrown( + errorMessage = "$TOOL_NAME: no number found in the matched element's text " + + "'${capture.text}'.", + ) + } else { + toolExecutionContext.memory.remember(variable, number) + TrailblazeToolResult.Success( + message = "$TOOL_NAME: remembered $variable = " + + renderCaptured(toolExecutionContext, variable, number), + ) + } + } + } +} + +private const val TOOL_NAME = "rememberNumberBySelector" diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberTextBySelectorTrailblazeTool.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberTextBySelectorTrailblazeTool.kt new file mode 100644 index 000000000..2ba7bcb21 --- /dev/null +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberTextBySelectorTrailblazeTool.kt @@ -0,0 +1,56 @@ +package xyz.block.trailblaze.toolcalls.commands.memory + +import ai.koog.agents.core.tools.annotations.LLMDescription +import kotlinx.serialization.Serializable +import xyz.block.trailblaze.api.TrailblazeNodeSelector +import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool +import xyz.block.trailblaze.toolcalls.ReadOnlyTrailblazeTool +import xyz.block.trailblaze.toolcalls.TrailblazeToolClass +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult + +@Serializable +@TrailblazeToolClass( + name = "rememberTextBySelector", + surfaceToLlm = false, +) +@LLMDescription( + "Captures the text of the element matching a selector into a memory variable, with no LLM call.", +) +/** + * ----- DO NOT GIVE THIS TOOL TO THE LLM ----- + * + * The deterministic counterpart to [RememberTextTrailblazeTool]. Where `rememberText` describes the + * element in natural language and spends an LLM call to pick a locator, this names the element with + * the same [TrailblazeNodeSelector] grammar `assertVisibleBySelector` / `findMatches` use and reads + * its text straight off the captured tree. Zero LLM calls, so a capture replays on a + * recording-only leg and on iOS, where the prompt path's locator step fails outright. + * + * `rememberText` is unchanged and still the right tool when the value can only be described, not + * selected. + */ +data class RememberTextBySelectorTrailblazeTool( + val reason: String? = null, + /** Selector for the element whose text is captured. Required — [execute] enforces non-null. */ + val nodeSelector: TrailblazeNodeSelector? = null, + /** Memory variable name the captured text is stored under. */ + val variable: String, +) : ExecutableTrailblazeTool, ReadOnlyTrailblazeTool { + + override suspend fun execute( + toolExecutionContext: TrailblazeToolExecutionContext, + ): TrailblazeToolResult = when ( + val capture = captureSelectorText(TOOL_NAME, toolExecutionContext, nodeSelector) + ) { + is SelectorTextCapture.Failed -> capture.error + is SelectorTextCapture.Captured -> { + toolExecutionContext.memory.remember(variable, capture.text) + TrailblazeToolResult.Success( + message = "$TOOL_NAME: remembered $variable = " + + renderCaptured(toolExecutionContext, variable, capture.text), + ) + } + } +} + +private const val TOOL_NAME = "rememberTextBySelector" diff --git a/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/SelectorTextCapture.kt b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/SelectorTextCapture.kt new file mode 100644 index 000000000..60e482e4d --- /dev/null +++ b/trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/SelectorTextCapture.kt @@ -0,0 +1,156 @@ +package xyz.block.trailblaze.toolcalls.commands.memory + +import xyz.block.trailblaze.api.DriverNodeDetail +import xyz.block.trailblaze.api.DriverNodeMatch +import xyz.block.trailblaze.api.MatchDescriptorBuilder +import xyz.block.trailblaze.api.TargetTemplateContext +import xyz.block.trailblaze.api.TrailblazeNode +import xyz.block.trailblaze.api.TrailblazeNodeSelector +import xyz.block.trailblaze.api.TrailblazeNodeSelectorResolver +import xyz.block.trailblaze.toolcalls.SnapshotCache +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult + +/** + * Outcome of [captureSelectorText]: either the matched node's text, or the error the calling tool + * should return verbatim. + */ +internal sealed interface SelectorTextCapture { + data class Captured(val text: String) : SelectorTextCapture + + data class Failed(val error: TrailblazeToolResult.Error) : SelectorTextCapture +} + +/** + * Reads the driver-native text of the ONE node matching [nodeSelector] on the current screen. + * + * Shared by [RememberTextBySelectorTrailblazeTool] and [RememberNumberBySelectorTrailblazeTool] — + * the deterministic, zero-LLM counterparts of the prompt-based `rememberText` / `rememberNumber`, + * whose locator step always spends a model call and therefore cannot replay on a recording-only + * leg. Resolution goes through the same [TrailblazeNodeSelectorResolver] as `findMatches` / + * `assertVisibleBySelector`. The captured text comes from the field the selector matched on (see + * [pinnedTextField]), falling back to the cross-driver [MatchDescriptorBuilder.extractIdentity] + * rule `findMatches` reports as `matchedText` when the selector doesn't name one. + * + * A selector that matches more than once is an ERROR rather than a first-match pick: which node + * won would depend on resolver traversal order, reintroducing the non-determinism this path exists + * to remove. + * + * Point-in-time, like `findMatches` with no `timeoutMs`: it reads the screen as captured, and does + * not wait for an element to render. Precede it with an assertion that waits (e.g. + * `assertVisibleBySelector`) when the value appears after a navigation. + */ +internal fun captureSelectorText( + toolName: String, + toolExecutionContext: TrailblazeToolExecutionContext, + nodeSelector: TrailblazeNodeSelector?, +): SelectorTextCapture { + val selector = nodeSelector + ?: return failed("$toolName requires `nodeSelector` to be non-null.") + + // Read-only, so routing through the batch's SnapshotCache frame reuses a tree an earlier query + // already paid for; outside a frame this falls back to a direct capture. + val provider = toolExecutionContext.screenStateProvider + val screenState = if (provider != null) { + SnapshotCache.snapshot(provider, toolExecutionContext.traceId?.traceId) + } else { + toolExecutionContext.screenState + } + val tree: TrailblazeNode = screenState?.trailblazeNodeTree + ?: return failed( + "$toolName: current driver does not produce a TrailblazeNode tree " + + "(platform=${screenState?.trailblazeDevicePlatform?.name ?: "unknown"}). " + + "The selector cannot be resolved.", + ) + + val target = toolExecutionContext.resolvedTarget?.let { resolved -> + TargetTemplateContext(appId = toolExecutionContext.appId, appIds = resolved.appIds) + } + val selectorDesc = selector.description() + val matches = when (val result = TrailblazeNodeSelectorResolver.resolve(tree, selector, target)) { + is TrailblazeNodeSelectorResolver.ResolveResult.NoMatch -> emptyList() + is TrailblazeNodeSelectorResolver.ResolveResult.SingleMatch -> listOf(result.node) + is TrailblazeNodeSelectorResolver.ResolveResult.MultipleMatches -> result.nodes + } + if (matches.isEmpty()) { + return failed("$toolName: no element matched '$selectorDesc'.") + } + if (matches.size > 1) { + return failed( + "$toolName: '$selectorDesc' matched ${matches.size} elements; the captured value would " + + "depend on resolution order. Narrow the selector so it matches exactly one element.", + ) + } + + val detail = matches.single().driverDetail + val text = pinnedTextField(detail, selector.driverMatch) + ?: MatchDescriptorBuilder.extractIdentity(detail).matchedText + if (text.isNullOrEmpty()) { + return failed("$toolName: the element matching '$selectorDesc' carries no text to capture.") + } + return SelectorTextCapture.Captured(text) +} + +private fun failed(message: String): SelectorTextCapture.Failed = + SelectorTextCapture.Failed(TrailblazeToolResult.Error.ExceptionThrown(errorMessage = message)) + +/** + * The value of the one text-bearing field [match] constrained, or null when it named none or + * several. + * + * Every driver collapses its several text fields into one by a fixed priority — AXe reads AXLabel + * before AXValue — which answers the wrong field whenever the selector matched on a lower-priority + * one. A Contacts row labels the field type ("home") and carries the number in AXValue, so + * selecting it by `valueRegex` and reading the priority remembers "home". Only the fields a + * driver's priority chooses between are considered, so this can only ever re-point the capture + * within that set; a selector naming several leaves the priority to break the tie. + * + * Same-shape selectors only: the `iosMaestro` → AXe bridge matches one pattern against a cluster + * of fields, so which one matched isn't recoverable there. + */ +private fun pinnedTextField(detail: DriverNodeDetail, match: DriverNodeMatch?): String? = when { + match is DriverNodeMatch.IosAxe && detail is DriverNodeDetail.IosAxe -> listOfNotNull( + match.labelRegex?.let { detail.label }, + match.valueRegex?.let { detail.value }, + match.titleRegex?.let { detail.title }, + ) + + match is DriverNodeMatch.AndroidAccessibility && + detail is DriverNodeDetail.AndroidAccessibility -> listOfNotNull( + match.textRegex?.let { detail.resolveText() }, + match.hintTextRegex?.let { detail.hintText }, + match.contentDescriptionRegex?.let { detail.contentDescription }, + ) + + match is DriverNodeMatch.Compose && detail is DriverNodeDetail.Compose -> listOfNotNull( + match.textRegex?.let { detail.resolveText() }, + match.editableTextRegex?.let { detail.editableText }, + match.contentDescriptionRegex?.let { detail.contentDescription }, + ) + + match is DriverNodeMatch.AndroidMaestro && detail is DriverNodeDetail.AndroidMaestro -> + listOfNotNull( + match.textRegex?.let { detail.resolveText() }, + match.hintTextRegex?.let { detail.hintText }, + match.accessibilityTextRegex?.let { detail.accessibilityText }, + ) + + match is DriverNodeMatch.IosMaestro && detail is DriverNodeDetail.IosMaestro -> listOfNotNull( + match.textRegex?.let { detail.resolveText() }, + match.hintTextRegex?.let { detail.hintText }, + match.accessibilityTextRegex?.let { detail.accessibilityText }, + ) + + else -> emptyList() +}.singleOrNull() + +/** + * The captured value as it may appear in a result message. A `--secret` / `rememberSensitive` key + * is a session-lifetime redaction promise, and this message rides into logs and the LLM-facing + * result surface — so a sensitive variable renders redacted rather than echoing the value back out. + */ +internal fun renderCaptured( + toolExecutionContext: TrailblazeToolExecutionContext, + variable: String, + value: String, +): String = if (variable in toolExecutionContext.memory.sensitiveKeys) "[REDACTED]" else "'$value'" diff --git a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/maestro/AssertionLoggerTest.kt b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/maestro/AssertionLoggerTest.kt new file mode 100644 index 000000000..9426b5d6d --- /dev/null +++ b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/maestro/AssertionLoggerTest.kt @@ -0,0 +1,104 @@ +package xyz.block.trailblaze.maestro + +import kotlinx.datetime.Clock +import maestro.DeviceInfo +import maestro.Maestro +import maestro.TreeNode +import maestro.device.Platform +import maestro.orchestra.AssertConditionCommand +import maestro.orchestra.Condition +import maestro.orchestra.ElementSelector +import maestro.orchestra.MaestroCommand +import xyz.block.trailblaze.api.CaptureCoverage +import xyz.block.trailblaze.api.ScreenState +import xyz.block.trailblaze.api.ViewHierarchyTreeNode +import xyz.block.trailblaze.devices.TrailblazeDeviceClassifier +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.logs.client.LogEmitter +import xyz.block.trailblaze.logs.client.ScreenStateLogger +import xyz.block.trailblaze.logs.client.TrailblazeLog +import xyz.block.trailblaze.logs.client.TrailblazeLogger +import xyz.block.trailblaze.logs.client.TrailblazeSession +import xyz.block.trailblaze.logs.model.SessionId +import xyz.block.trailblaze.viewmatcher.matching.ViewHierarchyOnlyDriver +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * The report's `accessibility_truncation` signal rolls [CaptureCoverage] up from driver logs, so a + * capture whose log drops it is invisible to that signal — including the assert's own capture, which + * is exactly the one a spurious assertion failure needs explained. + */ +class AssertionLoggerTest { + + @Test + fun `assertion log carries the capture coverage of the screen state it asserted against`() { + val coverage = CaptureCoverage( + contentNodes = 12, + zeroBoundsContentNodes = 9, + horizontalCoverage = 0.2, + verticalCoverage = 0.95, + looksTruncated = true, + reason = "content jammed against the right edge", + ) + + assertEquals(coverage, logAssertionAgainst(FakeScreenState(captureCoverage = coverage)).captureCoverage) + } + + @Test + fun `assertion log carries no capture coverage when the driver produced none`() { + assertNull(logAssertionAgainst(FakeScreenState(captureCoverage = null)).captureCoverage) + } + + private fun logAssertionAgainst(screenState: ScreenState): TrailblazeLog.AgentDriverLog { + val captured = mutableListOf() + AssertionLogger( + maestro = viewHierarchyOnlyMaestro(), + screenStateProvider = { screenState }, + trailblazeLogger = TrailblazeLogger( + logEmitter = LogEmitter(captured::add), + screenStateLogger = ScreenStateLogger { it.fileName }, + ), + sessionProvider = { + TrailblazeSession(sessionId = SessionId("test_session"), startTime = Clock.System.now()) + }, + ).logSuccessfulAssertionCommand( + MaestroCommand( + AssertConditionCommand(condition = Condition(visible = ElementSelector(textRegex = "Welcome"))), + ), + ) + return captured.filterIsInstance().single() + } + + private fun viewHierarchyOnlyMaestro(): Maestro = Maestro( + driver = ViewHierarchyOnlyDriver( + rootTreeNode = TreeNode(), + deviceInfo = DeviceInfo( + platform = Platform.ANDROID, + widthPixels = 1080, + heightPixels = 2400, + widthGrid = 1080, + heightGrid = 2400, + ), + ), + ) + + private class FakeScreenState( + override val captureCoverage: CaptureCoverage?, + ) : ScreenState { + // PNG magic number: TrailblazeLogger.logScreenState rejects short/absent bytes, which would + // leave the log without a screenshot file. + override val screenshotBytes: ByteArray = byteArrayOf( + 0x89.toByte(), 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, + ) + override val deviceWidth: Int = 1080 + override val deviceHeight: Int = 2400 + override val viewHierarchy: ViewHierarchyTreeNode = ViewHierarchyTreeNode( + nodeId = 1, + className = "FrameLayout", + ) + override val trailblazeDevicePlatform: TrailblazeDevicePlatform = TrailblazeDevicePlatform.ANDROID + override val deviceClassifiers: List = emptyList() + } +} diff --git a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertToolNodeSelectorTest.kt b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertToolNodeSelectorTest.kt index 9a3f0e840..6536a15f8 100644 --- a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertToolNodeSelectorTest.kt +++ b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertToolNodeSelectorTest.kt @@ -180,6 +180,26 @@ class AssertToolNodeSelectorTest { assertTrue(agent.maestroCommandsExecuted, "Should have fallen back to Maestro after null") } + @Test + fun `AssertVisibleBySelector forwards timeoutMs to the Maestro path`() = runBlocking { + val nodeSelector = TrailblazeNodeSelector.withMatch( + DriverNodeMatch.AndroidAccessibility(textRegex = "Authorizing"), + ) + + val budgeted = AssertVisibleBySelectorTrailblazeTool(nodeSelector = nodeSelector, timeoutMs = 60_000) + assertEquals( + "60000", + assertIs(budgeted.toMaestroCommands().single()).timeout, + "an author-set appearance budget must reach Maestro rather than being replaced by its default", + ) + + val unbudgeted = AssertVisibleBySelectorTrailblazeTool(nodeSelector = nodeSelector) + assertNull( + assertIs(unbudgeted.toMaestroCommands().single()).timeout, + "an unset timeoutMs must leave Maestro on its own default", + ) + } + // endregion // region AssertNotVisibleBySelectorTrailblazeTool diff --git a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberBySelectorTrailblazeToolTest.kt b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberBySelectorTrailblazeToolTest.kt new file mode 100644 index 000000000..1a4afa1c6 --- /dev/null +++ b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/RememberBySelectorTrailblazeToolTest.kt @@ -0,0 +1,336 @@ +package xyz.block.trailblaze.toolcalls.commands.memory + +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlinx.coroutines.runBlocking +import kotlinx.datetime.Clock +import xyz.block.trailblaze.AgentMemory +import xyz.block.trailblaze.api.AnnotationElement +import xyz.block.trailblaze.api.DriverNodeDetail +import xyz.block.trailblaze.api.DriverNodeMatch +import xyz.block.trailblaze.api.ScreenState +import xyz.block.trailblaze.api.TrailblazeNode +import xyz.block.trailblaze.api.TrailblazeNodeSelector +import xyz.block.trailblaze.api.ViewHierarchyTreeNode +import xyz.block.trailblaze.devices.TrailblazeDeviceClassifier +import xyz.block.trailblaze.devices.TrailblazeDeviceId +import xyz.block.trailblaze.devices.TrailblazeDeviceInfo +import xyz.block.trailblaze.devices.TrailblazeDevicePlatform +import xyz.block.trailblaze.devices.TrailblazeDriverType +import xyz.block.trailblaze.logs.client.TrailblazeLogger +import xyz.block.trailblaze.logs.client.TrailblazeSession +import xyz.block.trailblaze.logs.client.TrailblazeSessionProvider +import xyz.block.trailblaze.logs.model.SessionId +import xyz.block.trailblaze.toolcalls.SnapshotCache +import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext +import xyz.block.trailblaze.toolcalls.TrailblazeToolResult + +/** + * Every case here runs the tool with a context that carries no LLM client and no + * [xyz.block.trailblaze.utils.ElementComparator] — the capture succeeding under those conditions is + * the zero-LLM property, enforced by construction rather than by counting calls. + */ +class RememberBySelectorTrailblazeToolTest { + + @AfterTest + fun cleanup() { + repeat(SnapshotCache.frameDepth()) { SnapshotCache.popFrame() } + } + + // -- Fixtures -- + + private fun androidNode( + text: String? = null, + resourceId: String? = null, + nodeId: Long = 0, + children: List = emptyList(), + ): TrailblazeNode = TrailblazeNode( + nodeId = nodeId, + children = children, + bounds = TrailblazeNode.Bounds(0, 0, 100, 50), + driverDetail = DriverNodeDetail.AndroidAccessibility(text = text, resourceId = resourceId), + ) + + private fun selector(textRegex: String) = TrailblazeNodeSelector.withMatch( + DriverNodeMatch.AndroidAccessibility(textRegex = textRegex), + ) + + private fun iosAxeNode( + label: String? = null, + value: String? = null, + nodeId: Long = 0, + children: List = emptyList(), + ): TrailblazeNode = TrailblazeNode( + nodeId = nodeId, + children = children, + bounds = TrailblazeNode.Bounds(0, 0, 100, 50), + driverDetail = DriverNodeDetail.IosAxe(label = label, value = value), + ) + + private class FakeScreenState( + val root: TrailblazeNode?, + platform: TrailblazeDevicePlatform, + ) : ScreenState { + override val screenshotBytes: ByteArray? = null + override val deviceWidth: Int = 1080 + override val deviceHeight: Int = 1920 + override val viewHierarchy: ViewHierarchyTreeNode = ViewHierarchyTreeNode() + override val trailblazeDevicePlatform: TrailblazeDevicePlatform = platform + override val deviceClassifiers: List = emptyList() + override val trailblazeNodeTree: TrailblazeNode? = root + override val annotationElements: List? = null + } + + private fun ctx( + tree: TrailblazeNode?, + memory: AgentMemory = AgentMemory(), + platform: TrailblazeDevicePlatform = TrailblazeDevicePlatform.ANDROID, + ): TrailblazeToolExecutionContext { + val state = FakeScreenState(tree, platform) + return TrailblazeToolExecutionContext( + screenState = state, + traceId = null, + trailblazeDeviceInfo = TrailblazeDeviceInfo( + trailblazeDeviceId = TrailblazeDeviceId( + instanceId = "test", + trailblazeDevicePlatform = platform, + ), + trailblazeDriverType = TrailblazeDriverType.ANDROID_ONDEVICE_INSTRUMENTATION, + widthPixels = 1080, + heightPixels = 1920, + ), + sessionProvider = TrailblazeSessionProvider { + TrailblazeSession(sessionId = SessionId("test"), startTime = Clock.System.now()) + }, + screenStateProvider = { state }, + trailblazeLogger = TrailblazeLogger.createNoOp(), + memory = memory, + ) + } + + private fun errorMessage(result: TrailblazeToolResult): String { + assertIs(result) + return result.errorMessage + } + + // -- rememberTextBySelector -- + + /** + * Negative control: the screen carries two option rows with DIFFERENT labels, and only the + * selected one may land in memory. An implementation that grabs the root, the first node, or any + * node fails here — unlike a "nothing threw" assertion, which such an implementation would pass. + */ + @Test + fun `captures the text of the selected element, not a sibling`() = runBlocking { + val memory = AgentMemory() + val tree = androidNode( + nodeId = 1, + children = listOf( + androidNode(text = "Print at the end of the sale", nodeId = 2), + androidNode(text = "Print when the order is ready", nodeId = 3), + ), + ) + + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = selector("Print when the order is ready"), + variable = "currentOption", + ).execute(ctx(tree, memory)) + + assertIs(result) + assertEquals("Print when the order is ready", memory.variables["currentOption"]) + } + + /** + * A Contacts-style row labels the field type and carries the datum in AXValue, so the AXe text + * priority (label > value > title) answers "home" for a row the trail selected by its number. + * Capturing the field the selector matched on is what keeps rememberNumberBySelector able to + * parse a number out of a value that plainly contained one. + */ + @Test + fun `captures the AXe field the selector matched on, not the higher-priority label`() = runBlocking { + val memory = AgentMemory() + val tree = iosAxeNode( + nodeId = 1, + children = listOf(iosAxeNode(label = "home", value = "(555) 478-7672", nodeId = 2)), + ) + + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = TrailblazeNodeSelector.withMatch( + DriverNodeMatch.IosAxe(valueRegex = "(555) 478-7672"), + ), + variable = "phone", + ).execute(ctx(tree, memory, TrailblazeDevicePlatform.IOS)) + + assertIs(result) + assertEquals("(555) 478-7672", memory.variables["phone"]) + } + + /** + * The same rule on another driver: Android resolves text before contentDescription, so a node + * selected by its contentDescription must not be remembered as its text. + */ + @Test + fun `captures the Android field the selector matched on, not the higher-priority text`() = runBlocking { + val memory = AgentMemory() + val tree = androidNode( + nodeId = 1, + children = listOf( + TrailblazeNode( + nodeId = 2, + children = emptyList(), + bounds = TrailblazeNode.Bounds(0, 0, 100, 50), + driverDetail = DriverNodeDetail.AndroidAccessibility( + text = "home", + contentDescription = "(555) 478-7672", + ), + ), + ), + ) + + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = TrailblazeNodeSelector.withMatch( + DriverNodeMatch.AndroidAccessibility(contentDescriptionRegex = "(555) 478-7672"), + ), + variable = "phone", + ).execute(ctx(tree, memory)) + + assertIs(result) + assertEquals("(555) 478-7672", memory.variables["phone"]) + } + + @Test + fun `no match is an error and leaves memory untouched`() = runBlocking { + val memory = AgentMemory() + val tree = androidNode(nodeId = 1, children = listOf(androidNode(text = "Save", nodeId = 2))) + + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = selector("Absent"), + variable = "currentOption", + ).execute(ctx(tree, memory)) + + assertTrue(errorMessage(result).contains("no element matched")) + assertNull(memory.variables["currentOption"]) + } + + /** + * Ambiguity must fail loudly: picking a winner would make the captured value depend on resolver + * traversal order, which is the non-determinism this tool exists to remove. + */ + @Test + fun `multiple matches is an error and leaves memory untouched`() = runBlocking { + val memory = AgentMemory() + val tree = androidNode( + nodeId = 1, + children = listOf( + androidNode(text = "Item", nodeId = 2), + androidNode(text = "Item", nodeId = 3), + ), + ) + + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = selector("Item"), + variable = "currentOption", + ).execute(ctx(tree, memory)) + + assertTrue(errorMessage(result).contains("matched 2 elements")) + assertNull(memory.variables["currentOption"]) + } + + @Test + fun `an element with no text is an error and leaves memory untouched`() = runBlocking { + val memory = AgentMemory() + val tree = androidNode( + nodeId = 1, + children = listOf(androidNode(resourceId = "app:id/option_row", nodeId = 2)), + ) + + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = TrailblazeNodeSelector.withMatch( + DriverNodeMatch.AndroidAccessibility(resourceIdRegex = "app:id/option_row"), + ), + variable = "currentOption", + ).execute(ctx(tree, memory)) + + assertTrue(errorMessage(result).contains("no text to capture")) + assertNull(memory.variables["currentOption"]) + } + + @Test + fun `a missing nodeSelector is an error`() = runBlocking { + val result = RememberTextBySelectorTrailblazeTool(variable = "currentOption") + .execute(ctx(androidNode(nodeId = 1))) + + assertTrue(errorMessage(result).contains("requires `nodeSelector` to be non-null")) + } + + @Test + fun `a driver with no node tree is an error naming the platform`() = runBlocking { + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = selector("Save"), + variable = "currentOption", + ).execute(ctx(tree = null, platform = TrailblazeDevicePlatform.IOS)) + + assertTrue(errorMessage(result).contains("platform=IOS")) + } + + /** + * A `--secret` key is a session-lifetime redaction promise, and the result message reaches logs + * and the LLM-facing result surface — so the captured value must not ride back out in it. + */ + @Test + fun `a sensitive variable is redacted in the result message`() = runBlocking { + val memory = AgentMemory() + memory.rememberSensitive("pin", "0000") + val tree = androidNode(nodeId = 1, children = listOf(androidNode(text = "4821", nodeId = 2))) + + val result = RememberTextBySelectorTrailblazeTool( + nodeSelector = selector("4821"), + variable = "pin", + ).execute(ctx(tree, memory)) + + assertIs(result) + assertEquals("4821", memory.variables["pin"]) + assertTrue(result.message?.contains("[REDACTED]") == true) + assertTrue(result.message?.contains("4821") == false) + } + + // -- rememberNumberBySelector -- + + @Test + fun `captures the number out of the selected element's text`() = runBlocking { + val memory = AgentMemory() + val tree = androidNode( + nodeId = 1, + children = listOf( + androidNode(text = "Subtotal $12.00", nodeId = 2), + androidNode(text = "Total $42.50", nodeId = 3), + ), + ) + + val result = RememberNumberBySelectorTrailblazeTool( + nodeSelector = selector("Total .*"), + variable = "total", + ).execute(ctx(tree, memory)) + + assertIs(result) + assertEquals("42.50", memory.variables["total"]) + } + + @Test + fun `text with no number is an error and leaves memory untouched`() = runBlocking { + val memory = AgentMemory() + val tree = androidNode(nodeId = 1, children = listOf(androidNode(text = "Sold out", nodeId = 2))) + + val result = RememberNumberBySelectorTrailblazeTool( + nodeSelector = selector("Sold out"), + variable = "total", + ).execute(ctx(tree, memory)) + + assertTrue(errorMessage(result).contains("no number found")) + assertNull(memory.variables["total"]) + } +} diff --git a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/ToolSerializationTest.kt b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/ToolSerializationTest.kt index dbfc872e7..cc1cde003 100644 --- a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/ToolSerializationTest.kt +++ b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/ToolSerializationTest.kt @@ -31,7 +31,9 @@ import xyz.block.trailblaze.toolcalls.commands.WaitForIdleSyncTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.memory.AssertEqualsTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.memory.AssertMathTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.memory.AssertNotEqualsTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.memory.RememberNumberBySelectorTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.memory.RememberNumberTrailblazeTool +import xyz.block.trailblaze.toolcalls.commands.memory.RememberTextBySelectorTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.memory.RememberTextTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.memory.RememberWithAiTrailblazeTool import xyz.block.trailblaze.toolcalls.commands.MaestroTrailblazeTool @@ -1172,4 +1174,55 @@ trail: assertThat(tool.exact).isEqualTo(3) assertThat(tool.min).isEqualTo(null) } + + @Test + fun rememberTextBySelectorRoundTrip() { + val yaml = """ +config: {} +trail: + - step: recorded + recording: + android: + - rememberTextBySelector: + reason: Capture the currently selected check-reporting option. + variable: currentOption + nodeSelector: + androidAccessibility: + textRegex: "Print when the order is ready" + """.trimIndent() + + val tools = decodeRecordedTools(yaml) + val reDecoded = trailblazeYaml.decodeTools(trailblazeYaml.encodeTools(tools)) + assertThat(reDecoded.size).isEqualTo(1) + assertThat(reDecoded[0].name).isEqualTo("rememberTextBySelector") + val tool = reDecoded[0].trailblazeTool as RememberTextBySelectorTrailblazeTool + assertThat(tool.variable).isEqualTo("currentOption") + assertThat(tool.reason).isEqualTo("Capture the currently selected check-reporting option.") + val match = tool.nodeSelector!!.driverMatch as DriverNodeMatch.AndroidAccessibility + assertThat(match.textRegex).isEqualTo("Print when the order is ready") + } + + @Test + fun rememberNumberBySelectorRoundTrip() { + val yaml = """ +config: {} +trail: + - step: recorded + recording: + android: + - rememberNumberBySelector: + variable: total + nodeSelector: + androidAccessibility: + textRegex: "Total .*" + """.trimIndent() + + val tools = decodeRecordedTools(yaml) + val reDecoded = trailblazeYaml.decodeTools(trailblazeYaml.encodeTools(tools)) + assertThat(reDecoded.size).isEqualTo(1) + assertThat(reDecoded[0].name).isEqualTo("rememberNumberBySelector") + val tool = reDecoded[0].trailblazeTool as RememberNumberBySelectorTrailblazeTool + assertThat(tool.variable).isEqualTo("total") + assertThat(tool.reason).isEqualTo(null) + } } diff --git a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/TrailYamlValidationTest.kt b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/TrailYamlValidationTest.kt index 4409884be..b1862b92b 100644 --- a/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/TrailYamlValidationTest.kt +++ b/trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/yaml/TrailYamlValidationTest.kt @@ -5,9 +5,21 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.runBlocking import org.junit.Test +import xyz.block.trailblaze.config.ScriptedToolNameDiscoverer +import xyz.block.trailblaze.config.ToolNameResolver +import xyz.block.trailblaze.config.project.TrailDiscovery +import xyz.block.trailblaze.logs.client.TrailblazeSerializationInitializer import xyz.block.trailblaze.util.GitUtils +import xyz.block.trailblaze.yaml.unified.TrailDocument import java.io.File +import java.io.IOException +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes import kotlin.test.assertFailsWith +import kotlin.test.assertFalse import kotlin.test.assertTrue import xyz.block.trailblaze.util.Console @@ -155,4 +167,365 @@ class TrailYamlValidationTest { "Strict parse should fail on the misspelled bound, but was: ${strictFailure.message}", ) } + + /** + * Fails when a trail calls a tool NAME that no backing in this repo claims — the gap the + * strictness above cannot see. Tool-name decode is permissive by construction + * ([TrailblazeToolYamlWrapperSerializer][xyz.block.trailblaze.yaml.serializers.TrailblazeToolYamlWrapperSerializer] + * falls back to [xyz.block.trailblaze.logs.client.temp.OtherTrailblazeTool] for any unrecognized + * key), so a misspelled tool parses cleanly and only fails hours later on a device, inside + * `TrailblazeToolRepo.toolCallToTrailblazeTool` — at whatever step happens to look first, which + * misattributes the cause. + * + * Names come from the real decoder (never a regex over the YAML) so this can't drift from what + * the runtime actually reads. See [ToolNameBackings] for how the known-name set is derived and + * where it is deliberately a superset. + */ + @Test + fun `every tool name a trail references resolves to a known backing`() { + val gitRoot = File( + GitUtils.getGitRootViaCommand() ?: error("Failed to determine git repository root"), + ) + val backings = ToolNameBackings.scan(gitRoot) + val strictParser = createTrailblazeYaml(strict = true) + + val trailFiles = TrailYamlValidator.findAllTrailYamlFiles(gitRoot) + val referencedIn = mutableMapOf>() + var callSites = 0 + var undecodable = 0 + trailFiles.forEach { file -> + val names = try { + ToolNameBackings.toolNamesReferencedBy(file.readText(), strictParser) + } catch (_: Exception) { + // A file that doesn't parse is the corpus test above's failure to report, not this one's. + undecodable++ + return@forEach + } + callSites += names.size + names.forEach { name -> + referencedIn.getOrPut(name) { mutableSetOf() }.add(file.relativeTo(gitRoot).path) + } + } + + val unknown = referencedIn.keys.filterNot { it in backings.known }.sorted() + // Gradle never prints a test's stdout, so a census that only went to the log would be invisible on + // the runs that matter. It rides in every failure message below as well. + val census = backings.census(referencedIn.keys, trailFiles.size, callSites, undecodable, unknown) + Console.log(census) + + // A resolution regression that empties either side of the comparison must not read as a pass. + assertTrue( + referencedIn.isNotEmpty(), + "Adjudicated 0 tool names across ${trailFiles.size} trail file(s) — the corpus walk " + + "resolved nothing, so this gate proved nothing. Check that trail discovery and " + + "`decodeTrailDocument` still surface `recording:` tool calls.\n$census", + ) + assertTrue( + backings.known.isNotEmpty(), + "Known-tool-name set is empty, so every name would read as a typo. Check tool discovery " + + "(${ToolNameBackings.REGISTRY_ENTRY_POINTS}) and the worktree scan under $gitRoot.\n$census", + ) + assertTrue( + unknown.isEmpty(), + buildString { + appendLine("${unknown.size} trail tool name(s) match no known backing in this repo:") + unknown.forEach { name -> + appendLine(" - $name") + referencedIn.getValue(name).sorted().forEach { appendLine(" $it") } + } + appendLine( + "Each is a tool name no `.tool.yaml` id, scripted-tool declaration or " + + "@TrailblazeToolClass in the tree claims. Fix the spelling in the trail, or declare " + + "the tool.", + ) + append(census) + }, + ) + } + + /** + * Guards the gate itself, the same way `strict parser rejects unknown keys` guards strictness: + * a clean corpus passes whether the gate works or is silently dead, so prove both halves move. + * The decoder must SURFACE a bogus tool name (it decodes permissively, so absence here would + * mean the walk is blind), and the known-name set must ACCEPT a real name while REJECTING the + * bogus one. The assertions key off the names, not the wording, so they survive rewording. + */ + @Test + fun `tool-name gate surfaces an unknown name and clears a known one`() { + val bogus = "assertVisibleBySelectorNoSuchToolProbe" + val probe = """ + config: + id: probe/tool-name-gate + title: probe + target: square + trail: + - step: s + recording: + ios: + - assertVisibleBySelector: + reason: r + nodeSelector: + iosMaestro: + textRegex: More + - $bogus: + reason: r + """.trimIndent() + + val names = ToolNameBackings.toolNamesReferencedBy(probe, createTrailblazeYaml(strict = true)) + assertTrue( + names.containsAll(listOf("assertVisibleBySelector", bogus)), + "The decoder must surface both tool names for the gate to see them, but got: $names", + ) + + val gitRoot = File( + GitUtils.getGitRootViaCommand() ?: error("Failed to determine git repository root"), + ) + val known = ToolNameBackings.scan(gitRoot).known + assertTrue("assertVisibleBySelector" in known, "A registered tool must read as known") + assertFalse(bogus in known, "An undeclared tool name must NOT read as known") + } + + /** + * `variable` is the load-bearing key — a capture that writes nothing leaves every downstream + * `{{token}}` literal, which reads as a trail bug far from its cause. It is declared required + * rather than defaulted precisely so a misspelling is rejected at parse time by BOTH parsers, + * not silently dropped by the lenient one. Also pins that the authoring shape parses strict. + */ + @Test + fun `both parsers reject a rememberTextBySelector with a misspelled variable`() { + val documented = """ + config: + id: probe/remember-text-by-selector + title: probe + target: square + trail: + - step: s + recording: + ios: + - rememberTextBySelector: + reason: Capture the currently selected check-reporting option. + variable: currentOption + nodeSelector: + iosAxe: + labelRegex: Print when the order is ready + """.trimIndent() + + createTrailblazeYaml(strict = true).decodeTrailDocument(documented) + + val misspelled = documented.replace("variable: currentOption", "varaible: currentOption") + assertFailsWith { + createTrailblazeYaml(strict = true).decodeTrailDocument(misspelled) + } + assertFailsWith { + createTrailblazeYaml().decodeTrailDocument(misspelled) + } + } +} + +/** + * Every place a tool NAME can be declared in this repo, and how the gate above decides a name is + * real. Split into two groups: + * + * **Registry (runtime).** [TrailblazeSerializationInitializer.buildAllTools] (class-backed), + * [TrailblazeSerializationInitializer.buildYamlDefinedTools] (`tools:`-mode YAML) and + * [ScriptedToolNameDiscoverer.discoverAllNames] (statically-named scripted descriptors), reached + * through [ToolNameResolver.isKnown]. These see only this test's CLASSPATH, which is a small slice + * of the repo: `:trailblaze-common:jvmTest` carries its own bundled trailmaps and nothing else — no + * `:trailblaze-playwright` / `:trailblaze-compose` tools, no `uitests-*` tools, and no workspace + * trailmap under `trails/config/trailmaps/` (`platformConfigResourceSource()` reads the classpath, + * and no workspace resolver is installed in a test JVM). + * + * **Worktree (filesystem).** The declarations the classpath can't reach, read from the git tree so + * the gate covers the whole repo rather than one module's slice: `id:` in + * `*.tool.yaml` / `*.shortcut.yaml` / `*.trailhead.yaml`, `name:` in scripted-tool descriptors and + * in a target's inline `tools:` entries, `registerTool("…")` in an MCP subprocess server, + * `@TrailblazeToolClass("…")` in Kotlin sources, and — for scripted TypeScript tools, whose names + * are only fully knowable by running the analyzer — a deliberate SUPERSET built from two + * independent extractors: every candidate `.ts` basename, and every + * `export const X = trailblaze.tool` declaration. + * + * The superset bias is the point: its only failure mode is MISSING a typo, never inventing one. A + * false failure here would get the gate disabled, which is strictly worse than a measured blind + * spot. Two blind spots follow from that and are reported on every run rather than implied: + * 1. a typo that collides with a `.ts` basename or an exported symbol reads as known; + * 2. tool calls nested inside another tool's arguments (a `block_runIf`-style wrapper, whose + * recorded body decodes to an opaque raw-args map) are not walked, so their names are not + * adjudicated at all. + * + * Never replace any of this with a hand-written list of tool names: a literal list rots into a + * second source of truth that says a real tool is a typo (or the reverse) with no signal. + */ +private object ToolNameBackings { + + const val REGISTRY_ENTRY_POINTS = + "TrailblazeSerializationInitializer.buildAllTools/buildYamlDefinedTools, " + + "ScriptedToolNameDiscoverer.discoverAllNames" + + /** + * Scope of the worktree scan for tool declarations: trailmap trees (`trailmaps//…`) and + * target configs (`targets/.yaml`, whose `target.tools:` entries name inline script tools). + * Anything outside `trails/config/` cannot declare a tool name, except Kotlin annotations, which + * are scanned repo-wide. + */ + private const val CONFIG_MARKER = "trails/config/" + private val WORD = "[A-Za-z0-9_${'$'}]" + private val ANNOTATED_NAME = Regex("""@TrailblazeToolClass\(\s*(?:name\s*=\s*)?"([^"]+)"""") + private val YAML_TOOL_ID = Regex("""^id:\s*(\S+)\s*$""", RegexOption.MULTILINE) + private val DESCRIPTOR_NAME = Regex("""^\s*(?:-\s+)?name:\s*"?($WORD+)"?\s*$""", RegexOption.MULTILINE) + private val EXPORTED_TOOL = Regex("""export\s+const\s+($WORD+)\s*=\s*trailblaze\s*\.\s*tool""") + + /** + * A tool advertised by an MCP server the host spawns as a subprocess at session start (the + * `sampleapp` trailmap's `tools/mcp/tools.ts`). Its name is a string literal in the + * `registerTool` / `tool` call, so it reads exactly rather than by heuristic — but it is invisible + * to every other extractor here (it is not a `trailblaze.tool`, has no descriptor YAML, and is + * deliberately excluded from the generated typed surface). + */ + private val MCP_REGISTERED_TOOL = Regex("""\.(?:registerTool|tool)\(\s*"([^"]+)"""") + + /** One partition of the known-name set: where the names came from, and whether it is exact. */ + class Partition(val label: String, val names: Set, val exact: Boolean = true) + + class Backings(val partitions: List) { + val known: Set = partitions.flatMapTo(mutableSetOf()) { it.names } + + /** Names only a superset partition claims — matched, but not proven to exist. */ + private val supersetOnly: Set = + partitions.filterNot { it.exact }.flatMapTo(mutableSetOf()) { it.names } - + partitions.filter { it.exact }.flatMapTo(mutableSetOf()) { it.names } + + fun census( + adjudicated: Set, + trailFiles: Int, + callSites: Int, + undecodable: Int, + unknown: List, + ): String = buildString { + appendLine("\n=== Trail tool-name resolution ===") + appendLine("Known-name set, by declaring backing:") + partitions.forEach { p -> + appendLine(" ${p.names.size.toString().padStart(4)} ${p.label}${if (p.exact) "" else " [SUPERSET — contributes names, never adjudicated or failed on]"}") + } + appendLine(" ${known.size.toString().padStart(4)} union") + appendLine("Corpus: $trailFiles trail file(s), $callSites recorded tool call(s)") + appendLine(" adjudicated: ${adjudicated.size} distinct tool name(s)") + appendLine(" skipped: $undecodable file(s) that did not decode (reported by the parse gate, not here)") + val bySuperset = adjudicated.count { it in supersetOnly } + appendLine( + " of the adjudicated, $bySuperset matched ONLY via the scripted-TypeScript superset — " + + "known-not-a-typo, not known-to-exist", + ) + appendLine( + " blind spot: tool calls nested inside another tool's args (e.g. a `block_runIf` body) " + + "decode to opaque raw args and are NOT walked", + ) + appendLine(if (unknown.isEmpty()) " unknown: 0" else " unknown: ${unknown.size} -> ${unknown.joinToString(", ")}") + } + } + + /** Tool names this trail YAML references, taken from the real decoder rather than a regex. */ + fun toolNamesReferencedBy(yaml: String, parser: TrailblazeYaml): List = + when (val doc = parser.decodeTrailDocument(yaml)) { + is TrailDocument.Unified -> + (listOfNotNull(doc.trail.trailhead) + doc.trail.trail) + .flatMap { step -> step.recordings.values.flatten() } + .map { it.name } + } + + fun scan(gitRoot: File): Backings { + val resolver = ToolNameResolver.fromBuiltInAndCustomTools() + val yamlToolIds = mutableSetOf() + val descriptorNames = mutableSetOf() + val scriptBasenames = mutableSetOf() + val exportedToolNames = mutableSetOf() + val mcpAdvertisedNames = mutableSetOf() + val annotatedNames = mutableSetOf() + + val root = gitRoot.toPath() + Files.walkFileTree( + root, + object : SimpleFileVisitor() { + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { + val name = dir.fileName?.toString() + return if (dir != root && name in TrailDiscovery.DEFAULT_EXCLUDED_DIRS) { + FileVisitResult.SKIP_SUBTREE + } else { + FileVisitResult.CONTINUE + } + } + + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { + if (!attrs.isRegularFile) return FileVisitResult.CONTINUE + val name = file.fileName?.toString() ?: return FileVisitResult.CONTINUE + val inConfig = file.toString().replace(File.separatorChar, '/').contains(CONFIG_MARKER) + when { + name.endsWith(".kt") -> read(file)?.let { text -> + if ("@TrailblazeToolClass" in text) { + ANNOTATED_NAME.findAll(text).forEach { annotatedNames.add(it.groupValues[1]) } + } + } + + !inConfig -> Unit + + name.endsWith(".tool.yaml") || name.endsWith(".shortcut.yaml") || + name.endsWith(".trailhead.yaml") -> read(file)?.let { text -> + YAML_TOOL_ID.findAll(text).forEach { yamlToolIds.add(it.groupValues[1]) } + } + + name.endsWith(".yaml") -> read(file)?.let { text -> + DESCRIPTOR_NAME.findAll(text).forEach { descriptorNames.add(it.groupValues[1]) } + } + + name.endsWith(".d.ts") || name.endsWith(".test.ts") -> Unit + + name.endsWith(".ts") -> { + scriptBasenames.add(name.removeSuffix(".ts")) + read(file)?.let { text -> + EXPORTED_TOOL.findAll(text).forEach { exportedToolNames.add(it.groupValues[1]) } + MCP_REGISTERED_TOOL.findAll(text).forEach { mcpAdvertisedNames.add(it.groupValues[1]) } + } + } + } + return FileVisitResult.CONTINUE + } + + // A file we can't read contributes no names; it must not abort the scan (which would + // shrink the known set and turn real tools into reported typos). + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult = + FileVisitResult.CONTINUE + }, + ) + + return Backings( + listOf( + Partition( + "classpath registry (class-backed / YAML-defined / scripted descriptors)", + registryNames().filter { resolver.isKnown(it) }.toSet(), + ), + Partition("worktree *.tool|shortcut|trailhead.yaml `id:`", yamlToolIds), + Partition("worktree `name:` in scripted descriptors / target `tools:` entries", descriptorNames), + Partition("worktree Kotlin @TrailblazeToolClass(\"…\")", annotatedNames), + Partition("worktree MCP subprocess registerTool(\"…\")", mcpAdvertisedNames), + Partition("worktree scripted .ts basenames", scriptBasenames, exact = false), + Partition("worktree scripted .ts `export const … = trailblaze.tool`", exportedToolNames, exact = false), + ), + ) + } + + /** + * The registry's own names, so the census can report the classpath partition's size. Filtered + * through [ToolNameResolver.isKnown] at the call site above so resolution — not this enumeration + * — remains the thing that decides a name is known. + */ + private fun registryNames(): Set = buildSet { + TrailblazeSerializationInitializer.buildAllTools().keys.forEach { add(it.toolName) } + TrailblazeSerializationInitializer.buildYamlDefinedTools().keys.forEach { add(it.toolName) } + ScriptedToolNameDiscoverer.discoverAllNames().forEach { add(it.toolName) } + } + + private fun read(file: Path): String? = try { + file.toFile().readText() + } catch (_: Exception) { + null + } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/desktop/TrailblazeDesktopAppConfig.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/desktop/TrailblazeDesktopAppConfig.kt index 701e3913e..5836f44df 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/desktop/TrailblazeDesktopAppConfig.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/desktop/TrailblazeDesktopAppConfig.kt @@ -9,6 +9,7 @@ import xyz.block.trailblaze.host.ios.MobileDeviceUtils import xyz.block.trailblaze.host.rules.TrailblazeHostDynamicLlmClientProvider import xyz.block.trailblaze.http.TrailblazeHttpClientFactory import xyz.block.trailblaze.llm.LlmProviderEnvVarUtil +import xyz.block.trailblaze.llm.config.BuiltInLlmModelRegistry import xyz.block.trailblaze.llm.config.TrailblazeConfigPaths import xyz.block.trailblaze.llm.TrailblazeLlmModel import xyz.block.trailblaze.llm.TrailblazeLlmModelList @@ -251,8 +252,13 @@ abstract class TrailblazeDesktopAppConfig( ?: defaultProviderModelList val selectedTrailblazeLlmModel: TrailblazeLlmModel = - currentProviderModelList.entries.firstOrNull { it.modelId == savedModelId } - ?: defaultLlmModel + resolveSavedModelWithinProvider( + entries = currentProviderModelList.entries, + savedModelId = savedModelId, + providerDefaultModelId = BuiltInLlmModelRegistry.defaultModelForProvider( + currentProviderModelList.provider, + ), + ) ?: defaultLlmModel return selectedTrailblazeLlmModel } @@ -517,3 +523,23 @@ abstract class TrailblazeDesktopAppConfig( } } } + +/** + * Resolves a persisted model selection against the models its provider currently offers. + * + * Exact id match wins. When the saved id is gone — the catalog retired it, or a workspace + * `llm.providers` block narrowed the provider to a different set — this falls back to the + * provider's own `default_model` rather than letting the caller drop to its global default. + * That global default is `NONE` in the OSS desktop distribution, so without this step + * retiring a model id silently disables a user's LLM on upgrade instead of moving them to + * the current default for the provider they chose. + * + * Returns null only when neither the saved id nor the provider default is present, which + * leaves the caller's own fallback in charge. + */ +internal fun resolveSavedModelWithinProvider( + entries: List, + savedModelId: String, + providerDefaultModelId: String?, +): TrailblazeLlmModel? = entries.firstOrNull { it.modelId == savedModelId } + ?: providerDefaultModelId?.let { defaultId -> entries.firstOrNull { it.modelId == defaultId } } diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostMaestroTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostMaestroTrailblazeAgent.kt index c96ac25ec..7ca46d6d7 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostMaestroTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostMaestroTrailblazeAgent.kt @@ -22,6 +22,7 @@ import xyz.block.trailblaze.logs.client.TrailblazeSessionProvider import xyz.block.trailblaze.logs.model.SessionId import xyz.block.trailblaze.logs.model.TraceId import xyz.block.trailblaze.model.NodeSelectorMode +import xyz.block.trailblaze.model.TapRouteOverride import xyz.block.trailblaze.model.ResolvedTarget import xyz.block.trailblaze.toolcalls.TrailblazeToolRepo import xyz.block.trailblaze.toolcalls.TrailblazeToolResult @@ -82,6 +83,8 @@ class HostMaestroTrailblazeAgent( override suspend fun executeNodeSelectorTap( nodeSelector: TrailblazeNodeSelector, longPress: Boolean, + // Ignored — this agent has only the coordinate-gesture path, so there is no route to pin. + tapRoute: TapRouteOverride?, traceId: TraceId?, ): TrailblazeToolResult? { val tree = getCurrentTrailblazeNodeTree(nodeSelector) ?: return null diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostOnDeviceRpcTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostOnDeviceRpcTrailblazeAgent.kt index 6ecf0ee55..98faca2a3 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostOnDeviceRpcTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/HostOnDeviceRpcTrailblazeAgent.kt @@ -29,6 +29,7 @@ import xyz.block.trailblaze.mcp.android.ondevice.rpc.OnDeviceRpcClient import xyz.block.trailblaze.mcp.android.ondevice.rpc.RpcResult import xyz.block.trailblaze.mcp.android.ondevice.rpc.GetScreenStateResponse import xyz.block.trailblaze.mcp.utils.RpcScreenStateAdapter +import xyz.block.trailblaze.model.TapRouteOverride import xyz.block.trailblaze.host.recording.StreamFrameMonitor import xyz.block.trailblaze.host.recording.StreamScreenshotScreenState import xyz.block.trailblaze.host.recording.DeviceStreamScreenshotSource @@ -543,6 +544,7 @@ class HostOnDeviceRpcTrailblazeAgent( override suspend fun executeNodeSelectorTap( nodeSelector: TrailblazeNodeSelector, longPress: Boolean, + tapRoute: TapRouteOverride?, traceId: TraceId?, ): TrailblazeToolResult? { // Accessibility-shaped selectors are recorded under the accessibility driver and must @@ -565,6 +567,7 @@ class HostOnDeviceRpcTrailblazeAgent( tool = TapOnByElementSelector( nodeSelector = nodeSelector, longPress = longPress, + tapRoute = tapRoute, ), traceId = traceId, ) diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/axe/AxeDeviceManager.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/axe/AxeDeviceManager.kt index bf9c465ec..5430c20d0 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/axe/AxeDeviceManager.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/axe/AxeDeviceManager.kt @@ -33,6 +33,20 @@ class AxeDeviceManager( ) : IosDeviceManager { companion object { + /** + * Env knob selecting how `clearState` is satisfied. Default (unset / any other value) is + * the byte-identical existing behavior: [SimctlCli.clearAppState]'s uninstall + bundle + * copy + reinstall. `container` routes to [SimctlCli.clearAppDataContainer] — an in-place + * delete of the data container's children (the host path's `mobile_clearAppData` wipe), + * measured ~9s cheaper per clear on a 1.3GB app bundle. Read on every clear (not cached + * at JVM start), so it can be flipped between runs. + */ + private const val CLEAR_STATE_MODE_ENV = "TRAILBLAZE_IOS_CLEAR_STATE_MODE" + + /** Pure gate for [CLEAR_STATE_MODE_ENV] — `container` (case-insensitive, trimmed) opts in. */ + internal fun useContainerClearState(raw: String?): Boolean = + raw?.trim()?.equals("container", ignoreCase = true) == true + /** Polling interval for element-resolution loops. Balances responsiveness with CPU usage. */ private const val POLL_INTERVAL_MS = 150L @@ -298,7 +312,7 @@ class AxeDeviceManager( // REINSTALL semantics — same clean-state guarantee Maestro's clearAppState gives. // Hard-fail on error: silently launching against dirty state is the bug this closes. // (clearAppState terminates the app itself, so stopFirst is subsumed.) - SimctlCli.clearAppState(udid, action.bundleId).throwIfError("clearState ${action.bundleId}") + clearStateViaConfiguredMode(action.bundleId) } else if (action.stopFirst) { // Best-effort: `simctl terminate` exits nonzero when the app isn't running, // which is a fine starting state for a force restart. @@ -344,12 +358,13 @@ class AxeDeviceManager( } is IosDriverAction.ClearState -> { // Standalone clearState — no launch afterwards, so no tree-readiness wait either. - SimctlCli.clearAppState(udid, action.bundleId).throwIfError("clearState ${action.bundleId}") + clearStateViaConfiguredMode(action.bundleId) // Maestro resets permissions to unset on a standalone clearState (Orchestra's // clearAppStateCommand, for Android parity). This path is deliberately MORE lenient - // than Maestro — Orchestra propagates setPermissions failures, but here the reinstall - // above already wiped the app's TCC rows, so this is redundant hardening not worth - // failing a completed wipe. Also disarms the notification auto-dismiss. + // than Maestro — Orchestra propagates setPermissions failures, but here the wipe + // above already reset the app's TCC rows (both modes do), so this is redundant + // hardening not worth failing a completed wipe. Also disarms the notification + // auto-dismiss. val privacyReset = SimctlCli.privacy(udid, "reset", "all", action.bundleId) if (!privacyReset.success) { Console.log( @@ -396,6 +411,21 @@ class AxeDeviceManager( notificationsAutoDismiss = plan.notificationsValue } + /** + * Routes a clearState request per [CLEAR_STATE_MODE_ENV] (see [useContainerClearState]): + * default = uninstall/reinstall via [SimctlCli.clearAppState]; `container` = in-place data + * container wipe. Both hard-fail on error — silently launching against dirty state is the + * bug clearState closes. + */ + private fun clearStateViaConfiguredMode(bundleId: String) { + if (useContainerClearState(System.getenv(CLEAR_STATE_MODE_ENV))) { + Console.log("[AxeDeviceManager] clearState via data-container wipe ($CLEAR_STATE_MODE_ENV=container) for $bundleId") + SimctlCli.clearAppDataContainer(udid, bundleId).throwIfError("clearState(container) $bundleId") + } else { + SimctlCli.clearAppState(udid, bundleId).throwIfError("clearState $bundleId") + } + } + /** * Polls `describe-ui` until the tree looks like real app content instead of a blind fixed * sleep — launch/deep-link render time is app-dependent, so a fixed delay is either too diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/IosDriverTrailblazeAgent.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/IosDriverTrailblazeAgent.kt index f51df213f..888437fa7 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/IosDriverTrailblazeAgent.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/IosDriverTrailblazeAgent.kt @@ -13,6 +13,7 @@ import xyz.block.trailblaze.logs.model.SessionId import xyz.block.trailblaze.logs.model.TraceId import xyz.block.trailblaze.model.NodeSelectorMode import xyz.block.trailblaze.model.ResolvedTarget +import xyz.block.trailblaze.model.TapRouteOverride import xyz.block.trailblaze.toolcalls.DelegatingTrailblazeTool import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool import xyz.block.trailblaze.toolcalls.TrailblazeTool @@ -100,6 +101,8 @@ class IosDriverTrailblazeAgent( override suspend fun executeNodeSelectorTap( nodeSelector: TrailblazeNodeSelector, longPress: Boolean, + // Ignored — the iOS driver has only one dispatch path, so there is no route to pin. + tapRoute: TapRouteOverride?, traceId: TraceId?, ): TrailblazeToolResult = IosDriverTrailRunner.runActions( actions = listOf(IosDriverAction.TapOnElement(nodeSelector, longPress = longPress)), diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/SimctlCli.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/SimctlCli.kt index 1a66b2710..1000bbd7d 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/SimctlCli.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/SimctlCli.kt @@ -1,10 +1,14 @@ package xyz.block.trailblaze.host.ios +import java.io.IOException +import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.LinkOption import java.nio.file.Path import java.nio.file.Paths +import java.nio.file.SimpleFileVisitor import java.nio.file.StandardCopyOption +import java.nio.file.attribute.BasicFileAttributes import java.util.concurrent.TimeUnit import kotlin.io.path.CopyActionResult import kotlin.io.path.ExperimentalPathApi @@ -66,6 +70,96 @@ object SimctlCli { fun keychainReset(udid: String, timeoutSeconds: Long = 10): Result = run(listOf("xcrun", "simctl", "keychain", udid, "reset"), timeoutSeconds) + /** + * Cheap clearState: terminate, then delete the *children* of the app's data container AND of + * each of its app-group containers, in place. Skips the uninstall + bundle copy + reinstall + * that [clearAppState] pays (measured ~9s for a 1.3GB bundle vs ~0.1s for this), at the cost + * of not resetting the bundle container itself. The group-container wipe is load-bearing: + * uninstall removes group containers, and an app whose group-container DB survives a + * data-container-only wipe + keychain reset comes back with undecryptable residue (observed + * on a large production app: "Failed to decrypt encrypted SyncEntity" + a sign-in flow that + * skips its verification gate). Also resets the app's TCC privacy grants (`simctl privacy + * reset all`), which uninstall would have dropped implicitly. Keychain is handled separately + * by callers (clearKeychain), matching [clearAppState]'s contract. + */ + fun clearAppDataContainer(udid: String, bundleId: String): Result { + terminate(udid, bundleId) // nonzero when the app isn't running — a fine starting state + // Same reason [clearAppState] polls: `terminate` returns as soon as the signal is sent, and a + // still-dying process can flush state back to disk after the wipe. That matters MORE here — + // there is no reinstall behind this wipe to overwrite whatever got flushed. + ensureStopped(udid, bundleId) + val container = run(listOf("xcrun", "simctl", "get_app_container", udid, bundleId, "data"), 10) + if (!container.success) return container + val dataPath = Paths.get(container.stdout.trim()) + if (!Files.isDirectory(dataPath)) { + return Result(-1, "", "data container not found at $dataPath") + } + // `groups` lists one `\t` per app-group container and exits 0 with EMPTY + // stdout when the app declares none (verified against simctl on a live device), so a + // nonzero exit is always a real failure (unknown bundle, timeout) — propagate it rather + // than silently skip a load-bearing wipe. + val groups = run(listOf("xcrun", "simctl", "get_app_container", udid, bundleId, "groups"), 10) + if (!groups.success) return groups + val wipeRoots = listOf(dataPath) + + groups.stdout.lineSequence() + .mapNotNull { line -> line.substringAfter('\t', "").trim().takeIf { it.isNotEmpty() } } + .map { Paths.get(it) } + .filter { Files.isDirectory(it) } + val undeleted = mutableListOf() + wipeRoots.forEach { root -> deleteChildrenWithoutFollowingLinks(root, undeleted) } + if (undeleted.isNotEmpty()) { + // Same hard-fail contract as clearAppState: a partial wipe that reports success would + // launch the next trail against dirty state, which is the exact bug clearState closes. + return Result(-1, "", "failed to delete: ${undeleted.joinToString(", ")}") + } + // Uninstall implicitly drops the app's TCC privacy grants (location, camera, ...), so flows + // that expect a fresh permission prompt would silently keep a prior trail's grant under an + // in-place wipe. Reset them explicitly to keep [clearAppState]'s fresh-install semantics. + return run(listOf("xcrun", "simctl", "privacy", udid, "reset", "all", bundleId), 10) + } + + /** + * Deletes everything under [root] (keeping [root] itself) without following symlinks. + * Container contents are app-controlled, and `File.deleteRecursively()` follows symlinks — + * a link inside the container pointing outside it would get its *target* tree deleted (same + * hazard `CliReportGenerator`'s cleanup documents). [Files.walkFileTree]'s default options + * visit a symlink as a plain entry without descending into its target, so the link itself is + * removed and the target is untouched. Failures are collected into [undeleted] rather than + * thrown so the caller's error names every path that survived the wipe. + */ + private fun deleteChildrenWithoutFollowingLinks(root: Path, undeleted: MutableList) { + Files.newDirectoryStream(root).use { children -> + children.forEach { child -> + Files.walkFileTree( + child, + object : SimpleFileVisitor() { + override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult { + if (!tryDelete(file)) undeleted += file.toString() + return FileVisitResult.CONTINUE + } + + override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult { + undeleted += file.toString() + return FileVisitResult.CONTINUE + } + + override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult { + if (exc != null || !tryDelete(dir)) undeleted += dir.toString() + return FileVisitResult.CONTINUE + } + }, + ) + } + } + } + + private fun tryDelete(path: Path): Boolean = try { + Files.deleteIfExists(path) + true + } catch (e: IOException) { + false + } + /** * Clears app state the way Maestro does on simulators (`LocalSimulatorUtils.clearAppState`): * terminate, then reinstall the app from its own installed bundle — reinstalling is the most diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/rules/TrailblazeHostDynamicLlmTokenProvider.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/rules/TrailblazeHostDynamicLlmTokenProvider.kt index c0785ab22..827b051d3 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/host/rules/TrailblazeHostDynamicLlmTokenProvider.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/host/rules/TrailblazeHostDynamicLlmTokenProvider.kt @@ -7,6 +7,7 @@ import ai.koog.prompt.executor.clients.anthropic.AnthropicLLMClient import ai.koog.prompt.executor.clients.google.GoogleLLMClient import ai.koog.prompt.executor.clients.openai.OpenAILLMClient import ai.koog.prompt.executor.clients.openrouter.OpenRouterLLMClient +import ai.koog.prompt.executor.ollama.client.ContextWindowStrategy import ai.koog.prompt.executor.ollama.client.OllamaClient import io.ktor.client.HttpClient import xyz.block.trailblaze.host.llm.OpenAICompatibleLlmClientFactory @@ -16,6 +17,7 @@ import xyz.block.trailblaze.llm.config.BuiltInLlmModelRegistry import xyz.block.trailblaze.llm.config.LlmConfigLoader import xyz.block.trailblaze.llm.config.LlmConfigResolver import xyz.block.trailblaze.llm.config.LlmProviderType +import xyz.block.trailblaze.llm.config.OllamaContextWindow import xyz.block.trailblaze.llm.providers.TrailblazeDynamicLlmTokenProvider import xyz.block.trailblaze.mcp.utils.JvmLLMProvidersUtil @@ -94,6 +96,16 @@ object TrailblazeHostDynamicLlmTokenProvider : TrailblazeDynamicLlmTokenProvider OllamaClient( baseUrl = ollamaBaseUrl ?: "http://localhost:11434", httpClientFactory = httpClientFactory, + // Request num_ctx on every call — without it Ollama sizes the window to the + // memory it has available, which on a modest machine is below a single agent + // turn (~20K tokens) and fails with exceed_context_size_error no matter what + // the registry entry declares. Fixed (not per-prompt) because Ollama reloads + // the model whenever the requested context changes; koog clamps it to the + // model's declared max. + // See OllamaContextWindow for the rationale and TRAILBLAZE_OLLAMA_NUM_CTX. + contextWindowStrategy = ContextWindowStrategy.Companion.Fixed( + OllamaContextWindow.resolveNumCtx(System.getenv(OllamaContextWindow.ENV_VAR)), + ), ) } else { null diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingScreenComposable.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingScreenComposable.kt index 360c56bdb..82e074251 100644 --- a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingScreenComposable.kt +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingScreenComposable.kt @@ -884,10 +884,7 @@ fun RecordingScreenComposable( is TapOnByElementSelector -> { recorder.replaceInteractionTool( interaction = interaction, - newTool = TapOnByElementSelector( - nodeSelector = candidate.selector, - longPress = original.longPress, - ), + newTool = original.retargetedAt(candidate.selector), newToolName = TapOnByElementSelector::class.toolName().toolName, ) generatedTrailYaml = null // Invalidate stale LLM output diff --git a/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewrites.kt b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewrites.kt new file mode 100644 index 000000000..c0502865d --- /dev/null +++ b/trailblaze-host/src/main/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewrites.kt @@ -0,0 +1,19 @@ +package xyz.block.trailblaze.ui.tabs.recording + +import xyz.block.trailblaze.api.TrailblazeNodeSelector +import xyz.block.trailblaze.toolcalls.commands.TapOnByElementSelector + +/** + * Re-targets a recorded selector tap at [selector], keeping everything else the author already + * decided about this step. + * + * Written as a [copy] rather than a fresh constructor call so a field added to + * [TapOnByElementSelector] later cannot be silently dropped here: the picker only means to change + * *which element* is tapped, and a rebuild that enumerates fields quietly reverts every one it + * forgets. `tapRoute` is the field that made this concrete — losing it un-pins a step that was + * measured to need its route, and the tap starts being absorbed again with nothing in the diff to + * explain it. + */ +internal fun TapOnByElementSelector.retargetedAt( + selector: TrailblazeNodeSelector, +): TapOnByElementSelector = copy(nodeSelector = selector) diff --git a/trailblaze-host/src/test/java/xyz/block/trailblaze/desktop/ResolveSavedModelWithinProviderTest.kt b/trailblaze-host/src/test/java/xyz/block/trailblaze/desktop/ResolveSavedModelWithinProviderTest.kt new file mode 100644 index 000000000..5c17820bb --- /dev/null +++ b/trailblaze-host/src/test/java/xyz/block/trailblaze/desktop/ResolveSavedModelWithinProviderTest.kt @@ -0,0 +1,96 @@ +package xyz.block.trailblaze.desktop + +import org.junit.Test +import xyz.block.trailblaze.llm.TrailblazeLlmModel +import xyz.block.trailblaze.llm.TrailblazeLlmProvider +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * Pins how a persisted LLM selection survives a catalog change. + * + * Retiring a model id is routine — the shipped catalog tracks what providers actually serve. + * What must not happen is a user who picked a now-retired id silently losing their LLM on + * upgrade, which is what happens when this resolution falls through to the caller's global + * default (`NONE` in the OSS desktop distribution). + */ +class ResolveSavedModelWithinProviderTest { + + private fun model(id: String) = TrailblazeLlmModel( + trailblazeLlmProvider = TrailblazeLlmProvider.OPENAI, + modelId = id, + inputCostPerOneMillionTokens = 1.0, + outputCostPerOneMillionTokens = 1.0, + contextLength = 128_000, + maxOutputTokens = 8_192, + capabilityIds = emptyList(), + ) + + private val entries = listOf(model("gpt-5.6-sol"), model("gpt-5.6-terra"), model("gpt-5.6-luna")) + + @Test + fun `an id the provider still offers resolves to itself`() { + assertEquals( + "gpt-5.6-luna", + resolveSavedModelWithinProvider( + entries = entries, + savedModelId = "gpt-5.6-luna", + providerDefaultModelId = "gpt-5.6-terra", + )?.modelId, + "An exact match must win over the provider default — the user's pick is honored " + + "whenever it is still available.", + ) + } + + @Test + fun `a retired id falls back to the provider default, not to no-LLM`() { + assertEquals( + "gpt-5.6-terra", + resolveSavedModelWithinProvider( + entries = entries, + savedModelId = "gpt-4.1", + providerDefaultModelId = "gpt-5.6-terra", + )?.modelId, + "A saved id the catalog dropped must move the user to the current default for the " + + "provider they chose. Returning null here disables their LLM on upgrade.", + ) + } + + @Test + fun `no provider default leaves the caller's fallback in charge`() { + assertNull( + resolveSavedModelWithinProvider( + entries = entries, + savedModelId = "gpt-4.1", + providerDefaultModelId = null, + ), + "With nothing to fall back to within the provider, this must not invent a choice — " + + "the caller's own default decides.", + ) + } + + @Test + fun `a provider default the entries do not contain is not fabricated`() { + assertNull( + resolveSavedModelWithinProvider( + entries = entries, + savedModelId = "gpt-4.1", + providerDefaultModelId = "gpt-9-not-shipped", + ), + "A default_model naming an id absent from the resolved entries (e.g. a workspace " + + "config narrowed the provider) must not resolve to a model that isn't there.", + ) + } + + @Test + fun `an empty provider resolves to nothing`() { + assertNull( + resolveSavedModelWithinProvider( + entries = emptyList(), + savedModelId = "gpt-5.6-terra", + providerDefaultModelId = "gpt-5.6-terra", + ), + "A provider offering no models has nothing to select.", + ) + } +} diff --git a/trailblaze-host/src/test/java/xyz/block/trailblaze/host/axe/AxeClearStateModeTest.kt b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/axe/AxeClearStateModeTest.kt new file mode 100644 index 000000000..bd2ac3038 --- /dev/null +++ b/trailblaze-host/src/test/java/xyz/block/trailblaze/host/axe/AxeClearStateModeTest.kt @@ -0,0 +1,29 @@ +package xyz.block.trailblaze.host.axe + +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * Pure-gate tests for the `TRAILBLAZE_IOS_CLEAR_STATE_MODE` routing + * ([AxeDeviceManager.useContainerClearState]) — the env read stays outside the gate so this + * needs no simulator and no environment mutation. + */ +class AxeClearStateModeTest { + + @Test + fun `container opts in, case-insensitively and trimmed`() { + assertTrue(AxeDeviceManager.useContainerClearState("container")) + assertTrue(AxeDeviceManager.useContainerClearState("CONTAINER")) + assertTrue(AxeDeviceManager.useContainerClearState(" Container ")) + } + + @Test + fun `unset or any other value keeps the default reinstall path`() { + assertFalse(AxeDeviceManager.useContainerClearState(null)) + assertFalse(AxeDeviceManager.useContainerClearState("")) + assertFalse(AxeDeviceManager.useContainerClearState("1")) + assertFalse(AxeDeviceManager.useContainerClearState("true")) + assertFalse(AxeDeviceManager.useContainerClearState("reinstall")) + } +} diff --git a/trailblaze-host/src/test/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewritesTest.kt b/trailblaze-host/src/test/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewritesTest.kt new file mode 100644 index 000000000..4694b9dcb --- /dev/null +++ b/trailblaze-host/src/test/java/xyz/block/trailblaze/ui/tabs/recording/RecordingToolRewritesTest.kt @@ -0,0 +1,73 @@ +package xyz.block.trailblaze.ui.tabs.recording + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import xyz.block.trailblaze.api.DriverNodeMatch +import xyz.block.trailblaze.api.TrailblazeNodeSelector +import xyz.block.trailblaze.model.TapRouteOverride +import xyz.block.trailblaze.toolcalls.commands.TapOnByElementSelector + +/** + * Pins what the recording UI's selector picker is allowed to change when an author re-targets a + * recorded tap: the selector, and nothing else. + * + * The picker exists to answer "which element does this step tap", so every other decision already + * recorded on the step has to survive it. `tapRoute` is the one that bites — a step pinned to a + * dispatch route was pinned because it was measured to need it, and silently reverting that turns + * a passing tap back into an absorbed one with nothing in the diff pointing at the cause. + */ +class RecordingToolRewritesTest { + + private fun selector(text: String) = + TrailblazeNodeSelector(androidAccessibility = DriverNodeMatch.AndroidAccessibility(textRegex = text)) + + @Test + fun `re-targeting keeps the pinned route`() { + val pinned = TapOnByElementSelector( + reason = "Select the option row in the open dropdown sheet.", + tapRoute = TapRouteOverride.ACTION_CLICK, + nodeSelector = selector("All add-ons"), + ) + + val retargeted = pinned.retargetedAt(selector("Every add-on")) + + assertEquals(TapRouteOverride.ACTION_CLICK, retargeted.tapRoute) + assertEquals(selector("Every add-on"), retargeted.nodeSelector) + } + + /** + * Negative control: the pin is carried, not manufactured. An unpinned tap has to come out + * unpinned, otherwise the assertion above would pass just as happily against a rewrite that + * hardcoded a route. + */ + @Test + fun `re-targeting an unpinned tap leaves it unpinned`() { + val unpinned = TapOnByElementSelector( + reason = "Open the category dropdown.", + nodeSelector = selector("For your business"), + ) + + val retargeted = unpinned.retargetedAt(selector("Categories")) + + assertNull(retargeted.tapRoute) + assertEquals(selector("Categories"), retargeted.nodeSelector) + } + + @Test + fun `re-targeting keeps the author's other recorded decisions`() { + val original = TapOnByElementSelector( + reason = "Long-press the row to open its context menu.", + longPress = true, + tapRoute = TapRouteOverride.GESTURE, + nodeSelector = selector("Team Management"), + ) + + val retargeted = original.retargetedAt(selector("Payroll")) + + assertEquals(original.copy(nodeSelector = selector("Payroll")), retargeted) + assertEquals("Long-press the row to open its context menu.", retargeted.reason) + assertEquals(true, retargeted.longPress) + assertEquals(TapRouteOverride.GESTURE, retargeted.tapRoute) + } +} diff --git a/trailblaze-models/api/android/trailblaze-models.api b/trailblaze-models/api/android/trailblaze-models.api index 3b6235c73..38b2ed4e1 100644 --- a/trailblaze-models/api/android/trailblaze-models.api +++ b/trailblaze-models/api/android/trailblaze-models.api @@ -8188,6 +8188,13 @@ public final class xyz/block/trailblaze/llm/config/LlmScreenshotConfig$Companion public final fun serializer ()Lkotlinx/serialization/KSerializer; } +public final class xyz/block/trailblaze/llm/config/OllamaContextWindow { + public static final field DEFAULT_NUM_CTX J + public static final field ENV_VAR Ljava/lang/String; + public static final field INSTANCE Lxyz/block/trailblaze/llm/config/OllamaContextWindow; + public final fun resolveNumCtx (Ljava/lang/String;)J +} + public final class xyz/block/trailblaze/llm/config/PlatformConfigResourceSource_androidKt { public static final fun bundledConfigResourceSource ()Lxyz/block/trailblaze/llm/config/ConfigResourceSource; public static final fun platformConfigResourceSource ()Lxyz/block/trailblaze/llm/config/ConfigResourceSource; @@ -10919,6 +10926,19 @@ public final class xyz/block/trailblaze/model/SetOfMarkConfig { public static final field INSTANCE Lxyz/block/trailblaze/model/SetOfMarkConfig; } +public final class xyz/block/trailblaze/model/TapRouteOverride : java/lang/Enum { + public static final field ACTION_CLICK Lxyz/block/trailblaze/model/TapRouteOverride; + public static final field Companion Lxyz/block/trailblaze/model/TapRouteOverride$Companion; + public static final field GESTURE Lxyz/block/trailblaze/model/TapRouteOverride; + public static fun getEntries ()Lkotlin/enums/EnumEntries; + public static fun valueOf (Ljava/lang/String;)Lxyz/block/trailblaze/model/TapRouteOverride; + public static fun values ()[Lxyz/block/trailblaze/model/TapRouteOverride; +} + +public final class xyz/block/trailblaze/model/TapRouteOverride$Companion { + public final fun serializer ()Lkotlinx/serialization/KSerializer; +} + public abstract class xyz/block/trailblaze/model/TrailExecutionResult { } diff --git a/trailblaze-models/api/jvm/trailblaze-models.api b/trailblaze-models/api/jvm/trailblaze-models.api index 542ebfcb9..54848b60f 100644 --- a/trailblaze-models/api/jvm/trailblaze-models.api +++ b/trailblaze-models/api/jvm/trailblaze-models.api @@ -8193,6 +8193,13 @@ public final class xyz/block/trailblaze/llm/config/LlmScreenshotConfig$Companion public final fun serializer ()Lkotlinx/serialization/KSerializer; } +public final class xyz/block/trailblaze/llm/config/OllamaContextWindow { + public static final field DEFAULT_NUM_CTX J + public static final field ENV_VAR Ljava/lang/String; + public static final field INSTANCE Lxyz/block/trailblaze/llm/config/OllamaContextWindow; + public final fun resolveNumCtx (Ljava/lang/String;)J +} + public final class xyz/block/trailblaze/llm/config/PlatformConfigResourceSource_jvmKt { public static final fun bundledConfigResourceSource ()Lxyz/block/trailblaze/llm/config/ConfigResourceSource; public static final fun platformConfigResourceSource ()Lxyz/block/trailblaze/llm/config/ConfigResourceSource; @@ -10935,6 +10942,19 @@ public final class xyz/block/trailblaze/model/SetOfMarkConfig { public static final field INSTANCE Lxyz/block/trailblaze/model/SetOfMarkConfig; } +public final class xyz/block/trailblaze/model/TapRouteOverride : java/lang/Enum { + public static final field ACTION_CLICK Lxyz/block/trailblaze/model/TapRouteOverride; + public static final field Companion Lxyz/block/trailblaze/model/TapRouteOverride$Companion; + public static final field GESTURE Lxyz/block/trailblaze/model/TapRouteOverride; + public static fun getEntries ()Lkotlin/enums/EnumEntries; + public static fun valueOf (Ljava/lang/String;)Lxyz/block/trailblaze/model/TapRouteOverride; + public static fun values ()[Lxyz/block/trailblaze/model/TapRouteOverride; +} + +public final class xyz/block/trailblaze/model/TapRouteOverride$Companion { + public final fun serializer ()Lkotlinx/serialization/KSerializer; +} + public abstract class xyz/block/trailblaze/model/TrailExecutionResult { } diff --git a/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindow.kt b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindow.kt new file mode 100644 index 000000000..20163f8b8 --- /dev/null +++ b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindow.kt @@ -0,0 +1,55 @@ +package xyz.block.trailblaze.llm.config + +/** + * The `num_ctx` (context window) value Trailblaze requests from an Ollama server. + * + * A request that omits `num_ctx` does not get the model's own maximum context. Ollama + * sizes the window itself, against the memory it has available rather than against what + * the model declares (and `OLLAMA_CONTEXT_LENGTH`, when set on the server, replaces that + * automatic sizing). The effective context is therefore a property of the host machine: on + * a memory-constrained one it lands well below a single Trailblaze agent turn — ~20K + * tokens for a set-of-mark screenshot + view hierarchy + tool definitions on a + * content-heavy screen — and the turn fails with `exceed_context_size_error` no matter what + * `context_length` the model's registry entry declares. Requesting `num_ctx` per call is + * what makes the context predictable instead of hardware-dependent. + * + * The value is deliberately a stable constant rather than fitted per prompt: Ollama + * reloads the model whenever the requested context length changes, and a reload costs + * far more than the memory headroom saved. 64K is the middle of the range — several + * multi-turn agent loops fit, and the KV cache still fits a laptop — and it is clamped + * down to the model's own declared maximum by the client-side strategy when the model + * supports less. On a machine with a lot of memory Ollama's automatic choice can exceed + * 64K, and an explicit request replaces it; [ENV_VAR] is how such a machine raises it. + * + * **Every `OllamaClient` construction must pass this** as + * `contextWindowStrategy = ContextWindowStrategy.Fixed(...)`; a site that omits it silently + * runs at whatever window Ollama picks for the machine and fails on real screens there. + * Host-side sites resolve the value through [resolveNumCtx] with [ENV_VAR]; on-device sites + * use [DEFAULT_NUM_CTX] directly, since the instrumentation process has no host environment + * to read. + * + * That split means [ENV_VAR] does not reach on-device clients, and both ends can address the + * same Ollama server (the device's base URL is host-forwarded). Setting [ENV_VAR] to + * anything but [DEFAULT_NUM_CTX] while on-device AI legs run against that server therefore + * makes the two ends request different lengths, and Ollama reloads the model on every + * alternation. Closing this means forwarding the resolved value to the device as an + * instrumentation arg alongside the base URL, in `LlmAuthResolver.toInstrumentationArgs`. + */ +object OllamaContextWindow { + + /** Environment variable that overrides the requested `num_ctx` on the host. */ + const val ENV_VAR: String = "TRAILBLAZE_OLLAMA_NUM_CTX" + + /** Default `num_ctx` requested from Ollama when [ENV_VAR] is unset. */ + const val DEFAULT_NUM_CTX: Long = 65536 + + /** + * Resolves the `num_ctx` to request: [rawOverride] (the value of [ENV_VAR]) when it is + * a positive integer, otherwise [DEFAULT_NUM_CTX]. Malformed or non-positive values + * fall back to the default, matching the repo's other environment knobs. + */ + fun resolveNumCtx(rawOverride: String?): Long { + val parsed = rawOverride?.trim()?.toLongOrNull() + return if (parsed != null && parsed > 0) parsed else DEFAULT_NUM_CTX + } +} diff --git a/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TapRouteOverride.kt b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TapRouteOverride.kt new file mode 100644 index 000000000..38d76e0b4 --- /dev/null +++ b/trailblaze-models/src/commonMain/kotlin/xyz/block/trailblaze/model/TapRouteOverride.kt @@ -0,0 +1,42 @@ +package xyz.block.trailblaze.model + +import kotlinx.serialization.Serializable + +/** + * Pins the dispatch route of a single recorded selector-resolved tap, overriding the route the + * Android accessibility driver would otherwise choose for it. + * + * That choice is made from the resolved node's static fields alone, before the tap. It is a + * heuristic for one question — is this an interactive leaf, or a container whose real click + * handler lives elsewhere — and two rows that answer it differently can be field-identical on a + * given API level. When they are, no global predicate separates them and the recording is the only + * place that knows which route actually actuates the row. + * + * Leave unset unless a specific step has been measured to need it. Only the leaf-vs-container + * judgement is overridable: the conditions that make `ACTION_CLICK` dispatchable at all (the node + * advertises the action, is enabled, visible, and not editable; the tap is not a long-press) still + * apply, so a pin that contradicts one of them routes to gesture rather than dispatching an action + * the node cannot answer. + */ +@Serializable +enum class TapRouteOverride { + /** + * Dispatch via `AccessibilityNodeInfo.ACTION_CLICK`. + * + * For a textless clickable container whose handler IS reachable via `View.performClick()` — a + * Compose `selectable` / `toggleable` row installs role, state and click handler on one + * semantics node — but which publishes no state the route decision can see, so it reads as an + * inert wrapper and goes to gesture, where the tap is absorbed with no effect on the screen. + */ + ACTION_CLICK, + + /** + * Dispatch via coordinate gesture. + * + * For the mirror-image shape: a container that does publish state, and so would be granted + * `ACTION_CLICK`, but whose `ACTION_CLICK` performs a *different* action than a real touch — an + * accordion row that selects its option semantically while only a touch expands it to reveal the + * sub-options the recording goes on to tap. + */ + GESTURE, +} diff --git a/trailblaze-models/src/commonMain/resources/trails/config/providers/google.yaml b/trailblaze-models/src/commonMain/resources/trails/config/providers/google.yaml index 31b9a0b5f..fc4120a9b 100644 --- a/trailblaze-models/src/commonMain/resources/trails/config/providers/google.yaml +++ b/trailblaze-models/src/commonMain/resources/trails/config/providers/google.yaml @@ -50,17 +50,13 @@ models: cached_input_per_million: 0.20 # Gemma 4 — open weights, served on the Gemini API at $0 on a rate-limited free tier. - # Multimodal (text + image) with native function calling; 256K context on all three. + # Multimodal (text + image) with native function calling; 256K context on both. # max_output_tokens is 32768 rather than the 65536 the Gemini models above use: Google # publishes no output cap for Gemma 4, and 32768 is the largest figure any host documents. - - id: gemma-4-12b-it - context_length: 262144 - max_output_tokens: 32768 - cost: - input_per_million: 0.0 - output_per_million: 0.0 - cached_input_per_million: 0.0 - + # + # Only these two Gemma 4 variants are callable through the Gemini API: + # https://ai.google.dev/gemma/docs/core/gemma_on_gemini_api + # The other sizes (12B, E4B, E2B) are download-only — use them via the `ollama` provider. - id: gemma-4-26b-a4b-it context_length: 262144 max_output_tokens: 32768 diff --git a/trailblaze-models/src/jvmTest/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindowTest.kt b/trailblaze-models/src/jvmTest/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindowTest.kt new file mode 100644 index 000000000..51cf370dc --- /dev/null +++ b/trailblaze-models/src/jvmTest/kotlin/xyz/block/trailblaze/llm/config/OllamaContextWindowTest.kt @@ -0,0 +1,44 @@ +package xyz.block.trailblaze.llm.config + +import kotlin.test.Test +import kotlin.test.assertEquals + +class OllamaContextWindowTest { + + // Both values below are pinned as literals on purpose. They are external contracts — the + // env var is a name users type, and the default is what reaches the Ollama wire — and + // every other assertion here compares against the symbols, so a typo in either constant + // would otherwise satisfy this suite. `apiCheck` doesn't catch it either: the baseline + // records the field signatures, not the values. + @Test + fun `the requested default and env var name are the documented ones`() { + assertEquals(65536L, OllamaContextWindow.DEFAULT_NUM_CTX) + assertEquals("TRAILBLAZE_OLLAMA_NUM_CTX", OllamaContextWindow.ENV_VAR) + } + + @Test + fun `unset override resolves to the default`() { + assertEquals(OllamaContextWindow.DEFAULT_NUM_CTX, OllamaContextWindow.resolveNumCtx(null)) + } + + @Test + fun `a positive override is honored`() { + assertEquals(32768, OllamaContextWindow.resolveNumCtx("32768")) + } + + @Test + fun `surrounding whitespace is tolerated`() { + assertEquals(32768, OllamaContextWindow.resolveNumCtx(" 32768 ")) + } + + @Test + fun `malformed and non-positive overrides fall back to the default`() { + for (raw in listOf("", " ", "abc", "64K", "0", "-1")) { + assertEquals( + OllamaContextWindow.DEFAULT_NUM_CTX, + OllamaContextWindow.resolveNumCtx(raw), + "raw override '$raw' must fall back to the default", + ) + } + } +} diff --git a/trailblaze-report/src/main/java/xyz/block/trailblaze/report/utils/LogsRepo.kt b/trailblaze-report/src/main/java/xyz/block/trailblaze/report/utils/LogsRepo.kt index 0fa37832f..0a2c1622f 100644 --- a/trailblaze-report/src/main/java/xyz/block/trailblaze/report/utils/LogsRepo.kt +++ b/trailblaze-report/src/main/java/xyz/block/trailblaze/report/utils/LogsRepo.kt @@ -629,15 +629,24 @@ class LogsRepo( */ fun getSessionInfoSummary(sessionId: SessionId): SessionInfo? { val files = readLogFilesFromDisk(sessionId) - val statusLogs = files - .filter { it.name.contains("TrailblazeSessionStatusChangeLog") } - .mapNotNull { parseTrailblazeLogFromFile(it) } - .filterIsInstance() + val statusLogs = readSessionStatusLogs(files) if (statusLogs.isEmpty()) return null val lastActivityMs = files.maxOfOrNull { it.lastModified() } ?: 0L return buildSessionInfo(statusLogs, lastActivityMsOverride = lastActivityMs) } + /** + * The session-status logs among [logFiles], selected by filename so the large driver logs are + * never deserialized. Every writer that puts log files in a session directory embeds the log + * class's simple name: [saveLogToDisk] here, and the CI on-device log reshaper. + */ + private fun readSessionStatusLogs( + logFiles: List, + ): List = logFiles + .filter { it.name.contains("TrailblazeSessionStatusChangeLog") } + .mapNotNull { parseTrailblazeLogFromFile(it) } + .filterIsInstance() + fun getSessionInfo(sessionId: SessionId): SessionInfo? { // Use cached logs from the flow if available, otherwise read from disk val allLogs = getCachedLogsForSession(sessionId) @@ -776,9 +785,10 @@ class LogsRepo( // appends are dropped. Without this, cancelling a run lets the killed runner's async failure // land an Ended.Failed on top of the user's Ended.Cancelled, and the run reads as Failed. // Read disk (not the cached flow) — the cache can lag the just-written status. + // Status logs only: deserializing the whole session here exhausted a 512 MB trail-driver heap + // at session end, when the heap is already at its fullest. if (logEvent is TrailblazeLog.TrailblazeSessionStatusChangeLog && logEvent.sessionStatus is SessionStatus.Ended) { - val alreadyEnded = getLogsForSession(logEvent.session) - .filterIsInstance() + val alreadyEnded = readSessionStatusLogs(readLogFilesFromDisk(logEvent.session)) .any { it.sessionStatus is SessionStatus.Ended } if (alreadyEnded) { Console.log( @@ -807,6 +817,14 @@ class LogsRepo( logEvent, ), ) + // Streamed to stdout because only `api`-source CI builds upload session logs at all, and stdout + // also survives a hard kill: on every other build this is the sole record of capture size vs heap. + val runtime = Runtime.getRuntime() + Console.log( + "[log-size] ${logEvent::class.java.simpleName} ${jsonLogFilename.length() / 1024}KB " + + "heap ${(runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024)}/" + + "${runtime.maxMemory() / (1024 * 1024)}MB ${jsonLogFilename.name}", + ) // The flow will be updated automatically via the file watcher return jsonLogFilename } diff --git a/trailblaze-report/src/main/resources/xyz/block/trailblaze/trailrunner/web/app/run-report-core.test.ts b/trailblaze-report/src/main/resources/xyz/block/trailblaze/trailrunner/web/app/run-report-core.test.ts index 81072dd6a..fa1b09846 100644 --- a/trailblaze-report/src/main/resources/xyz/block/trailblaze/trailrunner/web/app/run-report-core.test.ts +++ b/trailblaze-report/src/main/resources/xyz/block/trailblaze/trailrunner/web/app/run-report-core.test.ts @@ -1382,14 +1382,21 @@ describe("RUN_REPORT_VIEWER (rendered output)", () => { // Only the retried iOS cell gets a chevron; the single-attempt android cell does not. expect(collapsed.match(/data-cell-toggle/g)).toHaveLength(1); expect(collapsed).toContain('data-cell-toggle="trail:checkout:demo:ios"'); - // The retried cell links to the latest attempt and shows the attempt-history dots. + // The retried cell links to the latest attempt; the chevron rail (the control that expands + // the history) previews it as a bare attempt count. expect(collapsed).toContain('
` : ''; + // The main button always reads latest-outcome dot + duration; the chevron rail — the control + // that expands the attempt history — previews it as a bare attempt count, so the stats line + // never shares width with variable-length history (long durations were wrapping mid-value). + // Per-attempt outcomes live only in the expanded panel. + const value = ``; + const chev = retried ? `` : ''; // The open-latest and expand controls are sibling ${chev}
`; + return `
${chev}
`; }; const renderMatrixRow = (row) => { const title = row.latest.s.meta.title || ('Run ' + (row.latest.i + 1)); diff --git a/trailblaze-report/src/test/java/xyz/block/trailblaze/report/utils/LogsRepoDiskTruthTest.kt b/trailblaze-report/src/test/java/xyz/block/trailblaze/report/utils/LogsRepoDiskTruthTest.kt index 0a20860b0..e2af950f0 100644 --- a/trailblaze-report/src/test/java/xyz/block/trailblaze/report/utils/LogsRepoDiskTruthTest.kt +++ b/trailblaze-report/src/test/java/xyz/block/trailblaze/report/utils/LogsRepoDiskTruthTest.kt @@ -8,6 +8,7 @@ import xyz.block.trailblaze.logs.model.SessionStatus import java.io.File import java.nio.file.Files import kotlin.test.Test +import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertTrue @@ -16,23 +17,36 @@ import kotlin.test.assertTrue * log lands on disk but the in-memory cache flow was snapshotted before it, the cached read * stays stale while the direct (disk) read surfaces `Ended`. The handler's post-completion * poll loop must use the disk read or it spins the full timeout on an already-finished run. + * + * Also pins the terminal-status ("first Ended wins") guard that read serves, including that it + * finds an existing Ended without deserializing the session's large driver logs. */ class LogsRepoDiskTruthTest { private fun tempLogsDir(): File = Files.createTempDirectory("logs-repo-disk-truth").toFile() - private fun writeEndedLogToDisk(logsDir: File, sessionId: SessionId) { + private fun writeEndedLogToDisk( + logsDir: File, + sessionId: SessionId, + // First hex char + .json so readLogFilesFromDisk picks it up. + fileName: String = "0_TrailblazeSessionStatusChangeLog.json", + ) { val sessionDir = File(logsDir, sessionId.value).apply { mkdirs() } val log: TrailblazeLog = TrailblazeLog.TrailblazeSessionStatusChangeLog( sessionStatus = SessionStatus.Ended.Succeeded(durationMs = 1234L), session = sessionId, timestamp = Clock.System.now(), ) - // First hex char + .json so readLogFilesFromDisk picks it up. - File(sessionDir, "0_TrailblazeSessionStatusChangeLog.json") + File(sessionDir, fileName) .writeText(TrailblazeJsonInstance.encodeToString(log)) } + private fun lateEnded(sessionId: SessionId) = TrailblazeLog.TrailblazeSessionStatusChangeLog( + sessionStatus = SessionStatus.Ended.Failed(durationMs = 99L, exceptionMessage = "late failure"), + session = sessionId, + timestamp = Clock.System.now(), + ) + @Test fun `direct read surfaces on-disk Ended while the stale cache does not`() { val logsDir = tempLogsDir() @@ -100,4 +114,60 @@ class LogsRepoDiskTruthTest { "first Ended must win; got ${statuses.map { it.sessionStatus }}", ) } + + @Test + fun `an Ended under the restarted-session filename shape is still found`() { + val logsDir = tempLogsDir() + val logsRepo = LogsRepo(logsDir, watchFileSystem = false) + val sessionId = SessionId("session-ended-timestamped-filename") + + // saveLogToDisk inserts the timestamp when a session resumes mid-run. + writeEndedLogToDisk(logsDir, sessionId, "007_1755000000000_TrailblazeSessionStatusChangeLog.json") + + val written = logsRepo.saveLogToDisk(lateEnded(sessionId)) + + assertEquals( + "noop", + written.name, + "the timestamped-shape Ended must be found, so the later Ended is dropped", + ) + } + + @Test + fun `a session holding only non-status logs records its first Ended`() { + val logsDir = tempLogsDir() + val logsRepo = LogsRepo(logsDir, watchFileSystem = false) + val sessionId = SessionId("session-still-running") + + // A live session's bulk: driver logs carrying whole view hierarchies, no terminal status yet. + val sessionDir = File(logsDir, sessionId.value).apply { mkdirs() } + File(sessionDir, "002_AgentDriverLog.json") + .writeText("""{"viewHierarchy":"${"x".repeat(200_000)}"}""") + + val written = logsRepo.saveLogToDisk(lateEnded(sessionId)) + + assertTrue( + written.name.endsWith("_TrailblazeSessionStatusChangeLog.json"), + "no Ended is on disk, so this one must be recorded; got ${written.name}", + ) + } + + @Test + fun `the guard reads status logs by filename and never opens driver logs`() { + val logsDir = tempLogsDir() + val logsRepo = LogsRepo(logsDir, watchFileSystem = false) + val sessionId = SessionId("session-with-decoy-driver-log") + + // A valid Ended status log planted under a driver-log filename. Reading it would suppress the + // incoming Ended, so a write here is the proof that the driver-log file was never opened — + // which is the point of the fix: the guard must not deserialize the session's large logs. + writeEndedLogToDisk(logsDir, sessionId, "002_AgentDriverLog.json") + + val written = logsRepo.saveLogToDisk(lateEnded(sessionId)) + + assertTrue( + written.name.endsWith("_TrailblazeSessionStatusChangeLog.json"), + "driver-log files must not be read by the guard; got ${written.name}", + ) + } } diff --git a/trailblaze-server/src/main/java/xyz/block/trailblaze/logs/server/TrailblazeMcpServer.kt b/trailblaze-server/src/main/java/xyz/block/trailblaze/logs/server/TrailblazeMcpServer.kt index af5b84212..98078f260 100644 --- a/trailblaze-server/src/main/java/xyz/block/trailblaze/logs/server/TrailblazeMcpServer.kt +++ b/trailblaze-server/src/main/java/xyz/block/trailblaze/logs/server/TrailblazeMcpServer.kt @@ -2676,7 +2676,13 @@ class TrailblazeMcpServer( // The new surface is fully registered, so now drop only what it no longer contains. // Logged (like the host-tool diff) so a target switch that drops custom tools leaves a // trace — otherwise "my tool disappeared" is invisible. - val staleTrailblazeToolNames = previouslyRegisteredToolNames - newlyRegisteredToolNames + val liveHostToolNames = hostMcpToolRegistryBySession[mcpSessionId.sessionId] + ?.tools?.map { it.descriptor.name }?.toSet() ?: emptySet() + val staleTrailblazeToolNames = computeStaleToolNamesToRemove( + previouslyRegisteredToolNames = previouslyRegisteredToolNames, + newlyRegisteredToolNames = newlyRegisteredToolNames, + liveHostToolNames = liveHostToolNames, + ) if (staleTrailblazeToolNames.isNotEmpty()) { Console.log( "[TrailblazeMcpServer] Removing ${staleTrailblazeToolNames.size} target-scoped tool(s) " + @@ -3096,3 +3102,20 @@ class TrailblazeMcpServer( } } } + +/** + * The target-scoped tool names a re-registration should un-advertise: what the session + * advertised before, minus what the freshly-resolved surface still contains, minus every + * tool name the host registry is currently serving. + * + * Subtracting [liveHostToolNames] is the part that isn't obvious. Host tools are registered + * earlier in the same re-registration pass, and MCP tools live in one flat name-keyed map — + * so a stale target-scoped name that happens to equal a live host tool's name would remove + * the host tool that had just taken that name. The host surface runs its own diff in + * `addToolsAsMcpToolsFromRegistry`; this removal must not reach into it. + */ +internal fun computeStaleToolNamesToRemove( + previouslyRegisteredToolNames: Set, + newlyRegisteredToolNames: Set, + liveHostToolNames: Set, +): Set = previouslyRegisteredToolNames - newlyRegisteredToolNames - liveHostToolNames diff --git a/trailblaze-server/src/test/kotlin/xyz/block/trailblaze/logs/server/ComputeStaleToolNamesToRemoveTest.kt b/trailblaze-server/src/test/kotlin/xyz/block/trailblaze/logs/server/ComputeStaleToolNamesToRemoveTest.kt new file mode 100644 index 000000000..d69e5bfc8 --- /dev/null +++ b/trailblaze-server/src/test/kotlin/xyz/block/trailblaze/logs/server/ComputeStaleToolNamesToRemoveTest.kt @@ -0,0 +1,71 @@ +package xyz.block.trailblaze.logs.server + +import org.junit.Test +import kotlin.test.assertEquals + +/** + * Pins which tool names an MCP re-registration un-advertises. + * + * MCP tools live in one flat name-keyed map shared by the host surface and the + * target-scoped TrailblazeTool surface, so "remove what the new surface dropped" has to be + * computed against the host surface too — otherwise a name that moved from one surface to + * the other gets removed right after it was registered. + */ +class ComputeStaleToolNamesToRemoveTest { + + @Test + fun `drops names the new surface no longer contains`() { + assertEquals( + setOf("seedData"), + computeStaleToolNamesToRemove( + previouslyRegisteredToolNames = setOf("tap", "seedData"), + newlyRegisteredToolNames = setOf("tap"), + liveHostToolNames = emptySet(), + ), + "A target switch that drops a custom tool must stop advertising it — a client can " + + "otherwise still call a tool the session can't run.", + ) + } + + @Test + fun `keeps names the new surface still contains`() { + assertEquals( + emptySet(), + computeStaleToolNamesToRemove( + previouslyRegisteredToolNames = setOf("tap", "tapOnPoint"), + newlyRegisteredToolNames = setOf("tap", "tapOnPoint"), + liveHostToolNames = emptySet(), + ), + "Re-registering an unchanged surface must remove nothing.", + ) + } + + @Test + fun `never removes a name the host registry is currently serving`() { + assertEquals( + emptySet(), + computeStaleToolNamesToRemove( + previouslyRegisteredToolNames = setOf("endSession"), + newlyRegisteredToolNames = emptySet(), + liveHostToolNames = setOf("endSession"), + ), + "The host tools are registered earlier in the same pass. A stale target-scoped name " + + "that collides with a live host tool must not un-register the host tool that now " + + "owns that name.", + ) + } + + @Test + fun `a host collision does not shield unrelated stale names`() { + assertEquals( + setOf("seedData"), + computeStaleToolNamesToRemove( + previouslyRegisteredToolNames = setOf("endSession", "seedData"), + newlyRegisteredToolNames = emptySet(), + liveHostToolNames = setOf("endSession"), + ), + "Protecting host names must stay scoped to the colliding name — everything else the " + + "surface dropped still has to go.", + ) + } +}