From 04b7179529ff95f528ea754f45f8e2df907e3f46 Mon Sep 17 00:00:00 2001 From: Sam Edwards <264948+handstandsam@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:03:51 -0400 Subject: [PATCH 1/2] Upstream 2026.08.10 --- .trailblaze-sync | 2 +- docs/CLI.md | 2 +- ...026-08-06-all-universal-root-classifier.md | 107 +++ docs/devlog/index.md | 1 + docs/generated/LLM_MODELS.md | 2 +- docs/generated/external-config.md | 2 +- docs/generated/functions/custom/wait.md | 9 +- .../functions/custom/waitForChange.md | 6 +- .../tools/contacts_ios_openContact.ts | 10 +- .../tools/contacts_ios_searchContacts.test.ts | 187 +++++- .../tools/contacts_ios_searchContacts.ts | 119 +++- .../IosSampleApp/FormsScreen.swift | 6 + .../IosSampleApp/IosSampleAppApp.swift | 4 +- .../IosSampleApp/WebContentScreen.swift | 69 ++ examples/ios-sample-app/README.md | 14 +- .../src/generated/trailrunner-dtos.ts | 3 + .../src/matcher/matcher-parity-fixtures.json | 84 +++ .../src/matcher/matcher-parity.test.ts | 32 + sdks/typescript/src/matcher/resolver.test.ts | 17 + sdks/typescript/src/matcher/resolver.ts | 20 +- sdks/typescript/src/testing.test.ts | 41 +- sdks/typescript/src/testing.ts | 21 + .../trailblaze/agent/TrailblazeRunner.kt | 2 +- .../resources/trailblaze_system_prompt.md | 4 +- .../baselines/composed_default_prompt.txt | 4 +- .../baselines/default_platform_prompt.txt | 4 +- .../android/AndroidTrailblazeRule.kt | 4 +- .../AccessibilityDeviceManager.kt | 86 ++- .../accessibility/AccessibilityTrailRunner.kt | 2 +- .../AccessibilityTrailblazeAgent.kt | 73 +- .../accessibility/MaestroCommandConverter.kt | 46 +- .../TrailblazeAccessibilityService.kt | 65 +- ...ilityTrailblazeAgentBatchValidationTest.kt | 76 +++ ...MaestroCommandConverterAppLifecycleTest.kt | 12 +- .../MaestroCommandConverterAssertTest.kt | 26 +- .../MaestroCommandConverterBatchTest.kt | 52 ++ .../MaestroCommandConverterClipboardTest.kt | 16 +- ...estroCommandConverterDeviceSettingsTest.kt | 22 +- .../MaestroCommandConverterTapTest.kt | 10 +- .../accessibility/PlanActionClickRouteTest.kt | 115 ++++ .../capture/video/IosVideoCapture.kt | 26 +- .../capture/video/IosVideoCaptureTest.kt | 15 + .../capture/video/SubprocessRunnerTest.kt | 14 +- .../tools/assertMatchCount.tool.yaml | 2 + .../trailblaze/tools/sleep.tool.yaml | 2 + .../block/trailblaze/agent/DirectMcpAgent.kt | 2 +- .../block/trailblaze/agent/OuterLoopAgent.kt | 2 +- .../project/TrailblazeProjectConfigLoader.kt | 122 +++- .../mcp/agent/KoogStrategyGraphAgent.kt | 2 +- .../trailblaze/rules/TrailblazeLoggingRule.kt | 36 + .../trailblaze/toolcalls/TrailblazeToolSet.kt | 2 + .../toolcalls/TrailblazeToolSetCatalog.kt | 20 +- .../AssertMatchCountTrailblazeTool.kt | 155 +++++ ...ssertNotVisibleBySelectorTrailblazeTool.kt | 11 +- .../toolcalls/commands/SleepTrailblazeTool.kt | 82 +++ .../commands/TapOnPointTrailblazeTool.kt | 16 + .../commands/WaitForChangeTrailblazeTool.kt | 20 +- .../commands/WaitForIdleSyncTrailblazeTool.kt | 19 +- .../memory/AssertMathTrailblazeTool.kt | 13 +- .../trailblaze/util/CoreSimulatorTempFiles.kt | 46 ++ .../trailblaze/yaml/TrailYamlValidator.kt | 4 +- .../TrailblazeProjectConfigLoaderTest.kt | 191 ++++++ .../AgentDriverActionSerializationTest.kt | 33 + .../rules/TrailblazeLoggingRuleTest.kt | 110 +++- .../AssertMatchCountTrailblazeToolTest.kt | 438 ++++++++++++ .../commands/AssertToolNodeSelectorTest.kt | 20 + .../AssertVisibleTrailblazeToolTest.kt | 45 +- .../commands/SleepTrailblazeToolTest.kt | 143 ++++ .../commands/TapOnPointTrailblazeToolTest.kt | 18 + .../WaitForChangeTrailblazeToolTest.kt | 104 +++ .../WaitForIdleSyncTrailblazeToolTest.kt | 112 ++++ .../util/CoreSimulatorTempFilesTest.kt | 65 ++ .../trailblaze/yaml/ToolSerializationTest.kt | 118 ++++ .../yaml/TrailYamlValidationTest.kt | 37 ++ trailblaze-host/build.gradle.kts | 55 ++ .../xyz/block/trailblaze/cli/CheckCommand.kt | 79 ++- .../trailblaze/cli/CliReportGenerator.kt | 1 + .../trailblaze/cli/CliRunDriverResolver.kt | 42 ++ .../block/trailblaze/cli/CompileCommand.kt | 46 +- .../cli/DeviceClassifierResolver.kt | 15 +- .../xyz/block/trailblaze/cli/TrailCommand.kt | 46 +- .../trailblaze/cli/TrailblazeExitCode.kt | 21 + .../trailblaze/host/AppTargetDiscovery.kt | 75 ++- .../host/HostMaestroTrailblazeAgent.kt | 34 +- .../trailblaze/host/MaestroHostRunnerImpl.kt | 8 + .../host/TrailblazeHostYamlRunner.kt | 26 +- .../xyz/block/trailblaze/host/axe/AxeCli.kt | 91 ++- .../trailblaze/host/axe/AxeDeviceManager.kt | 313 +++++++-- .../trailblaze/host/axe/AxeJsonMapper.kt | 55 +- .../trailblaze/host/axe/AxeViewportClamp.kt | 69 ++ .../trailblaze/host/ios/IosDriverAction.kt | 14 +- .../host/ios/IosDriverTrailRunner.kt | 325 ++++++++- .../host/ios/IosDriverTrailblazeAgent.kt | 3 + .../host/ios/IosSimulatorPermissions.kt | 221 +++++++ ...aestroCommandToIosDriverActionConverter.kt | 23 +- .../block/trailblaze/host/ios/SimctlCli.kt | 118 +++- .../host/recording/AxeTreeOverlay.kt | 163 +++++ .../host/recording/DeviceConnectionService.kt | 3 +- .../recording/MaestroDeviceScreenStream.kt | 14 +- .../recording/MaestroScreenStateProvider.kt | 55 +- .../trailblaze/host/rules/BaseComposeTest.kt | 4 +- .../host/rules/BaseHostTrailblazeTest.kt | 4 +- .../host/rules/BasePlaywrightElectronTest.kt | 4 +- .../host/rules/BasePlaywrightNativeTest.kt | 4 +- .../host/screenstate/AxeScreenState.kt | 54 +- .../trailblaze/host/yaml/DesktopYamlRunner.kt | 46 +- .../trailblaze/mcp/TrailblazeMcpBridgeImpl.kt | 32 +- .../trailrunner/TrailIndexBuilder.kt | 12 +- .../block/trailblaze/ui/CliRunRejections.kt | 50 ++ .../trailblaze/ui/TrailblazeDesktopApp.kt | 31 +- .../trailblaze/ui/TrailblazeDeviceManager.kt | 12 +- .../tabs/devices/DevicesTabComposableJvm.kt | 3 +- .../trailrunner/web/app/data-core.tsx | 4 + .../trailrunner/web/app/runcommand.tsx | 2 +- .../trailrunner/web/app/runconfig.tsx | 4 +- .../web/app/screens/steps-board.tsx | 4 +- .../trailrunner/web/app/zip-report-core.js | 14 +- .../web/app/zip-report-core.test.ts | 12 + .../block/trailblaze/cli/CheckCommandTest.kt | 105 +++ .../cli/CliRunDriverResolverTest.kt | 53 ++ .../trailblaze/cli/CompileCommandTest.kt | 75 +++ .../cli/TrailblazeExitCodePolicyTest.kt | 31 + .../block/trailblaze/host/MockRpcServer.kt | 108 ++- .../trailblaze/host/MockRpcServerTest.kt | 198 ++++++ .../host/WorkspaceTypeScriptSetupTest.kt | 19 +- .../block/trailblaze/host/axe/AxeCliTest.kt | 43 ++ .../AxeDescribeUiSelectorResolutionTest.kt | 103 +++ .../host/axe/AxeDeviceManagerTest.kt | 96 +++ .../trailblaze/host/axe/AxeTreeDedupeTest.kt | 237 +++++++ .../host/axe/AxeViewportClampTest.kt | 227 +++++++ .../ios/IosDriverTrailRunnerLoggingTest.kt | 233 +++++++ .../host/ios/IosSimulatorPermissionsTest.kt | 294 +++++++++ ...roCommandToIosDriverActionConverterTest.kt | 44 +- .../trailblaze/host/ios/SimctlCliTest.kt | 153 +++++ .../host/recording/AxeTreeOverlayTest.kt | 179 +++++ .../MaestroScreenStateProviderTest.kt | 52 ++ .../host/screenstate/AxeScreenStateTest.kt | 173 +++++ .../yaml/DesktopYamlRunnerDriverPinTest.kt | 70 +- .../trailblaze/ui/CliRunRejectionsTest.kt | 64 ++ .../baselines/composed_mobile_prompt.txt | 4 +- .../help-trailblaze-run.txt | 2 +- .../api/android/trailblaze-models.api | 35 +- .../api/jvm/trailblaze-models.api | 35 +- .../block/trailblaze/api/AgentDriverAction.kt | 31 +- .../trailblaze/api/TrailblazeNodeCompat.kt | 13 + .../api/TrailblazeNodeSelectorResolver.kt | 20 +- .../devices/TrailblazeClassifierLineage.kt | 45 +- .../model/DesktopAppRunYamlParams.kt | 15 +- .../block/trailblaze/toolcalls/CoreTools.kt | 10 +- .../block/trailblaze/yaml/TrailblazeYaml.kt | 14 +- .../yaml/unified/RecordingResolution.kt | 74 ++- .../trailblaze/toolsets/core_interaction.yaml | 1 + .../llm/config/ClasspathResourceDiscovery.kt | 13 + .../api/MatcherParityFixturesTest.kt | 68 +- .../api/TrailblazeNodeCompatTest.kt | 33 + .../api/TrailblazeNodeSelectorResolverTest.kt | 23 + .../TrailblazeClassifierLineageTest.kt | 96 ++- .../devices/TrailblazeDriverTypeTest.kt | 15 +- .../yaml/unified/RecordingResolutionTest.kt | 163 ++++- .../yaml/unified/UnifiedTrailAdapterTest.kt | 52 ++ .../report/GenerateTestResultsCliCommand.kt | 1 + .../report/PerformanceAnalysisGenerator.kt | 2 + .../trailblaze/report/RunReportGenerator.kt | 11 +- .../xyz/block/trailblaze/report/WasmReport.kt | 29 +- .../trailblaze/report/models/ExecutionMode.kt | 34 +- .../trailblaze/report/models/SessionResult.kt | 9 + .../web/app/run-report-core.test.ts | 621 +++++++++++++++++- .../trailrunner/web/app/run-report-css.ts | 34 +- .../trailrunner/web/app/run-report-extract.ts | 80 ++- .../trailrunner/web/app/run-report-html.ts | 63 +- .../trailrunner/web/app/run-report-types.d.ts | 31 +- .../trailrunner/web/app/run-report-viewer.ts | 430 ++++++++---- .../report/RunReportGeneratorTest.kt | 67 +- .../report/WasmReportCompressLogsTest.kt | 128 ++++ .../McpSubprocessHandshakeTimeoutTest.kt | 24 +- .../McpSubprocessSessionConnectCleanupTest.kt | 56 +- .../logs/server/endpoints/CliRunEndpoint.kt | 19 +- .../mcp/sampling/SubagentOrchestrator.kt | 2 +- 178 files changed, 9735 insertions(+), 858 deletions(-) create mode 100644 docs/devlog/2026-08-06-all-universal-root-classifier.md create mode 100644 examples/ios-sample-app/IosSampleApp/WebContentScreen.swift create mode 100644 trailblaze-android/src/test/java/xyz/block/trailblaze/android/accessibility/AccessibilityTrailblazeAgentBatchValidationTest.kt create mode 100644 trailblaze-android/src/test/java/xyz/block/trailblaze/android/accessibility/MaestroCommandConverterBatchTest.kt create mode 100644 trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/assertMatchCount.tool.yaml create mode 100644 trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/sleep.tool.yaml create mode 100644 trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertMatchCountTrailblazeTool.kt create mode 100644 trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/SleepTrailblazeTool.kt create mode 100644 trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/util/CoreSimulatorTempFiles.kt create mode 100644 trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/AssertMatchCountTrailblazeToolTest.kt create mode 100644 trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/SleepTrailblazeToolTest.kt create mode 100644 trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/WaitForChangeTrailblazeToolTest.kt create mode 100644 trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/WaitForIdleSyncTrailblazeToolTest.kt create mode 100644 trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/util/CoreSimulatorTempFilesTest.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/cli/CliRunDriverResolver.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/axe/AxeViewportClamp.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/ios/IosSimulatorPermissions.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/host/recording/AxeTreeOverlay.kt create mode 100644 trailblaze-host/src/main/java/xyz/block/trailblaze/ui/CliRunRejections.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/cli/CliRunDriverResolverTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/MockRpcServerTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/axe/AxeTreeDedupeTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/axe/AxeViewportClampTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/ios/IosDriverTrailRunnerLoggingTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/ios/IosSimulatorPermissionsTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/ios/SimctlCliTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/recording/AxeTreeOverlayTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/recording/MaestroScreenStateProviderTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/host/screenstate/AxeScreenStateTest.kt create mode 100644 trailblaze-host/src/test/java/xyz/block/trailblaze/ui/CliRunRejectionsTest.kt create mode 100644 trailblaze-report/src/test/java/xyz/block/trailblaze/report/WasmReportCompressLogsTest.kt diff --git a/.trailblaze-sync b/.trailblaze-sync index ec971526f..75856ef8d 100644 --- a/.trailblaze-sync +++ b/.trailblaze-sync @@ -1 +1 @@ -6e278dc4acbede2988c75123f499d10b0cf7714f +8141286d5a206c6ee3b16169f1b7224ac2d881aa diff --git a/docs/CLI.md b/docs/CLI.md index a53d8411c..80829b0a5 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -311,7 +311,7 @@ trailblaze run [OPTIONS] [<>] | `--secret` | Pre-populate trail memory with a SENSITIVE KEY=VAL before any step runs. Same shape as --memory; the value is redacted in logs (via `rememberSensitive`), excluded from the scripting envelope, and omitted from the session-start snapshot. Only the KEY appears in `Started.sensitiveMemoryKeys` so replay knows it must re-supply the value. Repeatable. Use for passwords, tokens, API keys, PII. | - | | `--arg` | Supply a value for a parameter the trail DECLARES under `config.args:`. Repeatable (`--arg recipient=sam@example.com --arg retries=3`). Unlike --memory, args are declared and typed: the value is coerced to the arg's declared type, a missing required arg fails before the run starts, and an undeclared arg is rejected. Referenced as `{{args.name}}` in prompts and tool params. Overrides an --args-file entry with the same key. The value is always a string here; declaration-driven coercion turns `retries=3` into a number for an `integer` arg. Use --args-file for structured (array/object) values. Arg values are logged in cleartext, persisted into logs/recordings, and surfaced to the LLM — args are non-sensitive by design. Route passwords, tokens, or other sensitive data through --secret memory instead. | - | | `--args-file` | Read parameter values from a YAML or JSON file (a map of arg-name to value). Applied BEFORE --arg, so a --arg KEY=VAL overrides the file entry with the same key. A YAML-null value is rejected (args have no null) — use '' for an empty string. Arg values are logged in cleartext, persisted into logs/recordings, and surfaced to the LLM — args are non-sensitive by design. Route passwords, tokens, or other sensitive data through --secret memory instead. | - | -| `--max-llm-calls` | Cap the number of LLM calls per objective for the legacy TRAILBLAZE_RUNNER agent. Useful on metered or expensive providers to cut off a stuck self-heal loop. Must be a positive integer. Default: 50 (the runner's built-in cap). Not compatible with --agent MULTI_AGENT_V3. | - | +| `--max-llm-calls` | Cap the number of LLM calls per objective for the legacy TRAILBLAZE_RUNNER agent. Useful on metered or expensive providers to cut off a stuck self-heal loop. Must be a positive integer. Default: 25 (the runner's built-in cap). Not compatible with --agent MULTI_AGENT_V3. | - | | `--no-report` | Skip HTML report generation after execution | - | | `--full-report-payloads` | Embed full event payloads in the after-run HTML report even for sessions that passed, instead of applying the report size budgets (which truncate large successful network bodies and elide repeated intermediate snapshots to keep the report small). Failed sessions always embed full payloads regardless. The on-disk events/ artifacts are never budgeted, so an existing session can also be regenerated in full later via `trailblaze report --full-report-payloads`. Applies to in-process runs; a run delegated to an already-running daemon doesn't generate a report from this process. | - | | `--save-recording` | Save the recording back to the trail source directory after a successful run. Default: on. Use --no-save-recording to skip. Even when on, the recording is only saved when --self-heal was enabled OR this device isn't recorded yet — deterministic re-runs no-op the write so they can't clobber a hand-edited source. See --unified-recordings for the on-disk format. | - | diff --git a/docs/devlog/2026-08-06-all-universal-root-classifier.md b/docs/devlog/2026-08-06-all-universal-root-classifier.md new file mode 100644 index 000000000..cd10cab42 --- /dev/null +++ b/docs/devlog/2026-08-06-all-universal-root-classifier.md @@ -0,0 +1,107 @@ +--- +title: "`all:` — the universal root classifier" +type: decision +date: 2026-08-06 +--- + +# `all:` — the universal root classifier + +Every classifier lineage now ends at a single universal root, `all`. An entry +keyed `all:` in any classifier-keyed map — step `recordings:`, the trailhead's +recordings, waypoint blocks, `config.devices:` driver pins, `config.skip:` — +resolves for **every** device, at the **lowest** priority. + +## Background + +The [classifier lineage](2026-06-28-classifier-lineage-primitive.md) resolves +classifier-keyed maps closest-wins: walk the device's chain from most specific +(`ios-iphone`) up to its family root (`ios`) and take the first declared entry. +Each chain ended at its **platform family**, so there was no key that reached +every device. + +That forces cross-platform trails whose platforms genuinely share an entry to +declare it once per platform. The trailhead is where this bites hardest: it is +one tool call per device, and a target whose trailhead tool is itself +cross-platform (`supportedPlatforms: [android, ios]`, same args on both) still +needs byte-identical `android:` and `ios:` blocks in every trail — a copy that +can silently drift. + +```yaml +trailhead: + step: Launch signed in on the target screen + recording: + android: + app_launchSignedIn: { route: /settings } + ios: + app_launchSignedIn: { route: /settings } # byte-identical duplicate +``` + +## What we decided + +### `all` is the implicit ancestor of every classifier + +`TrailblazeClassifierLineage` appends `all` as the final entry of every +non-empty chain, in both `chainFor` (single classifier) and `resolutionChain` +(a device's broad-first segments): + +``` +android-phone → android → all +ios-iphone → ios → all +[ios, iphone] → ios-iphone, ios, iphone, all +``` + +The duplicated trailhead above becomes one block: + +```yaml +trailhead: + step: Launch signed in on the target screen + recording: + all: + app_launchSignedIn: { route: /settings } +``` + +### A default, not a straitjacket + +`all` sits **strictly last** on every chain — after the compound identity, its +ancestors, and every bare-segment fallback. Any explicitly-declared classifier +outranks it, so a platform that genuinely diverges keys its own entry and wins +on that platform while the others keep the shared one: + +```yaml +recordings: + all: [...] # what the platforms share + ios: [...] # iOS diverges; wins on iOS only +``` + +In `resolutionChain` the append happens once at the end of the merged chain — +not inside each per-segment expansion — so `all` can never ride the compound +identity's lineage in ahead of a lower-priority segment fallback (an +`iphone:`-keyed entry still beats `all:`). + +### The vocabulary was already reserved + +`all` is already the format's "every one of them" meta-key: a target manifest's +`drivers:` list accepts `all` for every driver type (`DriverTypeKey`). This +change gives the same word the same meaning in the classifier namespace. +`registerParentOverride` now rejects `all` as a child — the universal root +cannot be given a parent. + +### One consumer adjusted: filename→platform backfill + +`TrailIndexBuilder.platformFromFileName` derived a legacy recording's platform +by taking the **last** entry of the filename stem's chain and checking it +against the platform roots. The last entry is now always `all`, so it takes the +first platform-rooted ancestor on the chain instead — same result for every +real stem, robust to the new root. + +## What did not change + +- **The recorder never writes `all:`.** A recording session keys its slot by + the device it ran on, as before. `all:` is an author-side move — typically + collapsing two platform slots after verifying both replay the same tools. +- **Legacy per-platform filename candidates** (`ios-iphone.trail.yaml`) are + computed by segment truncation, not the lineage — no `all.trail.yaml` is ever + probed. +- **Chain totality.** `chainFor` still returns a non-empty chain whose first + element is the input classifier; an empty/blank input still yields an empty + chain (`all` is a fallback for a device identity, not a substitute for one). diff --git a/docs/devlog/index.md b/docs/devlog/index.md index 76521df7d..151a27034 100644 --- a/docs/devlog/index.md +++ b/docs/devlog/index.md @@ -11,6 +11,7 @@ Entries tagged as **Decision** record significant architectural or technical cho | Date | Title | Type | | :--- | :--- | :--- | +| 2026-08-06 | [`all:` — the universal root classifier](2026-08-06-all-universal-root-classifier.md) | Decision | | 2026-07-03 | [Batched tool-execution scope: one context + one snapshot frame per recording](2026-07-03-batched-tool-execution-scope.md) | Decision | | 2026-07-01 | [Type-checking trail recordings by transpiling them to TypeScript](2026-07-01-trail-recording-type-validation.md) | Decision | | 2026-06-30 | [assertWaypoint tool replaces the per-step postcondition field](2026-06-30-assertwaypoint-tool-replaces-postcondition-field.md) | Decision | diff --git a/docs/generated/LLM_MODELS.md b/docs/generated/LLM_MODELS.md index 2a64128e3..c2d199f9d 100644 --- a/docs/generated/LLM_MODELS.md +++ b/docs/generated/LLM_MODELS.md @@ -21,7 +21,7 @@ Trailblaze ships with the following built-in models. When you reference a model | `gemini-3.1-flash-lite-preview` | 1M | 65K | $0.25 | $1.50 | $0.03 | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | | `gemini-3.1-pro-preview` | 1M | 65K | $2.00 | $12.00 | $0.20 | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | | `gemini-3.1-pro-preview-customtools` | 1M | 65K | $2.00 | $12.00 | $0.20 | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | -| `gemma-4-26b-a4b-it` | 262K | 65K | free | free | free | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | +| `gemma-4-31b-it` | 262K | 65K | free | free | free | basic-json-schema, completion, document, image, multipleChoices, openai-endpoint-chat-completions, openai-endpoint-responses, speculation, standard-json-schema, temperature, toolChoice, tools | ## Ollama diff --git a/docs/generated/external-config.md b/docs/generated/external-config.md index 864b4c8ff..06ebf288c 100644 --- a/docs/generated/external-config.md +++ b/docs/generated/external-config.md @@ -171,7 +171,7 @@ Toolsets are declared in `trailmaps//toolsets/*.yaml`. They are pure YAML gr | `android_primitives` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation` | 7 | | `compose_core` | No | `compose` | 6 | | `compose_verification` | No | `compose` | 3 | -| `core_interaction` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation`, `ios-axe`, `ios-host` | 20 | +| `core_interaction` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation`, `ios-axe`, `ios-host` | 21 | | `memory` | No | `all drivers` | 8 | | `meta` | Yes | `all drivers` | 1 | | `mobile_primitives` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation`, `ios-axe`, `ios-host` | 5 | diff --git a/docs/generated/functions/custom/wait.md b/docs/generated/functions/custom/wait.md index 862d91e0f..2985048c1 100644 --- a/docs/generated/functions/custom/wait.md +++ b/docs/generated/functions/custom/wait.md @@ -4,8 +4,11 @@ # `wait` -Wait for a specified amount of time. Use when you see a loading screen — prefer this over -pressing the back button. +Settle on a loading screen: block until the UI goes quiet, up to a ceiling. This returns as soon +as the UI is idle, so on an already-static screen it returns almost immediately rather than +waiting the full time — it is a ceiling, not a duration. Use when you see a loading screen — +prefer this over pressing the back button. If you are waiting for something specific to appear, +assert on that element instead: a quiet UI does not mean the thing you expect has arrived. ## Source @@ -23,7 +26,7 @@ pressing the back button. ### Optional parameters - `timeToWaitInSeconds` — `Integer` - Unit: seconds. Default Value: 5 seconds. + Ceiling on how long to settle for, in seconds — not a guaranteed duration. Default Value: 5 seconds. ## Output diff --git a/docs/generated/functions/custom/waitForChange.md b/docs/generated/functions/custom/waitForChange.md index efc0a79ff..ad7f66c8b 100644 --- a/docs/generated/functions/custom/waitForChange.md +++ b/docs/generated/functions/custom/waitForChange.md @@ -7,8 +7,10 @@ Wait until the UI has settled after your action. Use this instead of a fixed-duration wait when you've triggered an action (a new screen loads, content updates, a list scrolls) and want to block until the UI is quiet again. Returns immediately if the UI is already settled when this runs. -Known limit: it cannot wait for a delayed async change that hasn't started yet — for that, use a -specific-element wait (e.g. assertVisible on the element you expect to appear). +Known limit: it returns as soon as the UI is quiet, so it cannot wait out a change that has not +started yet. For that, poll the actual state — assertNotVisibleWithText to wait a loading indicator +out, or take a fresh snapshot and assertVisible a ref that is present (assertVisible checks the +current snapshot and does not itself wait for an element to appear). ## Source diff --git a/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_openContact.ts b/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_openContact.ts index cf3ea65f7..eac56a16c 100644 --- a/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_openContact.ts +++ b/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_openContact.ts @@ -43,9 +43,13 @@ export const contacts_ios_openContact = trailblaze.tool( openFirstResult: true, }); // Confirm we actually reached the contact DETAIL screen before trusting the - // heading. A name-only check is NOT sufficient: when the contact doesn't exist - // the row tap lands on the search field (no navigation) and the typed query keeps - // `name` visible, so a heading assert still passes — a false "opened". The detail + // heading. A name-only check is NOT sufficient: the typed query keeps `name` + // visible in the search field, so if navigation silently failed a heading + // assert would still pass — a false "opened". (The search tool's row tap is + // label-scoped so its node-selector path can no longer resolve the search + // field; the framework's Maestro fallback lowers it to a legacy text match + // that still could, which is why this anchor stays as the independent + // destination check.) The detail // screen's top-right "Edit" button is the reliable detail-only anchor (the // list/search screens surface "Add"/"Cancel" there, never "Edit"). We resolve it // via `findMatches` against the iOS accessibility tree — `assertVisibleWith- diff --git a/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.test.ts b/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.test.ts index 721144924..6d81b2512 100644 --- a/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.test.ts +++ b/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.test.ts @@ -9,8 +9,43 @@ import { describe, expect, test } from "bun:test"; import { createMockClient, createMockContext } from "@trailblaze/scripting/testing"; +import type { MatchDescriptor, TrailblazeNodeSelector } from "@trailblaze/scripting"; -import { contacts_ios_searchContacts } from "./contacts_ios_searchContacts"; +import { + contacts_ios_searchContacts, + type SearchContactsArgs, +} from "./contacts_ios_searchContacts"; + +/** One canned `findMatches` hit — the shape the daemon returns for a visible row. */ +const ROW_MATCH: MatchDescriptor = { + indexPath: [0, 1], + bounds: { left: 0, top: 200, right: 390, bottom: 260 }, + matchedText: "John Appleseed", +}; + +/** Stubs `findMatches` to report the result row as visible. */ +function stubRowVisible(client: ReturnType): void { + client.stub("findMatches", { textContent: "", structuredContent: [ROW_MATCH] }); +} + +/** + * Compiles a selector's emitted `*Regex` the way the device resolves it: node selectors + * full-match their pattern, so anchor it before testing candidate labels. + */ +function anchored(pattern: string | null | undefined): RegExp { + expect(typeof pattern).toBe("string"); + return new RegExp(`^(?:${pattern})$`); +} + +/** Runs the tool against a stubbed visible row and returns the selector the tap dispatched. */ +async function rowSelectorFor(args: SearchContactsArgs): Promise { + const client = createMockClient(); + stubRowVisible(client); + await contacts_ios_searchContacts(args, createMockContext({ platform: "ios" }), client); + const tapCall = client.calls.find((c) => c.tool === "tapOnElementBySelector"); + expect(tapCall).toBeDefined(); + return tapCall!.args.nodeSelector as TrailblazeNodeSelector; +} describe("contacts_ios_searchContacts", () => { test("throws a descriptive no-results error when the 'No Results' banner is present", async () => { @@ -61,11 +96,12 @@ describe("contacts_ios_searchContacts", () => { expect(client.calls).toHaveLength(6); }); - test("taps the first matching row when results are present", async () => { - // The default mock returns success for every dispatch — including the negative - // no-results probe, whose success means "no banner element on screen", i.e. results - // ARE present. The tool then taps the row matching `rowText` and returns. + test("waits for and taps the matching row via a label-scoped selector", async () => { + // The default mock returns success for the negative no-results probe (no banner → + // results ARE present); `findMatches` is stubbed to report the row as visible. The + // tool then taps the row matching `rowText` and returns. const client = createMockClient(); + stubRowVisible(client); const ctx = createMockContext({ platform: "ios" }); const result = await contacts_ios_searchContacts( @@ -81,13 +117,147 @@ describe("contacts_ios_searchContacts", () => { "tapOnElementWithText", // focus the "Search" input "inputText", // type the query "assertNotVisibleWithText", // no-results probe — passes (no banner) → results present - "tapOnElementWithText", // tap the result row + "findMatches", // wait for the result row to render + "tapOnElementBySelector", // tap the result row ]); - // The row tap targets `rowText`, not the raw query — the partial-prefix flow. - expect(client.calls[6]?.args).toMatchObject({ text: "John Appleseed" }); + // The row tap targets `rowText`, not the raw query — the partial-prefix flow — and the + // wait probe uses the same selector the tap dispatches, so they can't drift apart. + const tapSelector = client.calls[7]?.args.nodeSelector as TrailblazeNodeSelector; + expect(client.calls[6]?.args.selector).toEqual(tapSelector); + expect(tapSelector.iosMaestro?.accessibilityTextRegex).toContain("John Appleseed"); expect(result).toContain('opened the row matching "John Appleseed"'); }); + test("query == rowText: the node-selector row tap cannot resolve the search field", async () => { + // THE regression this tool shipped with (ios-contacts-replay-smoke red 23/25 on main): + // `contacts_ios_openContact` passes the same full name as `query` and `rowText`, and on + // the host driver a bare text tap resolved the search field's own typed text (a text + // field's Maestro `text` attribute is its VALUE), so no navigation happened. The + // contract pinned here: the row tap is `tapOnElementBySelector` matching on the AX + // *label* only — a text field's label is its placeholder, never the typed value — with + // no text/value-shaped predicate the typed query could satisfy, and no bare text tap of + // `rowText` is dispatched at all. The framework's Maestro fallback lowers + // `accessibilityTextRegex` to legacy `textRegex` (text | hintText | accessibilityText), + // which the search field's typed value CAN satisfy — but the same lowering carries the + // results-list `childOf` scope through, so the field stays out of the candidate set on + // that path too (see the results-list scope test below). + const client = createMockClient(); + stubRowVisible(client); + const ctx = createMockContext({ platform: "ios" }); + + await contacts_ios_searchContacts( + { query: "John Appleseed", rowText: "John Appleseed" }, + ctx, + client, + ); + + // No bare text tap ever targets the row text (the only tapOnElementWithText allowed is + // the "Search" field focus). + const bareTextTaps = client.calls + .filter((c) => c.tool === "tapOnElementWithText") + .map((c) => c.args.text); + expect(bareTextTaps).toEqual(["Search"]); + + const tapCall = client.calls.find((c) => c.tool === "tapOnElementBySelector"); + expect(tapCall).toBeDefined(); + const selector = tapCall!.args.nodeSelector as TrailblazeNodeSelector; + // Label-scoped only: no predicate that a text field's typed value can satisfy. + expect(selector.iosMaestro?.accessibilityTextRegex).toBeDefined(); + expect(selector.iosMaestro?.textRegex).toBeUndefined(); + expect(selector.iosMaestro?.hintTextRegex).toBeUndefined(); + // Deterministic single-node resolution when the cell and its inner label both match. + expect(selector.index).toBe(0); + }); + + test("row selector matches the row label literally, anywhere in the label (contains)", async () => { + // The emitted accessibilityTextRegex is a wire contract the device resolves. Verify its + // BEHAVIOR (not its exact string). Node selectors full-match their regex, so the emitted + // pattern must preserve the CONTAINS semantics the old `tapOnElementWithText` had: + // `rowText` may appear anywhere in the row label — a last-name-only query (rowText + // defaults to the query) must match the full-name row, and labels may append detail + // text after the name. A name containing regex metacharacters is matched literally, + // not as a pattern. + const client = createMockClient(); + stubRowVisible(client); + const ctx = createMockContext({ platform: "ios" }); + + await contacts_ios_searchContacts( + { query: "Dr", rowText: "Dr. O'Brien (Work)" }, + ctx, + client, + ); + + const tapCall = client.calls.find((c) => c.tool === "tapOnElementBySelector"); + const selector = tapCall!.args.nodeSelector as TrailblazeNodeSelector; + const pattern = anchored(selector.iosMaestro?.accessibilityTextRegex); + expect(pattern.test("Dr. O'Brien (Work)")).toBe(true); + expect(pattern.test("Dr. O'Brien (Work), mobile")).toBe(true); + // Contains, not prefix: the label may carry text BEFORE rowText too. + expect(pattern.test("Prof. Dr. O'Brien (Work)")).toBe(true); + // "." must not act as a wildcard — a literal-escape regression would match this. + expect(pattern.test("DrX O'Brien (Work)")).toBe(false); + }); + + test("a last-name-only rowText still matches the full-name row", async () => { + // The canonical contains case, and the one a prefix-shaped pattern would break: searching + // by surname (rowText defaults to the query) must still match the row whose label leads + // with the first name. + const selector = await rowSelectorFor({ query: "Appleseed" }); + const pattern = anchored(selector.iosMaestro?.accessibilityTextRegex); + expect(pattern.test("John Appleseed")).toBe(true); + expect(pattern.test("Appleseed")).toBe(true); + expect(pattern.test("Kate Bell")).toBe(false); + }); + + test("a rowText that also matches the search field's label is scoped to the results list", async () => { + // Contains matching on the label is necessary (previous test) but not sufficient. Any + // `rowText` that is a substring of the search chrome's labels — "ear" for a "Teddy Bear" + // row is a substring of the field's "Search" placeholder, of the "Search results" panel + // label, and of the "Clear text" button — makes the label predicate alone ambiguous. Both + // `findMatches` and the tap use this selector with `index: 0`, so if the search field were + // still a candidate the topmost match could be the field: the wait would succeed, the tap + // would only focus it, and the tool would report success without opening the row. + const selector = await rowSelectorFor({ query: "ear" }); + + // The ambiguity is real — this is why the extra constraint exists, not a regression. + const rowPattern = anchored(selector.iosMaestro?.accessibilityTextRegex); + expect(rowPattern.test("Teddy Bear")).toBe(true); + expect(rowPattern.test("Search")).toBe(true); + expect(rowPattern.test("Clear text")).toBe(true); + + // What removes the search field is the structural scope, not a narrower text match: the + // match must be a DESCENDANT of the search-results panel. On the real Contacts hierarchy + // the rows live under that panel while the search field and its chrome live under a + // sibling "Toolbar" branch, so the whole field/chrome family is out of the candidate set + // before `index: 0` is applied. The scope anchor is the panel and only the panel — it can + // never resolve to the search field itself, and `childOf` excludes the anchor, so the + // full-screen panel node can't be tapped either. + const scopePattern = anchored(selector.childOf?.iosMaestro?.accessibilityTextRegex); + expect(scopePattern.test("Search results")).toBe(true); + expect(scopePattern.test("Search")).toBe(false); + expect(scopePattern.test("Search: ear")).toBe(false); + expect(scopePattern.test("Clear text")).toBe(false); + expect(scopePattern.test("Toolbar")).toBe(false); + + // Index stays the last-resort row disambiguator (cell vs. its inner StaticText). + expect(selector.index).toBe(0); + }); + + test("throws a descriptive error when results exist but the rowText row never appears", async () => { + // No banner (all the tool actually established) but `findMatches` reports no row labeled + // `rowText` within the wait budget — the "wrong rowText" failure, distinct from "wrong + // query". No tap may be dispatched against a row that never rendered. + const client = createMockClient(); + client.stub("findMatches", { textContent: "", structuredContent: [] }); + const ctx = createMockContext({ platform: "ios" }); + + await expect( + contacts_ios_searchContacts({ query: "John", rowText: "Johnny Nonexistent" }, ctx, client), + ).rejects.toThrow(/shows no "No Results" banner, but no row labeled "Johnny Nonexistent"/); + + expect(client.calls.map((c) => c.tool)).not.toContain("tapOnElementBySelector"); + }); + test("returns early without probing No Results when openFirstResult is false", async () => { // `openFirstResult: false` is the "type the query and stop" branch — used by callers // that want to verify the inline autocomplete suggestions instead of opening a row. @@ -114,6 +284,7 @@ describe("contacts_ios_searchContacts", () => { test("applies module defaults when args fields are omitted", async () => { const client = createMockClient(); + stubRowVisible(client); const ctx = createMockContext({ platform: "ios" }); // No `query` → tool falls back to its `DEFAULT_QUERY` module constant. Under the diff --git a/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.ts b/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.ts index 52e46935e..652825b0f 100644 --- a/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.ts +++ b/examples/ios-contacts/trails/config/trailmaps/contacts/tools/contacts_ios_searchContacts.ts @@ -1,8 +1,26 @@ -import { trailblaze } from "@trailblaze/scripting"; +import { trailblaze, type TrailblazeNodeSelector } from "@trailblaze/scripting"; import { ensureContactsRoot, nonEmptyString, tryOrFalse } from "./contacts_ios_shared"; const DEFAULT_QUERY = "John"; +/** Wait budget for the matching result row to render after the query is typed. */ +const ROW_WAIT_MS = 5000; + +/** + * Accessibility label of the container iOS Contacts wraps the search-result rows in. + * Used as the row selector's `childOf` scope so a substring `rowText` can only ever + * resolve to a result row, never to the search field or its toolbar chrome. + * + * Deliberately a bare literal, not a contains pattern: node selectors full-match, and + * this string is exact. Provenance is the captured Contacts hierarchy in + * `trails/config/trailmaps/contacts/waypoints/ios/contacts_ios_search_results_with_clear.example.json` + * — exactly one node carries this label, the result-row cell and its inner StaticText are + * both inside its subtree, and the search field (with its magnifying-glass and "Clear text" + * children) is a sibling outside it. Six waypoints in that directory already match the + * same string as an exact literal. + */ +const RESULTS_LIST_LABEL = "Search results"; + export interface SearchContactsArgs { /** Query to type into the contacts list's pull-down search field. */ query?: string; @@ -64,13 +82,13 @@ export const contacts_ios_searchContacts = trailblaze.tool( return `Typed "${query}" into Contacts search and stopped (no result tapped).`; } - // Pre-flight: surface the no-results state before the row tap below. This is - // load-bearing — without it, a query that matches nothing falls through to - // `tapOnElementWithText(rowText)`, which then matches the query text still - // showing in the *search field* (not a contact row), so no navigation - // happens and a caller's open-and-verify-name probe false-positives on that - // same search-field text. (That's exactly how the create-then-delete - // defensive teardown failed on a fresh simulator.) + // Pre-flight: surface the no-results state before the row tap below, so a + // query that matches nothing fails with "wrong query" instead of falling + // through to the row wait and reporting "wrong rowText". (Historically this + // was also the only guard against the not-found tap resolving the query text + // still showing in the *search field* — the label-scoped, results-list-scoped + // row selector below now rules that out, but the distinct error stays + // valuable.) // // iOS renders the banner as `No Results for ""`, so we must match it // as a substring/regex. `assertVisibleWithAccessibilityText` is exact-match @@ -87,7 +105,90 @@ export const contacts_ios_searchContacts = trailblaze.tool( ); } - await ctx.tools.tapOnElementWithText({ text: rowText }); + // Tap the result ROW via its accessibility label, scoped to the results list — never via + // a bare text match. + // + // 1. LABEL, not text. A bare `tapOnElementWithText(rowText)` matches any node whose + // text / hintText / accessibilityText contains `rowText` — and on the host + // (Maestro/XCUITest) tree a text field's `text` attribute is its typed VALUE, so when + // `query == rowText` (e.g. `contacts_ios_openContact` passing the same full name to + // both) the tap resolved the search field's own typed text instead of the result row, + // no navigation happened, and the caller's detail-screen anchor never appeared. + // `accessibilityTextRegex` matches the AX *label* on both iOS drivers (host: + // `accessibilityText` = AXLabel; AXe: the iosMaestro→AXe bridge maps it to `.label`), + // and a search field's label is its placeholder ("Search") — never the typed value. + // The captured hierarchy cited on `RESULTS_LIST_LABEL` above shows this directly: the + // search field is one node carrying `accessibilityText: "Search"`, `hintText: "Search"`, + // and `text: "Kate"` (the query that had been typed when the capture was taken). + // + // 2. CONTAINS, not prefix. Node selectors full-match their regex, so a bare `rowText` + // would only match labels that equal (or, with a trailing `.*`, start with) it — + // breaking substring queries like a last name (`rowText: "Appleseed"` must still match + // the "John Appleseed" row) and labels that append detail text after the name. The + // surrounding `.*` restores the old `tapOnElementWithText` CONTAINS contract. + // + // 3. SCOPED to the results list. Contains-matching on the label is necessary but not + // sufficient: the search field's own label ("Search") contains plenty of substrings a + // caller may legitimately pass as `rowText` — `rowText: "ear"` opening a "Teddy Bear" + // row also matches "Search", and so do "Search results" and the "Clear text" button. + // Since `findMatches` and the tap share this selector with `index: 0`, the topmost of + // those could be the search field: the wait would succeed, the tap would only focus the + // field, and the tool would report success without ever opening the row. `childOf` + // fixes that structurally instead of narrowing the text match: on the real Contacts + // hierarchy every result row is a descendant of the "Search results" container, while + // the search field and all its chrome (the magnifying-glass image, "Clear text", + // "close") live under a sibling "Toolbar" branch — so scoping the search to that + // container's descendants removes the whole search-field/chrome family from the + // candidate set. `childOf` also excludes the anchor itself, so the full-screen + // "Search results" container can't be picked either. + // + // Both iOS drivers evaluate this scope, and both via the same underlying AX attribute: + // on the host tree `accessibilityText` is the XCUIElement label; on an AXe tree the + // iosMaestro→AXe bridge routes `accessibilityTextRegex` to `.label` (AXLabel). Fields + // that only exist on one side were rejected for exactly this reason — `classNameRegex` + // is unusable here because the host iOS tree reports no `class` attribute for this app + // at all, so a class constraint would match nothing on the host driver. + // + // With the scope in place, `index: 0` is only disambiguating rows: it pins the topmost + // match so the tap resolves a single node even when the row cell and its inner StaticText + // both carry the label. + // + // The guarantee survives the Maestro fallback too. Under the default PREFER_NODE_SELECTOR + // mode, if the node-selector tap returns no node (transient tree-fetch failure, row stops + // resolving between the `findMatches` probe and the tap), `TapOnByElementSelector` falls + // back to Maestro — whose lowering turns `accessibilityTextRegex` into legacy `textRegex` + // (text | hintText | accessibilityText), which a typed query CAN satisfy. But the same + // lowering also carries `childOf` through, so even there the match set stays inside the + // results list and excludes the search field. + const rowSelector: TrailblazeNodeSelector = { + iosMaestro: { accessibilityTextRegex: `.*${escapeRegExp(rowText)}.*` }, + childOf: { iosMaestro: { accessibilityTextRegex: RESULTS_LIST_LABEL } }, + index: 0, + }; + // Bounded wait for the row to render (`findMatches` re-polls the live hierarchy + // until a match appears or the budget elapses — no fixed sleep), + // plus a distinct error for "results exist but none is labeled `rowText`" — a different + // failure from the no-results branch above (wrong rowText vs wrong query). + const rows = await ctx.tools.findMatches({ + selector: rowSelector, + timeoutMs: ROW_WAIT_MS, + }); + if (rows.length === 0) { + throw new Error( + `contacts_ios_searchContacts: query "${query}" shows no "No Results" banner, ` + + `but no row labeled "${rowText}" appeared in the "${RESULTS_LIST_LABEL}" list ` + + `within ${ROW_WAIT_MS}ms.`, + ); + } + await ctx.tools.tapOnElementBySelector({ + reason: `Open the "${rowText}" search result row.`, + nodeSelector: rowSelector, + }); return `Searched for "${query}" and opened the row matching "${rowText}".`; }, ); + +/** Escapes regex metacharacters so a contact name is matched literally. */ +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/examples/ios-sample-app/IosSampleApp/FormsScreen.swift b/examples/ios-sample-app/IosSampleApp/FormsScreen.swift index ef3447894..261e55902 100644 --- a/examples/ios-sample-app/IosSampleApp/FormsScreen.swift +++ b/examples/ios-sample-app/IosSampleApp/FormsScreen.swift @@ -56,6 +56,12 @@ struct FormsScreen: View { .accessibilityIdentifier("tv_submission_result") } + // Kept below the form so the field/button selectors above it stay put. + NavigationLink("Open Web Content") { + WebContentScreen() + } + .accessibilityIdentifier("link_web_content") + Spacer() } .padding(16) diff --git a/examples/ios-sample-app/IosSampleApp/IosSampleAppApp.swift b/examples/ios-sample-app/IosSampleApp/IosSampleAppApp.swift index 619adf1c1..e54157523 100644 --- a/examples/ios-sample-app/IosSampleApp/IosSampleAppApp.swift +++ b/examples/ios-sample-app/IosSampleApp/IosSampleAppApp.swift @@ -4,7 +4,9 @@ import SwiftUI struct IosSampleAppApp: App { var body: some Scene { WindowGroup { - FormsScreen() + NavigationStack { + FormsScreen() + } } } } diff --git a/examples/ios-sample-app/IosSampleApp/WebContentScreen.swift b/examples/ios-sample-app/IosSampleApp/WebContentScreen.swift new file mode 100644 index 000000000..1beec1571 --- /dev/null +++ b/examples/ios-sample-app/IosSampleApp/WebContentScreen.swift @@ -0,0 +1,69 @@ +import SwiftUI +import WebKit + +/// Controlled fixture for the `IOS_AXE` driver's web-content descent. +/// +/// iOS renders `WKWebView` page content in a separate WebContent process, so none of the elements +/// below appear in an in-process accessibility walk. They are only reachable by hit-testing across +/// the process boundary, which is what `axe describe-ui --include-web-content` does. Driven by +/// `trails/eval/ios/sample-app/webview-content.trail.yaml`. +struct WebContentScreen: View { + var body: some View { + VStack(alignment: .leading, spacing: 12) { + // Native, so a failing trail can distinguish "never reached the screen" from + // "reached it but could not see into the web view". + Text("Native label above the web view") + .font(.footnote) + .foregroundStyle(.secondary) + .accessibilityIdentifier("label_web_native") + + WebContentFixture() + .accessibilityIdentifier("webview_fixture") + } + .padding() + .navigationTitle("Web Content") + } +} + +private struct WebContentFixture: UIViewRepresentable { + /// Loaded from a string, not a URL: the fixture has to render identically on a CI simulator with + /// no network access. + private static let html = """ + + + + + + + +

Web heading in a separate process

+

Web paragraph rendered by WebKit

+ +
+
+ + + """ + + func makeUIView(context: Context) -> WKWebView { + let webView = WKWebView(frame: .zero) + webView.isOpaque = false + webView.scrollView.isScrollEnabled = false + webView.loadHTMLString(Self.html, baseURL: nil) + return webView + } + + func updateUIView(_ webView: WKWebView, context: Context) {} +} + +#Preview { + NavigationStack { + WebContentScreen() + } +} diff --git a/examples/ios-sample-app/README.md b/examples/ios-sample-app/README.md index 362e93366..9af5554d4 100644 --- a/examples/ios-sample-app/README.md +++ b/examples/ios-sample-app/README.md @@ -1,14 +1,15 @@ # ios-sample-app -Minimal SwiftUI app that mirrors the [`android-sample-app`](../android-sample-app/) Forms tab. Built to serve as a controlled target for Trailblaze iOS evals — currently just the [clipboard round-trip](../../../trails/eval/ios/sample-app/clipboard-round-trip.trail.yaml). +Minimal SwiftUI app that mirrors the [`android-sample-app`](../android-sample-app/) Forms tab. Built to serve as a controlled target for Trailblaze iOS evals: the [clipboard round-trip](../../../trails/eval/ios/sample-app/clipboard-round-trip.trail.yaml) and the [WKWebView content descent](../../../trails/eval/ios/sample-app/webview-content.trail.yaml). ## What's here ``` ios-sample-app/ ├── IosSampleApp/ ← SwiftUI sources (add new .swift files here) -│ ├── IosSampleAppApp.swift ← @main app entry -│ ├── FormsScreen.swift ← single screen: Name + Email, Submit, Clear All +│ ├── IosSampleAppApp.swift ← @main app entry (NavigationStack root) +│ ├── FormsScreen.swift ← Name + Email, Submit, Clear All, link to Web Content +│ ├── WebContentScreen.swift ← WKWebView fixture: page content in a separate process │ └── Assets.xcassets/ ├── IosSampleApp.xcodeproj/ ← Xcode project (uses synchronized folder groups) └── build-and-install.sh ← xcodebuild + simctl install wrapper @@ -29,13 +30,18 @@ Bundle id: `xyz.block.trailblaze.examples.iossampleapp`. The script builds for `iphonesimulator` and installs the resulting `.app` onto the booted simulator. xcodebuild output is captured to `build/build.log`; on failure the last 80 lines are written to stderr so build problems are visible. -## Running the clipboard eval against this app +## Running the evals against this app ```bash ./examples/ios-sample-app/build-and-install.sh ./trailblaze run trails/eval/ios/sample-app/clipboard-round-trip.trail.yaml --device ios +./trailblaze run trails/eval/ios/sample-app/webview-content.trail.yaml --device ios ``` +The web-content eval needs an `axe` whose `describe-ui` accepts `--include-web-content`; without it, +`WKWebView` page content never reaches the accessibility tree and the trail fails at its first web +assertion. CI skips that trail automatically on an agent whose `axe` predates the flag. + ## Adding new screens Because the Xcode project uses synchronized folder groups, **no `pbxproj` edit is needed** to add a new Swift file — drop it into `IosSampleApp/` and Xcode picks it up automatically. Same applies to assets under `IosSampleApp/Assets.xcassets/`. diff --git a/sdks/typescript/src/generated/trailrunner-dtos.ts b/sdks/typescript/src/generated/trailrunner-dtos.ts index 79f3c1c8b..8dfbf7456 100644 --- a/sdks/typescript/src/generated/trailrunner-dtos.ts +++ b/sdks/typescript/src/generated/trailrunner-dtos.ts @@ -1281,10 +1281,13 @@ export interface Swipe { type?: AgentActionType; } +export type TapDispatchRoute = "ACTION_CLICK" | "GESTURE" | "GESTURE_AFTER_ACTION_CLICK_MISS"; + export interface TapPoint { class: "xyz.block.trailblaze.api.AgentDriverAction.TapPoint"; x: number; y: number; + dispatchRoute?: TapDispatchRoute | null; type?: AgentActionType; } diff --git a/sdks/typescript/src/matcher/matcher-parity-fixtures.json b/sdks/typescript/src/matcher/matcher-parity-fixtures.json index 33b36f39c..522ee06cd 100644 --- a/sdks/typescript/src/matcher/matcher-parity-fixtures.json +++ b/sdks/typescript/src/matcher/matcher-parity-fixtures.json @@ -316,5 +316,89 @@ "nativeMatches": true, "maestroMatches": true } + ], + "_iosMaestroHintBridgeComment": "Node-shaped contract for the iosMaestro→iosAxe bridge's hintTextRegex leg: help matches on any type; label/value match ONLY on text-input types (placeholder surfaces as AXLabel on newer iOS runtimes, as the empty field's AXValue on older ones). Consumed by the same two suites as `cases`. If either implementation's type gate or property set drifts, that side's suite fails.", + "iosMaestroHintBridgeCases": [ + { + "name": "placeholder surfaced as AXLabel (newer runtimes) matches a text input", + "hintTextRegex": "Search", + "type": "TextField", + "label": "Search", + "value": null, + "help": null, + "matches": true + }, + { + "name": "placeholder surfaced as AXValue (older runtimes) matches an empty text input", + "hintTextRegex": "Search", + "type": "TextField", + "label": null, + "value": "Search", + "help": null, + "matches": true + }, + { + "name": "help carries the hint regardless of element type", + "hintTextRegex": "Search", + "type": "Image", + "label": null, + "value": null, + "help": "Search", + "matches": true + }, + { + "name": "decorative Image carrying the hint as its label does not match", + "hintTextRegex": "Search", + "type": "Image", + "label": "Search", + "value": null, + "help": null, + "matches": false + }, + { + "name": "decorative Image carrying the hint as its value does not match", + "hintTextRegex": "Search", + "type": "Image", + "label": null, + "value": "Search", + "help": null, + "matches": false + }, + { + "name": "StaticText is not a text-input type", + "hintTextRegex": "Search", + "type": "StaticText", + "label": "Search", + "value": null, + "help": null, + "matches": false + }, + { + "name": "SecureTextField is a text-input type (placeholder-as-value)", + "hintTextRegex": "Passcode", + "type": "SecureTextField", + "label": null, + "value": "Passcode", + "help": null, + "matches": true + }, + { + "name": "hint bridge keeps the Maestro dialect on the value leg (case-insensitive)", + "hintTextRegex": "Search", + "type": "TextField", + "label": null, + "value": "search", + "help": null, + "matches": true + }, + { + "name": "a filled field whose typed value differs from the hint does not match", + "hintTextRegex": "Search", + "type": "TextField", + "label": null, + "value": "Steve", + "help": null, + "matches": false + } ] } diff --git a/sdks/typescript/src/matcher/matcher-parity.test.ts b/sdks/typescript/src/matcher/matcher-parity.test.ts index a2e6c18f7..79c7f0be5 100644 --- a/sdks/typescript/src/matcher/matcher-parity.test.ts +++ b/sdks/typescript/src/matcher/matcher-parity.test.ts @@ -86,3 +86,35 @@ for (const c of fixtures.cases) { }); } } + +// Node-shaped contract for the iosMaestro→iosAxe bridge's hintTextRegex leg (help on any +// type; label/value only on text-input types). Same source of truth and drift guarantee as +// `cases` above — the Kotlin mirror is MatcherParityFixturesTest. +for (const c of fixtures.iosMaestroHintBridgeCases) { + test(`parity (hint bridge): ${c.name}`, () => { + const target: TrailblazeNode = { + nodeId: 2, + children: [], + bounds: { left: 0, top: 0, right: 100, bottom: 50 }, + driverDetail: { + class: "iosAxe", + type: c.type, + label: c.label, + value: c.value, + help: c.help, + }, + }; + const root: TrailblazeNode = { + nodeId: 1, + children: [target], + bounds: { left: 0, top: 0, right: 200, bottom: 100 }, + driverDetail: { class: "iosAxe" }, + }; + const result = resolve(root, selectors.iosMaestro({ hintTextRegex: c.hintTextRegex })); + const matched = result.kind === "singleMatch"; + expect( + matched, + `hintTextRegex=[${c.hintTextRegex}] type=[${c.type}] label=[${c.label}] value=[${c.value}] help=[${c.help}] expected matches=${c.matches}, got ${matched}`, + ).toBe(c.matches); + }); +} diff --git a/sdks/typescript/src/matcher/resolver.test.ts b/sdks/typescript/src/matcher/resolver.test.ts index d27f0fb11..aadfa9568 100644 --- a/sdks/typescript/src/matcher/resolver.test.ts +++ b/sdks/typescript/src/matcher/resolver.test.ts @@ -1106,6 +1106,23 @@ describe("resolve — IosMaestro selector against IosAxe tree (cross-dialect bri expect(asSingleMatch(result).nodeId).toBe(searchField.nodeId); }); + test("hintTextRegex matches a text input's placeholder-as-value", () => { + resetIds(); + // Older iOS runtimes (18.x) leave an empty text field's AXLabel null and surface the + // placeholder as AXValue instead (the Contacts search field: label=null, value="Search", + // help=null). The type gate still keeps the decorative magnifying-glass Image + // (label="Search") — or one carrying the hint as its VALUE — from false-matching. + const searchField = nodeOf({ class: "iosAxe", type: "TextField", value: "Search" }); + const searchIcon = nodeOf({ class: "iosAxe", type: "Image", label: "Search" }); + const valueDecoy = nodeOf({ class: "iosAxe", type: "Image", value: "Search" }); + const root = nodeOf( + { class: "iosAxe" }, + { children: [searchIcon, valueDecoy, searchField] }, + ); + const result = resolve(root, selectors.iosMaestro({ hintTextRegex: "Search" })); + expect(asSingleMatch(result).nodeId).toBe(searchField.nodeId); + }); + test("MAESTRO dialect is case-insensitive", () => { resetIds(); const target = nodeOf({ class: "iosAxe", label: "Log In" }); diff --git a/sdks/typescript/src/matcher/resolver.ts b/sdks/typescript/src/matcher/resolver.ts index 9e6a61aba..430856b20 100644 --- a/sdks/typescript/src/matcher/resolver.ts +++ b/sdks/typescript/src/matcher/resolver.ts @@ -566,7 +566,9 @@ function matchesIosAxe( * selectors actually carry (see `MAESTRO_IOS_CLASS_ALIASES`): Maestro's iOS tree reports * the *label view* (`LabelView`, `UILabel`, `UIButtonLabel`, `UITextFieldLabel`) where AXe * reports the semantic element (`StaticText`, `Button`, `TextField`). - * - `hintText` matches `help`. + * - `hintText` matches `help`, or — on text-input types only — `label`/`value`, the + * properties iOS actually mirrors a placeholder onto (version-dependent; see the + * `hintTextRegex` branch below). * * Uses the `"maestro"` dialect throughout: the selector was authored under Maestro's * lenient semantics, and that's what it should still mean here. @@ -638,20 +640,26 @@ function matchesIosMaestroAgainstAxe( } if (match.hintTextRegex != null) { // iOS surfaces a text input's placeholder (Maestro's hintText / XCUITest's - // placeholderValue) as AXLabel, not AXHelp — so accept label too, but only on - // text-input types so a decorative node whose label happens to equal the hint - // (e.g. a magnifying-glass Image labeled "Search") can't false-match. + // placeholderValue) on the input element itself, not on AXHelp — but WHICH property + // carries it varies by AXe/iOS version: newer runtimes expose the placeholder on + // AXLabel, while older ones leave AXLabel null and surface it only as the empty + // field's AXValue (the Contacts search field: label=null, value="Search"). Accept + // help, plus label OR value — the latter two only on text-input types so a decorative + // node whose label happens to equal the hint (e.g. a magnifying-glass Image labeled + // "Search") can't false-match. Empty fields only: once the field has text, AXValue is + // the typed text and AXLabel stays null on older runtimes, so a hint~ lookup of that + // same field won't match there. Nothing in the AXe tree still carries the placeholder. const matchesHelp = requirePattern(match.hintTextRegex, detail.help ?? null, dialect); const matchesPlaceholder = IOS_TEXT_INPUT_TYPES.has(detail.type ?? "") && - requirePattern(match.hintTextRegex, detail.label ?? null, dialect); + matchesAnyPattern(match.hintTextRegex, dialect, detail.label ?? null, detail.value ?? null); if (!matchesHelp && !matchesPlaceholder) return false; } return true; } -/** XCUIElementType names whose AXLabel carries the placeholder while the field is empty. */ +/** XCUIElementType names whose AXLabel/AXValue carries the placeholder while the field is empty. */ const IOS_TEXT_INPUT_TYPES = new Set(["TextField", "SecureTextField", "SearchField", "TextView"]); /** diff --git a/sdks/typescript/src/testing.test.ts b/sdks/typescript/src/testing.test.ts index 8c2752d64..8a3d8411b 100644 --- a/sdks/typescript/src/testing.test.ts +++ b/sdks/typescript/src/testing.test.ts @@ -10,7 +10,12 @@ import { describe, expect, test } from "bun:test"; import { _unwrapToolResult, type TrailblazeCallToolResult } from "./client.js"; -import { createMockClient, createMockContext, type MockStubResponse } from "./testing.js"; +import { + createMockClient, + createMockContext, + createQueuedFindMatchesClient, + type MockStubResponse, +} from "./testing.js"; describe("createMockClient: call recording", () => { test("records every tool call in order with the args verbatim", async () => { @@ -286,3 +291,37 @@ describe("mock unwrap parity with production _unwrapToolResult", () => { }); } }); + +describe("createQueuedFindMatchesClient: queueToolFailure", () => { + test("rejects only the queued calls, then resolves normally", async () => { + // The drain is the behavior under test: a retry loop can only be pinned across a failure if a + // later call to the same tool succeeds. + const client = createQueuedFindMatchesClient(); + client.queueToolFailure("tapOnElementBySelector", ["no node matched the selector"]); + const tap = (client.tools as Record) => Promise>)[ + "tapOnElementBySelector" + ]!; + + await expect(tap({ attempt: 1 })).rejects.toThrow("no node matched the selector"); + await tap({ attempt: 2 }); + + // Both dispatches are recorded — a rejected call still counts as an attempt. + expect(client.calls.map((c) => c.tool)).toEqual([ + "tapOnElementBySelector", + "tapOnElementBySelector", + ]); + }); + + test("a queued failure on findMatches pre-empts its response queue", async () => { + const client = createQueuedFindMatchesClient(); + client.queueFindMatches([[{ indexPath: [0] }]]); + client.queueToolFailure("findMatches", ["driver not ready"]); + const findMatches = ( + client.tools as Record) => Promise> + )["findMatches"]!; + + await expect(findMatches({})).rejects.toThrow("driver not ready"); + // The queued response is untouched, so the next call still serves it. + expect(await findMatches({})).toEqual([{ indexPath: [0] }]); + }); +}); diff --git a/sdks/typescript/src/testing.ts b/sdks/typescript/src/testing.ts index d9f66b56d..57fd6299c 100644 --- a/sdks/typescript/src/testing.ts +++ b/sdks/typescript/src/testing.ts @@ -405,6 +405,14 @@ export interface QueuedFindMatchesClient extends TrailblazeClient { * moving UI across both. Like the findMatches queue, exhaustion is loud (throws with the args). */ queueWaitUntilNotVisible(responses: boolean[]): void; + /** + * Queue rejections for the next calls to `toolName`, modelling a tool that fails on device — a tap + * whose selector resolved nothing, say. Each queued message rejects one call with the production + * client's `"tool failed: "` wording; once the queue drains, later calls resolve normally. + * That drain is the point: it is what lets a test pin how a retry loop behaves across a failure, + * rather than only its all-succeed and all-fail ends. + */ + queueToolFailure(toolName: string, messages: string[]): void; } /** @@ -419,9 +427,14 @@ export function createQueuedFindMatchesClient(): QueuedFindMatchesClient { const calls: Array<{ tool: string; args: Record }> = []; const findMatchesQueue: Array = []; const waitUntilNotVisibleQueue: boolean[] = []; + const failureQueues = new Map(); const dispatch = (name: string, args: Record): unknown => { calls.push({ tool: name, args }); + const queuedFailure = failureQueues.get(name)?.shift(); + if (queuedFailure !== undefined) { + throw new Error(`trailblaze.client.callTool("${name}") tool failed: ${queuedFailure}`); + } if (name === "findMatches") { if (findMatchesQueue.length === 0) { throw new Error( @@ -462,6 +475,14 @@ export function createQueuedFindMatchesClient(): QueuedFindMatchesClient { queueWaitUntilNotVisible(responses) { waitUntilNotVisibleQueue.push(...responses); }, + queueToolFailure(toolName, messages) { + const existing = failureQueues.get(toolName); + if (existing) { + existing.push(...messages); + } else { + failureQueues.set(toolName, [...messages]); + } + }, }; } diff --git a/trailblaze-agent/src/main/java/xyz/block/trailblaze/agent/TrailblazeRunner.kt b/trailblaze-agent/src/main/java/xyz/block/trailblaze/agent/TrailblazeRunner.kt index 45d3ee96d..d4cec40a3 100644 --- a/trailblaze-agent/src/main/java/xyz/block/trailblaze/agent/TrailblazeRunner.kt +++ b/trailblaze-agent/src/main/java/xyz/block/trailblaze/agent/TrailblazeRunner.kt @@ -443,7 +443,7 @@ class TrailblazeRunner( * resolves to this when the caller didn't specify one, and host/on-device wiring reads it from * here so the in-process, daemon, and on-device paths agree. */ - const val DEFAULT_MAX_STEPS: Int = 50 + const val DEFAULT_MAX_STEPS: Int = 25 /** * Sliding window over the last N tool fingerprints, used by `detectActionCycleHint` diff --git a/trailblaze-agent/src/main/resources/trailblaze_system_prompt.md b/trailblaze-agent/src/main/resources/trailblaze_system_prompt.md index dc02c824a..a02326325 100644 --- a/trailblaze-agent/src/main/resources/trailblaze_system_prompt.md +++ b/trailblaze-agent/src/main/resources/trailblaze_system_prompt.md @@ -2,9 +2,9 @@ - You will be provided with the current screen state, including a text representation of the current UI hierarchy as well as a screenshot of the device. The screenshot may be marked with colored boxes containing nodeIds in the bottom right corner of each box. **UI Interaction hints:** -- If the device is currently on a loading screen or a welcome screen, then always wait for the app to finish before choosing another tool. +- After an action triggers a load or screen transition (app launch, a tap that opens a screen, a submit that shows a spinner), wait on a POSITIVE signal rather than a fixed delay. Use `waitForChange` to let the UI settle, take a fresh snapshot, then `assertVisible` a ref that is present in that snapshot. `assertVisible` only checks the current snapshot and fails immediately on a ref that has not rendered — it does NOT poll for an element to appear — so always re-snapshot after the screen settles instead of asserting a ref you have not just seen. To wait out a "Loading" spinner, use `assertNotVisibleWithText` on the loading text, which polls until that text is gone. - Always use the accessibility text when interacting with Icons on the screen. Attempting to tap on them as an individual letter or symbol will not work. - Always use the close or back icons in the app to navigate vs using the device back button. The back button should only be used if there are no other. - A text field must be focused before you can enter text into it. - A disabled button will have no effect when clicked. -- Any blank or loading screens should use the wait tool in order to provide the next valid view state. +- `wait` and `waitForChange` are settles, not fixed timers: each returns as soon as the UI goes quiet (often well under a second on a static screen) and the duration you pass is only a ceiling, not a guaranteed pause. Chaining `waitForChange` then `wait` buys nothing but a longer ceiling, and neither can wait out a change that has not started yet. There is no LLM-callable fixed sleep, so never pad a load with a timer — key on the actual state: `assertVisible` a ref after a fresh snapshot, or `assertNotVisibleWithText` to poll a "Loading" spinner out. diff --git a/trailblaze-agent/src/test/resources/baselines/composed_default_prompt.txt b/trailblaze-agent/src/test/resources/baselines/composed_default_prompt.txt index e733550a9..78686485c 100644 --- a/trailblaze-agent/src/test/resources/baselines/composed_default_prompt.txt +++ b/trailblaze-agent/src/test/resources/baselines/composed_default_prompt.txt @@ -31,9 +31,9 @@ - You will be provided with the current screen state, including a text representation of the current UI hierarchy as well as a screenshot of the device. The screenshot may be marked with colored boxes containing nodeIds in the bottom right corner of each box. **UI Interaction hints:** -- If the device is currently on a loading screen or a welcome screen, then always wait for the app to finish before choosing another tool. +- After an action triggers a load or screen transition (app launch, a tap that opens a screen, a submit that shows a spinner), wait on a POSITIVE signal rather than a fixed delay. Use `waitForChange` to let the UI settle, take a fresh snapshot, then `assertVisible` a ref that is present in that snapshot. `assertVisible` only checks the current snapshot and fails immediately on a ref that has not rendered — it does NOT poll for an element to appear — so always re-snapshot after the screen settles instead of asserting a ref you have not just seen. To wait out a "Loading" spinner, use `assertNotVisibleWithText` on the loading text, which polls until that text is gone. - Always use the accessibility text when interacting with Icons on the screen. Attempting to tap on them as an individual letter or symbol will not work. - Always use the close or back icons in the app to navigate vs using the device back button. The back button should only be used if there are no other. - A text field must be focused before you can enter text into it. - A disabled button will have no effect when clicked. -- Any blank or loading screens should use the wait tool in order to provide the next valid view state. +- `wait` and `waitForChange` are settles, not fixed timers: each returns as soon as the UI goes quiet (often well under a second on a static screen) and the duration you pass is only a ceiling, not a guaranteed pause. Chaining `waitForChange` then `wait` buys nothing but a longer ceiling, and neither can wait out a change that has not started yet. There is no LLM-callable fixed sleep, so never pad a load with a timer — key on the actual state: `assertVisible` a ref after a fresh snapshot, or `assertNotVisibleWithText` to poll a "Loading" spinner out. diff --git a/trailblaze-agent/src/test/resources/baselines/default_platform_prompt.txt b/trailblaze-agent/src/test/resources/baselines/default_platform_prompt.txt index dc02c824a..a02326325 100644 --- a/trailblaze-agent/src/test/resources/baselines/default_platform_prompt.txt +++ b/trailblaze-agent/src/test/resources/baselines/default_platform_prompt.txt @@ -2,9 +2,9 @@ - You will be provided with the current screen state, including a text representation of the current UI hierarchy as well as a screenshot of the device. The screenshot may be marked with colored boxes containing nodeIds in the bottom right corner of each box. **UI Interaction hints:** -- If the device is currently on a loading screen or a welcome screen, then always wait for the app to finish before choosing another tool. +- After an action triggers a load or screen transition (app launch, a tap that opens a screen, a submit that shows a spinner), wait on a POSITIVE signal rather than a fixed delay. Use `waitForChange` to let the UI settle, take a fresh snapshot, then `assertVisible` a ref that is present in that snapshot. `assertVisible` only checks the current snapshot and fails immediately on a ref that has not rendered — it does NOT poll for an element to appear — so always re-snapshot after the screen settles instead of asserting a ref you have not just seen. To wait out a "Loading" spinner, use `assertNotVisibleWithText` on the loading text, which polls until that text is gone. - Always use the accessibility text when interacting with Icons on the screen. Attempting to tap on them as an individual letter or symbol will not work. - Always use the close or back icons in the app to navigate vs using the device back button. The back button should only be used if there are no other. - A text field must be focused before you can enter text into it. - A disabled button will have no effect when clicked. -- Any blank or loading screens should use the wait tool in order to provide the next valid view state. +- `wait` and `waitForChange` are settles, not fixed timers: each returns as soon as the UI goes quiet (often well under a second on a static screen) and the duration you pass is only a ceiling, not a guaranteed pause. Chaining `waitForChange` then `wait` buys nothing but a longer ceiling, and neither can wait out a change that has not started yet. There is no LLM-callable fixed sleep, so never pad a load with a timer — key on the actual state: `assertVisible` a ref after a fresh snapshot, or `assertNotVisibleWithText` to poll a "Loading" spinner out. diff --git a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidTrailblazeRule.kt b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidTrailblazeRule.kt index 8facb05f1..3599adfcf 100644 --- a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidTrailblazeRule.kt +++ b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidTrailblazeRule.kt @@ -677,11 +677,9 @@ open class AndroidTrailblazeRule( if (!trailblazeYaml.hasActionableSteps(trailItems)) { val trailName = trailConfig?.title ?: trailFilePath ?: "unknown" - val trailUrl = trailConfig?.metadata?.get("testRailUrl") throw TrailblazeException( "Trail '$trailName' has no executable steps — this would be a false positive pass. " + - "Add prompts or tool steps to this trail file." + - (trailUrl?.let { " $it" } ?: ""), + "Add prompts or tool steps to this trail file.", ) } diff --git a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt index 5729dba43..56d4ca071 100644 --- a/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt +++ b/trailblaze-android/src/main/java/xyz/block/trailblaze/android/accessibility/AccessibilityDeviceManager.kt @@ -15,6 +15,7 @@ import xyz.block.trailblaze.android.InstrumentationArgUtil import xyz.block.trailblaze.api.DriverDispatch import xyz.block.trailblaze.api.DriverNodeDetail import xyz.block.trailblaze.api.ScreenState +import xyz.block.trailblaze.api.TapDispatchRoute import xyz.block.trailblaze.api.TargetTypeGuard import xyz.block.trailblaze.api.TrailblazeNode import xyz.block.trailblaze.api.TrailblazeNodeSelector @@ -224,6 +225,7 @@ class AccessibilityDeviceManager( data class ExecutionResult( val resolvedX: Int? = null, val resolvedY: Int? = null, + val dispatchRoute: TapDispatchRoute? = null, ) /** @@ -872,8 +874,13 @@ class AccessibilityDeviceManager( warnIfTapPointOccluded( unfilteredTree, result.node, center.first, center.second, action.nodeSelector.description(), ) - tapOrLongPressOnResolvedNode(result.node, center.first, center.second, action.longPress) - return ExecutionResult(resolvedX = center.first, resolvedY = center.second) + val route = + tapOrLongPressOnResolvedNode(result.node, center.first, center.second, action.longPress) + return ExecutionResult( + resolvedX = center.first, + resolvedY = center.second, + dispatchRoute = route, + ) } is TrailblazeNodeSelectorResolver.ResolveResult.MultipleMatches -> { val chosen = pickPreferredMatch(result.nodes) @@ -886,8 +893,13 @@ class AccessibilityDeviceManager( warnIfTapPointOccluded( unfilteredTree, chosen, center.first, center.second, action.nodeSelector.description(), ) - tapOrLongPressOnResolvedNode(chosen, center.first, center.second, action.longPress) - return ExecutionResult(resolvedX = center.first, resolvedY = center.second) + val route = + tapOrLongPressOnResolvedNode(chosen, center.first, center.second, action.longPress) + return ExecutionResult( + resolvedX = center.first, + resolvedY = center.second, + dispatchRoute = route, + ) } is TrailblazeNodeSelectorResolver.ResolveResult.NoMatch -> { // Element not found yet, will retry after sleep @@ -958,10 +970,11 @@ class AccessibilityDeviceManager( centerX: Int, centerY: Int, longPress: Boolean, - ) { + ): TapDispatchRoute { if (actionClickRouteDisabled()) { Console.log("[tap-route] kill-switch set, using gesture at ($centerX,$centerY)") - return tapOrLongPress(centerX, centerY, longPress) + tapOrLongPress(centerX, centerY, longPress) + return TapDispatchRoute.GESTURE } val plan = planActionClickRoute(resolvedNode, longPress) if (plan == null) { @@ -973,9 +986,10 @@ class AccessibilityDeviceManager( "[tap-route] gesture at ($centerX,$centerY) — gate declined ACTION_CLICK " + "(${describeNodeForRouteLog(resolvedNode, longPress)})", ) - return tapOrLongPress(centerX, centerY, longPress) + tapOrLongPress(centerX, centerY, longPress) + return TapDispatchRoute.GESTURE } - dispatchAndAwaitSettleBlocking { + return dispatchAndAwaitSettleBlocking { val dispatched = TrailblazeAccessibilityService.tapByActionClickOnBounds( plan.bounds.toAndroidRect(), plan.className, @@ -983,6 +997,7 @@ class AccessibilityDeviceManager( ) if (dispatched) { Console.log("[tap-route] ACTION_CLICK dispatched on ${plan.className ?: ""}") + TapDispatchRoute.ACTION_CLICK } else { // Live tree didn't carry a node matching the resolved identity (tree mutated between // resolve and dispatch, or node no longer advertises ACTION_CLICK). Fall back to the @@ -999,6 +1014,7 @@ class AccessibilityDeviceManager( TrailblazeAccessibilityService.tap(centerX, centerY), "gesture-fallback tap at ($centerX, $centerY) after an ACTION_CLICK lookup miss", ) + TapDispatchRoute.GESTURE_AFTER_ACTION_CLICK_MISS } } } @@ -1136,8 +1152,8 @@ internal fun scrollToSwipeDirection(direction: AccessibilityAction.Direction): A * respects z-order, ACTION_CLICK would bypass that and fire the hidden node directly), * - the node is not editable (EditText caret placement requires the touch offset; ACTION_CLICK * merely focuses the field without honoring it), - * - the node carries its own text or contentDescription — distinguishes interactive **leaf** - * elements (`ExploreByTouchHelper` virtual buttons that emit a per-button + * - the node carries its own text or contentDescription, **or** is checkable — distinguishes + * interactive **leaf** elements (`ExploreByTouchHelper` virtual buttons that emit a per-button * `contentDescription`, standard `