Add a sleep tool that really waits, and make wait report the time it took

.trailblaze-sync

@@ -1 +1 @@
-6e278dc4acbede2988c75123f499d10b0cf7714f
+e5cd7a49580860da9bb50f5f91ac18924d361e4b

docs/generated/external-config.md

@@ -171,7 +171,7 @@ Toolsets are declared in `trailmaps/<id>/toolsets/*.yaml`. They are pure YAML gr
 | `android_primitives` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation` | 7 |
 | `compose_core` | No | `compose` | 6 |
 | `compose_verification` | No | `compose` | 3 |
-| `core_interaction` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation`, `ios-axe`, `ios-host` | 20 |
+| `core_interaction` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation`, `ios-axe`, `ios-host` | 21 |
 | `memory` | No | `all drivers` | 8 |
 | `meta` | Yes | `all drivers` | 1 |
 | `mobile_primitives` | Yes | `android-ondevice-accessibility`, `android-ondevice-instrumentation`, `ios-axe`, `ios-host` | 5 |

docs/generated/functions/custom/wait.md

@@ -4,8 +4,11 @@
 
 # `wait`
 
-Wait for a specified amount of time. Use when you see a loading screen — prefer this over
-pressing the back button.
+Settle on a loading screen: block until the UI goes quiet, up to a ceiling. This returns as soon
+as the UI is idle, so on an already-static screen it returns almost immediately rather than
+waiting the full time — it is a ceiling, not a duration. Use when you see a loading screen —
+prefer this over pressing the back button. If you are waiting for something specific to appear,
+assert on that element instead: a quiet UI does not mean the thing you expect has arrived.
 
 ## Source
 
@@ -23,7 +26,7 @@ pressing the back button.
 ### Optional parameters
 
 - `timeToWaitInSeconds` — `Integer`
-  Unit: seconds. Default Value: 5 seconds.
+  Ceiling on how long to settle for, in seconds — not a guaranteed duration. Default Value: 5 seconds.
 
 ## Output
 

trailblaze-android/src/main/java/xyz/block/trailblaze/android/AndroidTrailblazeRule.kt

@@ -677,11 +677,9 @@ open class AndroidTrailblazeRule(
 
     if (!trailblazeYaml.hasActionableSteps(trailItems)) {
       val trailName = trailConfig?.title ?: trailFilePath ?: "unknown"
-      val trailUrl = trailConfig?.metadata?.get("testRailUrl")
       throw TrailblazeException(
         "Trail '$trailName' has no executable steps — this would be a false positive pass. " +
-          "Add prompts or tool steps to this trail file." +
-          (trailUrl?.let { " $it" } ?: ""),
+          "Add prompts or tool steps to this trail file.",
       )
     }
 

trailblaze-common/src/commonMain/resources/trails/config/trailmaps/trailblaze/tools/sleep.tool.yaml

@@ -0,0 +1,2 @@
+id: sleep
+class: xyz.block.trailblaze.toolcalls.commands.SleepTrailblazeTool

trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/SleepTrailblazeTool.kt

@@ -0,0 +1,82 @@
+package xyz.block.trailblaze.toolcalls.commands
+
+import ai.koog.agents.core.tools.annotations.LLMDescription
+import kotlin.time.TimeSource
+import kotlinx.coroutines.delay
+import kotlinx.serialization.Serializable
+import xyz.block.trailblaze.toolcalls.HostLocalExecutableTrailblazeTool
+import xyz.block.trailblaze.toolcalls.TrailblazeToolClass
+import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext
+import xyz.block.trailblaze.toolcalls.TrailblazeToolResult
+
+/**
+ * The only tool that consumes wall-clock time unconditionally.
+ *
+ * `wait` and `waitForChange` are settles: they return the moment the UI is quiet, so on a static
+ * screen both come back in ~150ms no matter what duration was asked for. That makes them useless
+ * for the case this tool exists for — letting something happen OFF-screen (a server-side value
+ * propagating) before the trail navigates to the screen that reads it.
+ *
+ * Implemented as a [HostLocalExecutableTrailblazeTool] so `BaseTrailblazeAgent.runTrailblazeTools`
+ * runs it in-process for EVERY agent, before driver-specific dispatch. That is what makes the
+ * duration driver-independent: there is no path on which it can degrade to a Maestro
+ * `WaitForAnimationToEnd`, the defect this tool exists to avoid.
+ *
+ * On the trail path it is also RPC-free. The MCP `step` / `trailblaze tool` path is not:
+ * `TrailblazeMcpBridgeImpl.executeHostLocalTool` handles only Playwright, so on an on-device driver
+ * the call goes over RPC and is bounded by `OnDeviceRpcTimeouts.HANDLER_AWAIT_CAP_MS` (15 min). The
+ * sleep still runs to completion there — [MAX_DURATION_MS] keeps every legal duration well inside
+ * that ceiling — but the transport cap does exist on that path.
+ *
+ * `surfaceToLlm = false`: a fixed sleep is correct when an author knows about an off-screen
+ * dependency, and almost always wrong when picked autonomously — the agent should assert on the
+ * element it expects instead. Hidden from the LLM toolbox; still callable from hand-authored
+ * trail YAML and scripted tools, and still recorded.
+ */
+@Serializable
+@TrailblazeToolClass("sleep", surfaceToLlm = false)
+@LLMDescription(
+  """
+Block for a fixed amount of wall-clock time, always consuming the full duration regardless of
+what the UI is doing. Use only to let something happen off-screen (e.g. a server-side value
+propagating) before navigating to the screen that reads it. To wait for something visible,
+assert on that element instead — and to wait for the UI to settle, use waitForChange.
+    """,
+)
+data class SleepTrailblazeTool(
+  @LLMDescription("How long to sleep, in milliseconds. Must be between 100 and 300000 (5 minutes). Default 5000.")
+  val durationMs: Long = 5_000,
+) : HostLocalExecutableTrailblazeTool {
+
+  override val advertisedToolName: String get() = "sleep"
+
+  override suspend fun execute(
+    toolExecutionContext: TrailblazeToolExecutionContext,
+  ): TrailblazeToolResult {
+    // Out of range is an error, not a clamp: silently coercing a too-long sleep would return
+    // before the requested duration while reporting success — the defect this tool removes.
+    if (durationMs !in MIN_DURATION_MS..MAX_DURATION_MS) {
+      return TrailblazeToolResult.Error.ExceptionThrown(
+        errorMessage = "sleep requires durationMs between $MIN_DURATION_MS and $MAX_DURATION_MS, but was $durationMs",
+        command = this,
+      )
+    }
+    val startMark = TimeSource.Monotonic.markNow()
+    delay(durationMs)
+    val elapsedMs = startMark.elapsedNow().inWholeMilliseconds
+    return TrailblazeToolResult.Success(
+      message = "Slept ${elapsedMs}ms (requested ${durationMs}ms)",
+    )
+  }
+
+  companion object {
+    /** Anything shorter is a no-op that reads as a wait, and is almost always a units slip (`5` meaning 5 seconds). */
+    const val MIN_DURATION_MS = 100L
+
+    /**
+     * Held under the 10-minute run-poll inactivity window (`DaemonClient.RUN_POLL_TIMEOUT_MS`): a
+     * host-local sleep emits no progress, so a longer one is indistinguishable from a wedged run.
+     */
+    const val MAX_DURATION_MS = 300_000L
+  }
+}

trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/WaitForChangeTrailblazeTool.kt

@@ -1,6 +1,7 @@
 package xyz.block.trailblaze.toolcalls.commands
 
 import ai.koog.agents.core.tools.annotations.LLMDescription
+import kotlin.time.TimeSource
 import kotlinx.serialization.Serializable
 import maestro.orchestra.WaitForAnimationToEndCommand
 import xyz.block.trailblaze.toolcalls.ExecutableTrailblazeTool
@@ -41,19 +42,24 @@ data class WaitForChangeTrailblazeTool(
     )
     if (driverResult != null) return driverResult
 
-    // Unsupported driver (iOS / host / non-accessibility Android): degrade to a timed wait so
-    // the caller still gets a settle pause rather than a hard failure.
-    Console.log("waitForChange: driver has no change detection, falling back to a ${timeoutMs}ms timed wait")
+    // Unsupported driver (iOS / host / non-accessibility Android): degrade to an animation-end
+    // settle so the caller still gets a pause rather than a hard failure.
+    Console.log("waitForChange: driver has no change detection, falling back to a settle with a ${timeoutMs}ms ceiling")
     val agentForFallback = agent
       ?: return TrailblazeToolResult.Error.ExceptionThrown(
         errorMessage = "waitForChange could not run: no agent available to perform the wait",
       )
+    val startMark = TimeSource.Monotonic.markNow()
     val result = agentForFallback.runMaestroCommands(
       maestroCommands = listOf(WaitForAnimationToEndCommand(timeout = timeoutMs.toString())),
       traceId = toolExecutionContext.traceId,
     )
     if (result is TrailblazeToolResult.Success) {
-      return TrailblazeToolResult.Success(message = "waitForChange degraded to a ${timeoutMs}ms timed wait")
+      // `timeoutMs` is the ceiling the settle may take, never the time it spent (#5279).
+      val elapsedMs = startMark.elapsedNow().inWholeMilliseconds
+      return TrailblazeToolResult.Success(
+        message = "waitForChange degraded to a settle, which returned after ${elapsedMs}ms (ceiling ${timeoutMs}ms)",
+      )
     }
     return result
   }

trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/WaitForIdleSyncTrailblazeTool.kt

@@ -1,6 +1,7 @@
 package xyz.block.trailblaze.toolcalls.commands
 
 import ai.koog.agents.core.tools.annotations.LLMDescription
+import kotlin.time.TimeSource
 import kotlinx.serialization.Serializable
 import maestro.orchestra.Command
 import maestro.orchestra.WaitForAnimationToEndCommand
@@ -14,12 +15,15 @@ import xyz.block.trailblaze.toolcalls.isSuccess
 @TrailblazeToolClass("wait")
 @LLMDescription(
   """
-Wait for a specified amount of time. Use when you see a loading screen — prefer this over
-pressing the back button.
+Settle on a loading screen: block until the UI goes quiet, up to a ceiling. This returns as soon
+as the UI is idle, so on an already-static screen it returns almost immediately rather than
+waiting the full time — it is a ceiling, not a duration. Use when you see a loading screen —
+prefer this over pressing the back button. If you are waiting for something specific to appear,
+assert on that element instead: a quiet UI does not mean the thing you expect has arrived.
     """,
 )
 data class WaitForIdleSyncTrailblazeTool(
-  @LLMDescription("Unit: seconds. Default Value: 5 seconds.")
+  @LLMDescription("Ceiling on how long to settle for, in seconds — not a guaranteed duration. Default Value: 5 seconds.")
   val timeToWaitInSeconds: Int = 5,
 ) : MapsToMaestroCommands() {
   override fun toMaestroCommands(): List<Command> = listOf(
@@ -31,9 +35,16 @@ data class WaitForIdleSyncTrailblazeTool(
   override suspend fun execute(
     toolExecutionContext: TrailblazeToolExecutionContext,
   ): TrailblazeToolResult {
+    val startMark = TimeSource.Monotonic.markNow()
     val result = super.execute(toolExecutionContext)
     if (result.isSuccess()) {
-      return TrailblazeToolResult.Success(message = "Waited $timeToWaitInSeconds seconds")
+      // Report what actually elapsed. The old message stated the requested ceiling, which on a
+      // static screen overstated the real settle by ~30x and read in the log as a wait that
+      // had happened (#5279).
+      val elapsedMs = startMark.elapsedNow().inWholeMilliseconds
+      return TrailblazeToolResult.Success(
+        message = "Settled after ${elapsedMs}ms (ceiling ${timeToWaitInSeconds}s)",
+      )
     }
     return result
   }

trailblaze-common/src/jvmAndAndroid/kotlin/xyz/block/trailblaze/toolcalls/commands/memory/AssertMathTrailblazeTool.kt

@@ -32,7 +32,14 @@ data class AssertMathTrailblazeTool(
    * throws is surfaced. Reach for this only when a `[[prompt]]` read is subject to eventual
    * consistency (a value the app back-fills a beat after the action) — it bounds WHEN the read is
    * taken, never WHAT is asserted. Each retry re-reads through [ElementComparator.getElementValue],
-   * which captures a fresh screen, so the poll observes the updated value rather than a stale one.
+   * which captures the screen afresh.
+   *
+   * LIMITATION: a fresh capture is not a fresh fetch. This re-reads the rendered screen, not the
+   * underlying data, so it can only observe values the app itself updates in place. For a view the
+   * app populates once and refreshes only on navigation, every attempt re-reads the same stale
+   * number and no bound helps — polling cannot substitute for re-navigating. Observed against a
+   * balance screen of that shape: every capture across a minute of polling returned the same
+   * pre-update value, and shorter bounds failed identically.
    */
   val timeoutMs: Long? = null,
   /**
@@ -66,7 +73,8 @@ data class AssertMathTrailblazeTool(
    * A null [timeoutMs] runs [evaluateOnce] exactly once (unchanged single-shot behavior); a set
    * [timeoutMs] re-runs [evaluateOnce] each attempt — and because that path re-reads via
    * [ElementComparator.getElementValue] (a fresh screen capture per call), every retry observes the
-   * current screen instead of a cached snapshot.
+   * current screen instead of a cached snapshot. What the current screen *shows* is the app's
+   * business; see the LIMITATION on [timeoutMs].
    */
   internal fun executeWithClock(
     elementComparator: ElementComparator,

trailblaze-common/src/jvmAndAndroidTest/kotlin/xyz/block/trailblaze/toolcalls/commands/SleepTrailblazeToolTest.kt

@@ -0,0 +1,143 @@
+package xyz.block.trailblaze.toolcalls.commands
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+import kotlin.time.TimeSource
+import kotlinx.coroutines.runBlocking
+import kotlinx.datetime.Clock
+import xyz.block.trailblaze.AgentMemory
+import xyz.block.trailblaze.devices.TrailblazeDeviceId
+import xyz.block.trailblaze.devices.TrailblazeDeviceInfo
+import xyz.block.trailblaze.devices.TrailblazeDevicePlatform
+import xyz.block.trailblaze.devices.TrailblazeDriverType
+import xyz.block.trailblaze.logs.client.TrailblazeLogger
+import xyz.block.trailblaze.logs.client.TrailblazeSession
+import xyz.block.trailblaze.logs.client.TrailblazeSessionProvider
+import xyz.block.trailblaze.logs.model.SessionId
+import xyz.block.trailblaze.toolcalls.TrailblazeToolExecutionContext
+import xyz.block.trailblaze.toolcalls.TrailblazeToolResult
+
+/**
+ * `sleep` exists because every other wait in the framework is a settle that returns as soon as
+ * the UI is quiet (#5279). So the assertion that matters is the wall-clock FLOOR: a test that
+ * only asserted `Success` would pass against the very settle-based implementation this tool was
+ * added to replace.
+ */
+class SleepTrailblazeToolTest {
+
+  private val sleepMs = 400L
+
+  @Test
+  fun `consumes the full requested duration instead of returning early`() {
+    val outerMark = TimeSource.Monotonic.markNow()
+    val result = runBlocking { SleepTrailblazeTool(durationMs = sleepMs).execute(context()) }
+    val outerElapsedMs = outerMark.elapsedNow().inWholeMilliseconds
+
+    assertTrue(result is TrailblazeToolResult.Success, "expected Success but was $result")
+    // The load-bearing assertion. A settle-based implementation returns in ~150ms on a static
+    // screen; `WaitForAnimationToEnd` against a null driver returns in ~0ms. Both fail here.
+    assertTrue(
+      outerElapsedMs >= sleepMs,
+      "sleep(${sleepMs}ms) returned after only ${outerElapsedMs}ms — it did not consume wall-clock time",
+    )
+  }
+
+  @Test
+  fun `success message reports measured elapsed time, bounded by an independent measurement`() {
+    val outerMark = TimeSource.Monotonic.markNow()
+    val result = runBlocking { SleepTrailblazeTool(durationMs = sleepMs).execute(context()) }
+    val outerElapsedMs = outerMark.elapsedNow().inWholeMilliseconds
+
+    val message = (result as TrailblazeToolResult.Success).message
+    val reportedMs = Regex("^Slept (\\d+)ms").find(message.orEmpty())?.groupValues?.get(1)?.toLong()
+      ?: error("message did not report an elapsed time in the documented shape: $message")
+
+    // Brackets the reported number between the floor it must clear and a measurement taken
+    // strictly outside it — so the message cannot be an arbitrary constant, and in particular
+    // cannot be a duration that was never actually spent.
+    assertTrue(reportedMs >= sleepMs, "reported ${reportedMs}ms is below the requested ${sleepMs}ms")
+    assertTrue(
+      reportedMs <= outerElapsedMs,
+      "reported ${reportedMs}ms exceeds the ${outerElapsedMs}ms measured around the call, so it is not a real measurement",
+    )
+    assertTrue(
+      message.orEmpty().contains("requested ${sleepMs}ms"),
+      "message should also state what was requested, was: $message",
+    )
+  }
+
+  @Test
+  fun `negative duration fails loudly rather than silently returning success`() {
+    // `delay` clamps a negative duration to zero, which would make this tool a silent no-op
+    // reporting success — the exact defect class it exists to remove.
+    val result = runBlocking { SleepTrailblazeTool(durationMs = -1).execute(context()) }
+    assertTrue(
+      result is TrailblazeToolResult.Error,
+      "a negative duration must be an error, but was $result",
+    )
+  }
+
+  @Test
+  fun `duration below the floor fails loudly rather than passing as a no-op`() {
+    // `durationMs: 5` is a units slip (5 seconds meant), and succeeds as a 5ms non-wait.
+    val result = runBlocking {
+      SleepTrailblazeTool(durationMs = SleepTrailblazeTool.MIN_DURATION_MS - 1).execute(context())
+    }
+    assertTrue(result is TrailblazeToolResult.Error, "a sub-floor duration must be an error, but was $result")
+  }
+
+  @Test
+  fun `duration above the cap fails loudly instead of being silently clamped`() {
+    // Clamping would return before the requested duration while reporting success, which is the
+    // early-return defect this tool exists to remove — so the cap must reject, not coerce.
+    val requested = SleepTrailblazeTool.MAX_DURATION_MS + 1
+    val outerMark = TimeSource.Monotonic.markNow()
+    val result = runBlocking { SleepTrailblazeTool(durationMs = requested).execute(context()) }
+    val outerElapsedMs = outerMark.elapsedNow().inWholeMilliseconds
+
+    assertTrue(result is TrailblazeToolResult.Error, "an over-cap duration must be an error, but was $result")
+    assertTrue(
+      outerElapsedMs < SleepTrailblazeTool.MAX_DURATION_MS,
+      "rejection took ${outerElapsedMs}ms — it slept before failing",
+    )
+  }
+
+  @Test
+  fun `bounds bracket the default duration`() {
+    assertTrue(
+      SleepTrailblazeTool().durationMs in SleepTrailblazeTool.MIN_DURATION_MS..SleepTrailblazeTool.MAX_DURATION_MS,
+      "the default duration is outside the range the tool accepts",
+    )
+    // The cap must stay under the 10-minute run-poll inactivity window, which a silent
+    // host-local sleep cannot reset.
+    assertTrue(
+      SleepTrailblazeTool.MAX_DURATION_MS < 10 * 60 * 1000L,
+      "the cap allows a sleep long enough to trip the run-poll inactivity watchdog",
+    )
+  }
+
+  @Test
+  fun `default duration is five seconds`() {
+    assertEquals(5_000L, SleepTrailblazeTool().durationMs)
+  }
+
+  private fun context(): TrailblazeToolExecutionContext = TrailblazeToolExecutionContext(
+    screenState = null,
+    traceId = null,
+    trailblazeDeviceInfo = TrailblazeDeviceInfo(
+      trailblazeDeviceId = TrailblazeDeviceId(
+        instanceId = "test-device",
+        trailblazeDevicePlatform = TrailblazeDevicePlatform.ANDROID,
+      ),
+      trailblazeDriverType = TrailblazeDriverType.ANDROID_ONDEVICE_INSTRUMENTATION,
+      widthPixels = 1080,
+      heightPixels = 1920,
+    ),
+    sessionProvider = TrailblazeSessionProvider {
+      TrailblazeSession(sessionId = SessionId("test-session"), startTime = Clock.System.now())
+    },
+    trailblazeLogger = TrailblazeLogger.createNoOp(),
+    memory = AgentMemory(),
+  )
+}