diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index d54cab9b6d..201ced92c6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -71,6 +71,7 @@ @interface AgentDeviceRunnerViewController : UIViewController // took focus from an input that has stopped answering the accessibility query that found it. @property(nonatomic, strong) UILabel *textEntryFocusWitness; @property(nonatomic, assign) NSUInteger textEntryWriteBacks; +@property(nonatomic, assign) NSUInteger textEntryTotalEdits; @property(nonatomic, copy, nullable) NSString *textEntryRenderedValue; @property(nonatomic, assign) NSTimeInterval textEntryLastEditTime; @property(nonatomic, assign) NSTimeInterval textEntryBurstStartTime; @@ -97,6 +98,122 @@ - (NSString *)accessibilityValue { return [NSString stringWithFormat:@"%lu of 6 digits", (unsigned long)self.text.length]; } @end + +// A field that reformats its own text on every edit into `## ### ## ##` digit groups, the way a +// digit-grouping formatter in a Flutter or React Native input does. The accessibility value is +// what the user sees — the formatted string — so it never equals the digits a `fill` sent (#2634). +// The field owns the delegate it installs, keeping the formatter out of the view controller's +// own delegate callbacks. +@interface AgentDeviceDigitGroupingTextField : UITextField +@end + +@interface AgentDeviceCurrencyTextField : UITextField +@property(nonatomic, strong) UILabel *entryStatus; +@property(nonatomic, assign) NSUInteger insertedDigits; +@end + +@interface AgentDeviceSubmitClearingTextField : UITextField +@property(nonatomic, strong) UILabel *entryStatus; +@property(nonatomic, assign) NSUInteger submits; +@end + +static NSString *AgentDeviceFormatGroupedDigits(NSString *raw) { + NSMutableString *digits = [NSMutableString string]; + NSCharacterSet *nonDigits = [[NSCharacterSet decimalDigitCharacterSet] invertedSet]; + for (NSString *chunk in [raw componentsSeparatedByCharactersInSet:nonDigits]) { + [digits appendString:chunk]; + } + NSMutableArray *groupWidths = [NSMutableArray arrayWithArray:@[@2, @3, @2, @2]]; + NSMutableString *formatted = [NSMutableString string]; + NSUInteger cursor = 0; + for (NSNumber *width in groupWidths) { + if (cursor >= digits.length) { + break; + } + if (formatted.length > 0) { + [formatted appendString:@" "]; + } + NSUInteger end = MIN(cursor + width.unsignedIntegerValue, digits.length); + [formatted appendString:[digits substringWithRange:NSMakeRange(cursor, end - cursor)]]; + cursor = end; + } + // Digits past the last group keep grouping by threes rather than vanishing from the field. + while (cursor < digits.length) { + NSUInteger end = MIN(cursor + 3, digits.length); + [formatted appendString:[NSString stringWithFormat:@" %@", [digits substringWithRange:NSMakeRange(cursor, end - cursor)]]]; + cursor = end; + } + return formatted; +} + +static NSUInteger AgentDeviceDigitCount(NSString *text, NSUInteger limit) { + NSUInteger count = 0; + for (NSUInteger index = 0; index < limit && index < text.length; index++) { + if ([NSCharacterSet.decimalDigitCharacterSet characterIsMember:[text characterAtIndex:index]]) { + count += 1; + } + } + return count; +} + +@implementation AgentDeviceDigitGroupingTextField +- (BOOL)textField:(UITextField *)textField + shouldChangeCharactersInRange:(NSRange)range + replacementString:(NSString *)string { + NSString *combined = [textField.text stringByReplacingCharactersInRange:range withString:string]; + NSString *formatted = AgentDeviceFormatGroupedDigits(combined); + textField.text = formatted; + // Assigning `text` drops the caret at the end, which a real grouping formatter does not do: + // a mid-string edit must keep editing where the user put the caret. The digits before the + // caret survive the reformat unchanged, so the caret belongs before the digit that follows + // them — past the separators the grouping inserted behind them — and at the end when it + // followed all of them. Typed digits count as before the caret; a delete does not move it. + NSUInteger digitsBeforeCaret = + AgentDeviceDigitCount(textField.text, range.location) + AgentDeviceDigitCount(string, string.length); + NSUInteger caretOffset = formatted.length; + NSUInteger seen = 0; + for (NSUInteger index = 0; index < formatted.length; index++) { + if (![NSCharacterSet.decimalDigitCharacterSet characterIsMember:[formatted characterAtIndex:index]]) { + continue; + } + seen += 1; + if (seen == digitsBeforeCaret + 1) { + caretOffset = index; + break; + } + } + textField.selectedTextRange = [textField textRangeFromPosition:[textField positionFromPosition:textField.beginningOfDocument offset:caretOffset] + toPosition:[textField positionFromPosition:textField.beginningOfDocument offset:caretOffset]]; + return NO; +} +@end + +@implementation AgentDeviceCurrencyTextField +- (BOOL)textField:(UITextField *)textField + shouldChangeCharactersInRange:(NSRange)range + replacementString:(NSString *)string { + NSString *combined = [textField.text stringByReplacingCharactersInRange:range withString:string]; + NSMutableString *digits = [NSMutableString string]; + for (NSString *part in [combined componentsSeparatedByCharactersInSet:NSCharacterSet.decimalDigitCharacterSet.invertedSet]) { + [digits appendString:part]; + } + unsigned long long cents = digits.longLongValue; + textField.text = [NSString stringWithFormat:@"$%llu.%02llu", cents / 100, cents % 100]; + self.insertedDigits += AgentDeviceDigitCount(string, string.length); + self.entryStatus.text = [NSString stringWithFormat:@"%lu", (unsigned long)self.insertedDigits]; + textField.selectedTextRange = [textField textRangeFromPosition:textField.endOfDocument toPosition:textField.endOfDocument]; + return NO; +} +@end + +@implementation AgentDeviceSubmitClearingTextField +- (BOOL)textFieldShouldReturn:(UITextField *)textField { + self.submits += 1; + self.entryStatus.text = [NSString stringWithFormat:@"%lu", (unsigned long)self.submits]; + textField.text = @""; + return YES; +} +@end #endif @implementation AgentDeviceRunnerViewController @@ -202,11 +319,12 @@ - (void)updateAlertActionStatus { - (void)updateTextEntryWriteBackStatus { NSTimeInterval burstSpan = self.textEntryLastEditTime - self.textEntryBurstStartTime; self.textEntryWriteBackStatus.text = [NSString - stringWithFormat:@"write-backs=%lu burst-edits=%lu burst-ms=%lu min-gap-ms=%lu", + stringWithFormat:@"write-backs=%lu burst-edits=%lu burst-ms=%lu min-gap-ms=%lu total-edits=%lu", (unsigned long)self.textEntryWriteBacks, (unsigned long)self.textEntryBurstEdits, (unsigned long)llround(burstSpan * 1000), - (unsigned long)llround(self.textEntryBurstMinGap * 1000)]; + (unsigned long)llround(self.textEntryBurstMinGap * 1000), + (unsigned long)self.textEntryTotalEdits]; } - (void)presentAlertFixtureReplacement:(BOOL)replacement { @@ -292,6 +410,7 @@ - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { // typed faster than the app renders loses the characters that arrived while a render was in // flight, and the field settles stable short of the request. if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) { + self.textEntryTotalEdits += 1; NSTimeInterval now = NSProcessInfo.processInfo.systemUptime; NSTimeInterval gap = now - self.textEntryLastEditTime; BOOL overtookARender = self.textEntryRenderedValue != nil && gap < self.textEntryAcknowledgeWindowSeconds; @@ -408,8 +527,28 @@ - (void)viewDidLoad { if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) { BOOL digitCountValue = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-count-value"]; - UITextField *textField = - digitCountValue ? [[AgentDeviceDigitCountTextField alloc] init] : [[UITextField alloc] init]; + BOOL digitGroupingValue = + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-digit-grouping-value"]; + BOOL currencyValue = + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-currency-value"]; + BOOL submitClears = + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-submit-clears"]; + UITextField *textField; + if (digitCountValue) { + textField = [[AgentDeviceDigitCountTextField alloc] init]; + } else if (digitGroupingValue) { + textField = [[AgentDeviceDigitGroupingTextField alloc] init]; + } else if (currencyValue) { + textField = [[AgentDeviceCurrencyTextField alloc] init]; + textField.text = @"$0.00"; + } else if (submitClears) { + textField = [[AgentDeviceSubmitClearingTextField alloc] init]; + } else { + textField = [[UITextField alloc] init]; + } + if (digitGroupingValue || currencyValue || submitClears) { + textField.delegate = (id)textField; + } if (![NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unnamed-input"]) { textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; } @@ -434,6 +573,22 @@ - (void)viewDidLoad { [textField.widthAnchor constraintEqualToConstant:240], [textField.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], ]]; + if (currencyValue || submitClears) { + UILabel *entryStatus = [[UILabel alloc] init]; + entryStatus.accessibilityIdentifier = @"agent-device-text-entry-events"; + entryStatus.text = @"0"; + entryStatus.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:entryStatus]; + [NSLayoutConstraint activateConstraints:@[ + [entryStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [entryStatus.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:12], + ]]; + if (currencyValue) { + ((AgentDeviceCurrencyTextField *)textField).entryStatus = entryStatus; + } else { + ((AgentDeviceSubmitClearingTextField *)textField).entryStatus = entryStatus; + } + } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-unqueryable-on-focus"]) { self.textEntryFocusWitness = [[UILabel alloc] init]; self.textEntryFocusWitness.accessibilityIdentifier = @"agent-device-text-entry-focus"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 2753a40007..824f1f17fc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -522,11 +522,8 @@ extension RunnerTests { /// `awaitSynthesizedReplacementCommit` binds the real XCUI reads). The budget defaults to the /// shipped one, so only a test that is asking about time has to name it. /// - /// Only an exact match settles. A value with a hole in the middle ("ada@example" -> "aexample") - /// is the corruption this wait exists to catch, and `.replacement` mode has no formatter or - /// autocomplete carve-out: `isRepairableTextEntryMismatch` (RunnerTests+TextTyping.swift) treats - /// every `.replacement` mismatch as repairable, because `fill` owns the whole field via - /// select-all. A settled non-match is therefore always the wait's failure case. + /// This route has no element-bound baseline for unconfirmed evidence. Only exact read-back + /// confirms a commit; prefix progress extends the bounded wait without confirming the value. static func awaitSynthesizedReplacementCommitOutcome( expectedText: String, placeholder: String?, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 7b369fab00..1c9f5646e4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -63,10 +63,8 @@ extension RunnerTests { static let pollInterval: TimeInterval = 0.02 static let warmupValueTimeout: TimeInterval = 0.4 static let verificationStabilityWindow: TimeInterval = 0.2 - /// How long a replacement value may take to move off its baseline, and then how long an unechoed - /// value may keep changing before it must hold for `verificationStabilityWindow`; the app renders - /// the last characters' summary late. - static let unconfirmedSettleCeiling: TimeInterval = 1.0 + /// Initial replacement observation budget; a late first read gets one stability window. + static let replacementSettleCeiling: TimeInterval = 1.0 /// How long the commit wait tolerates seeing NO further progress toward the expected value. /// Numerically the flat deadline this replaced, so a pipeline that delivers nothing is /// condemned at exactly the same instant it always was (see `SynthesizedCommitDeadline`). diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift index 411e68e22b..f54b22ef6e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift @@ -1,6 +1,6 @@ import XCTest -// What a replacement's read-back can prove when the field's value does not echo the typed text. +// Target-bound observations and the replacement read-back policy. // The wire shape mirrors the cross-platform `FillUnconfirmedVerification` in // packages/contracts/src/fill-evidence.ts. extension RunnerTests { @@ -39,41 +39,74 @@ extension RunnerTests { let target: TextEntryElementIdentity } - /// Whether `observed` can be a degraded copy of the residual `baseline` plus the request: it is in - /// order inside baseline + request (dropped characters), or it contains the request in order and - /// the baseline did not already (residual text), or it contains baseline + request in order. A - /// value related to the entry in none of these ways, such as an OTP field announcing - /// "6 of 6 digits", is the app's own representation, so retyping cannot make it match. - static func textEntryValueEchoes(observed: String, expected: String, baseline: String) -> Bool { - let request = textEntryRequestWithoutSubmitKeys(expected) - let residualAndRequest = baseline + request - return isOrderedSubsequence(observed, of: residualAndRequest) - || (isOrderedSubsequence(request, of: observed) && !isOrderedSubsequence(request, of: baseline)) - || isOrderedSubsequence(residualAndRequest, of: observed) + struct ReplacementTextEntryConfirmation { + private let requested: String + private let baseline: TextEntryObservation? + private var deadline: Date + private var latest: TextEntryObservation? + private var stableSince: Date + + init(requested: String, baseline: TextEntryObservation?, startedAt: Date) { + self.requested = requested + self.baseline = baseline + deadline = startedAt.addingTimeInterval(TextEntryTiming.replacementSettleCeiling) + stableSince = startedAt + } + + mutating func observe(_ observed: TextEntryObservation, at sampledAt: Date) -> TextEntryResult? { + // A late first read still needs one stability window. + if latest == nil { + deadline = max(deadline, sampledAt.addingTimeInterval(TextEntryTiming.verificationStabilityWindow)) + } + if latest.map({ observed.isSettled(with: $0) }) != true { stableSince = sampledAt } + latest = observed + let result = RunnerTests.replacementTextEntryResult(requested: requested, baseline: baseline, observed: observed) + let settled = sampledAt.timeIntervalSince(stableSince) >= TextEntryTiming.verificationStabilityWindow + let moved = baseline.map { !observed.isSettled(with: $0) } ?? true + if settled && (moved || result.verified != false || sampledAt >= deadline) { + return result + } + if sampledAt >= deadline { + return TextEntryResult( + verified: nil, repaired: false, expectedText: requested, observedText: observed.value, + failure: .commitNotObserved + ) + } + return nil + } } - /// Classifies a replacement whose read-back never matched. The entry is unconfirmed, not failed, - /// only when the same element's value moved off its pre-entry baseline to one that does not echo - /// the request; every other mismatch stays a failure. - static func unconfirmedTextEntryEvidence( + /// Literal read-back policy; a mismatch does not establish why the app changed the text. + static func replacementTextEntryResult( requested: String, baseline: TextEntryObservation?, observed: TextEntryObservation? - ) -> TextEntryUnconfirmedEvidence? { - guard !textEntryRequestWithoutSubmitKeys(requested).isEmpty, - let baseline, - let observed, - baseline.identity.isSameElement(as: observed.identity), - observed.value != baseline.value, - !textEntryValueEchoes(observed: observed.value, expected: requested, baseline: baseline.value) - else { - return nil + ) -> TextEntryResult { + func result(_ verified: Bool?) -> TextEntryResult { + TextEntryResult(verified: verified, repaired: false, expectedText: requested, observedText: observed?.value) + } + guard let observed else { return result(nil) } + if let baseline, !baseline.identity.isSameElement(as: observed.identity) { + return result(false) } - return TextEntryUnconfirmedEvidence( - requested: requested, - before: baseline.value, - after: observed.value, - target: observed.identity + let request = observed.identity.elementType == elementTypeNamesByRawValue[XCUIElement.ElementType.textView.rawValue] + ? requested : textEntryRequestWithoutSubmitKeys(requested) + if observed.value == requested || observed.value == request { return result(true) } + // Any non-matching single-line submit result remains unverified, even a partial value. + if request != requested { return result(nil) } + guard !request.isEmpty, let baseline else { return result(false) } + if isOrderedSubsequence(observed.value, of: baseline.value + request) { return result(false) } + return TextEntryResult( + verified: nil, + repaired: false, + expectedText: requested, + observedText: observed.value, + unconfirmed: TextEntryUnconfirmedEvidence( + requested: requested, + before: baseline.value, + after: observed.value, + target: observed.identity + ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index ec88d4bbd8..e3db5aedc3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -342,13 +342,12 @@ extension RunnerTests { ) } let verifyStartedAt = Date() - var result = verifyTextEntryWithRepairIfNeeded( - app: app, - target: activeTarget, - expectedText: expectedText, - repairMode: repairMode, - baseline: entryBaseline - ) + var result: TextEntryResult + if repairMode == .replacement { + result = observeReplacementTextEntry(app: app, target: activeTarget, requested: text, baseline: entryBaseline) + } else { + result = verifyAppendTextEntryWithRepairIfNeeded(app: app, target: activeTarget, expectedText: expectedText) + } logTextEntryPhase( commandId: commandId, phase: "verify", @@ -389,12 +388,39 @@ extension RunnerTests { } } - private func verifyTextEntryWithRepairIfNeeded( + private func observeReplacementTextEntry( app: XCUIApplication, target: TextEntryTarget, - expectedText: String?, - repairMode: TextTypingRepairMode, + requested: String, baseline: TextEntryObservation? + ) -> TextEntryResult { + var confirmation = ReplacementTextEntryConfirmation(requested: requested, baseline: baseline, startedAt: Date()) + while true { + guard let element = resolveTextEntryElement(app: app, target: target) else { + let failure: TextEntryFailure? = target.boundIdentity != nil && !boundTextEntryInputIsGone(app: app, target: target) + ? .commitNotObserved : nil + if target.boundIdentity != nil && failure == nil { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY") + } + return TextEntryResult(verified: nil, repaired: false, expectedText: requested, observedText: nil, failure: failure) + } + guard let observed = textEntryObservation(for: element) else { + return TextEntryResult(verified: nil, repaired: false, expectedText: requested, observedText: nil) + } + if let result = confirmation.observe(observed, at: Date()) { + if result.unconfirmed != nil { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_UNCONFIRMED expectedLength=%d observedLength=%d repaired=0", requested.count, observed.value.count) + } + return result + } + sleepFor(TextEntryTiming.pollInterval) + } + } + + private func verifyAppendTextEntryWithRepairIfNeeded( + app: XCUIApplication, + target: TextEntryTarget, + expectedText: String? ) -> TextEntryResult { if target.boundIdentity != nil, resolveTextEntryElement(app: app, target: target) == nil { guard boundTextEntryInputIsGone(app: app, target: target) else { @@ -417,27 +443,16 @@ extension RunnerTests { expectedText: expectedText, repaired: false ) - // Retyping cannot make a value that does not echo the entry match, and it would deliver the - // text twice to an app that already acted on it. - if let unconfirmed = settledUnconfirmedTextEntry( - app: app, - target: target, - result: initialResult, - baseline: baseline - ) { - return unconfirmed - } #if os(iOS) guard initialResult.verified == false, let expectedText = initialResult.expectedText else { return initialResult } - guard shouldRepairTextEntry( + guard shouldRepairAppendTextEntry( app: app, target: target, - expectedText: expectedText, - repairMode: repairMode + expectedText: expectedText ) else { return verifyTextEntry( app: app, @@ -474,76 +489,12 @@ extension RunnerTests { expectedText: expectedText, repaired: true ) - return settledUnconfirmedTextEntry( - app: app, - target: repairedTarget, - result: repairedResult, - baseline: baseline - ) ?? repairedResult + return repairedResult #else return initialResult #endif } - /// Turns a failed replacement read-back into unconfirmed evidence once the same field's value has - /// settled on one that does not echo the request; nil leaves the failure standing. - private func settledUnconfirmedTextEntry( - app: XCUIApplication, - target: TextEntryTarget, - result: TextEntryResult, - baseline: TextEntryObservation? - ) -> TextEntryResult? { - guard result.verified == false, let requested = result.expectedText, let baseline else { - return nil - } - func observe() -> TextEntryObservation? { - textEntryObservation(for: resolveTextEntryElement(app: app, target: target)) - } - func evidence(_ observed: TextEntryObservation?) -> TextEntryUnconfirmedEvidence? { - Self.unconfirmedTextEntryEvidence(requested: requested, baseline: baseline, observed: observed) - } - let moveDeadline = Date().addingTimeInterval(TextEntryTiming.unconfirmedSettleCeiling) - var latest = observe() - while let unmoved = latest, unmoved.isSettled(with: baseline), Date() < moveDeadline { - sleepFor(TextEntryTiming.pollInterval) - latest = observe() - } - guard evidence(latest) != nil else { - return nil - } - let ceiling = Date().addingTimeInterval(TextEntryTiming.unconfirmedSettleCeiling) - var stableSince = Date() - while Date().timeIntervalSince(stableSince) < TextEntryTiming.verificationStabilityWindow { - guard Date() < ceiling else { - return nil - } - sleepFor(TextEntryTiming.pollInterval) - let next = observe() - if let next, let settled = latest, next.isSettled(with: settled) { - latest = next - continue - } - latest = next - stableSince = Date() - } - guard let unconfirmed = evidence(latest) else { - return nil - } - NSLog( - "AGENT_DEVICE_RUNNER_TEXT_ENTRY_UNCONFIRMED expectedLength=%d observedLength=%d repaired=%d", - requested.count, - unconfirmed.after.count, - result.repaired ? 1 : 0 - ) - return TextEntryResult( - verified: nil, - repaired: result.repaired, - expectedText: requested, - observedText: unconfirmed.after, - unconfirmed: unconfirmed - ) - } - private func verifyTextEntry( app: XCUIApplication, target: TextEntryTarget, @@ -631,11 +582,10 @@ extension RunnerTests { } } - private func shouldRepairTextEntry( + private func shouldRepairAppendTextEntry( app: XCUIApplication, target: TextEntryTarget, - expectedText: String, - repairMode: TextTypingRepairMode + expectedText: String ) -> Bool { #if os(iOS) var latestObservedText: String? @@ -652,10 +602,9 @@ extension RunnerTests { return false } latestObservedText = observedText - if !isRepairableTextEntryMismatch( + if !isRepairableAppendTextEntryMismatch( observedText: observedText, - expectedText: expectedText, - repairMode: repairMode + expectedText: expectedText ) { return false } @@ -672,27 +621,22 @@ extension RunnerTests { ) else { return false } - return isRepairableTextEntryMismatch( + return isRepairableAppendTextEntryMismatch( observedText: latestObservedText, - expectedText: expectedText, - repairMode: repairMode + expectedText: expectedText ) #else return false #endif } - private func isRepairableTextEntryMismatch( + private func isRepairableAppendTextEntryMismatch( observedText: String, - expectedText: String, - repairMode: TextTypingRepairMode + expectedText: String ) -> Bool { guard observedText != expectedText else { return false } - if repairMode == .replacement { - return true - } return observedText.isEmpty || isLikelyDroppedCharacterTextEntryMismatch( observedText: observedText, expectedText: expectedText diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 2e84a4d09c..c49336a82e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -108,7 +108,10 @@ extension RunnerTests { ok: false, error: ErrorPayload( code: "TEXT_ENTRY_MISMATCH", - message: "text entry verification failed: expected \"\(expected)\", observed \"\(observed)\"" + message: "text entry read-back did not match: expected \"\(expected)\", observed \"\(observed)\"", + hint: textEntryMode == .replacement + ? "The app may have formatted, filtered, or consumed the entry. Inspect the field or resulting screen before retrying. If it needs correction, retry fill with the full text and --delay-ms \(TextEntryTiming.recoveryDelayMilliseconds)." + : nil ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift index b189046dda..68a2646c34 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift @@ -16,77 +16,154 @@ extension RunnerTests { TextEntryObservation(value: value, identity: identity) } - func testDegradedEchoesStayEchoesAndSummariesDoNot() { - XCTAssertTrue(Self.textEntryValueEchoes(observed: "", expected: "123456", baseline: "")) - XCTAssertTrue(Self.textEntryValueEchoes(observed: "12456", expected: "123456", baseline: "")) - XCTAssertTrue(Self.textEntryValueEchoes(observed: "old123456", expected: "123456", baseline: "")) - XCTAssertTrue(Self.textEntryValueEchoes(observed: "(555) 123-4567", expected: "5551234567", baseline: "")) - XCTAssertTrue(Self.textEntryValueEchoes(observed: "hello", expected: "hello\n", baseline: "")) - XCTAssertFalse(Self.textEntryValueEchoes(observed: "6 of 6 digits", expected: "123456", baseline: "")) - XCTAssertFalse(Self.textEntryValueEchoes(observed: "6 digits", expected: "123456", baseline: "")) - } - - func testDigitCountSummaryThatMovedOffItsBaselineIsUnconfirmed() { - let evidence = Self.unconfirmedTextEntryEvidence( - requested: "123456", - baseline: Self.otpObservation("0 of 6 digits"), - observed: Self.otpObservation("6 of 6 digits") - ) + func testReplacementExactAndLiteralLossReadBack() { + let cases: [(String, String, String, Bool)] = [ + ("", "123456", "123456", true), + ("old", "123456", "123456", true), + ("", "hello\n", "hello", true), + ("", "123456", "12456", false), + ("", "ada@example", "adxe", false), + ("old", "123456", "old12345", false), + ("0 of 6 digits", "123456", "0 of 6 digits", false), + ("", "123456", "", false), + ("$0.00", "5", "$0.05", false), + ("", "hello ", "hello", false), + ("", "", "residual", false), + ] + for (before, requested, after, verified) in cases { + let result = Self.replacementTextEntryResult( + requested: requested, baseline: Self.otpObservation(before), observed: Self.otpObservation(after) + ) + XCTAssertEqual(result.verified, verified, "\(requested) -> \(after)") + XCTAssertEqual(result.observedText, after) + XCTAssertFalse(result.repaired) + XCTAssertNil(result.unconfirmed) + } + } - XCTAssertEqual( - evidence, - TextEntryUnconfirmedEvidence( - requested: "123456", - before: "0 of 6 digits", - after: "6 of 6 digits", - target: Self.otpFieldIdentity + func testReplacementDisclosesNonLiteralValuesWithoutInferringCorrectness() { + let cases: [(String, String, String)] = [ + ("", "000629177", "00 062 91 77"), + ("", "000629177", "00 062 91 7"), + ("", "000629177", "00 026 91 77"), + ("", "5551234567", "(555) 123-4567"), + ("12", "5551234567", "(555) 123-4567"), + ("12", "5678", "12 567 8"), + ("$0.00", "1000", "$10.00"), + ("", "10.50", "1,050"), + ("0 of 6 digits", "123456", "6 of 6 digits"), + ("0 of 6 digits", "6", "1 of 6 digits"), + ("", "6", "66"), + ("", "123456", "old123456"), + ] + for (before, requested, after) in cases { + let result = Self.replacementTextEntryResult( + requested: requested, baseline: Self.otpObservation(before), observed: Self.otpObservation(after) ) - ) + XCTAssertNil(result.verified, "\(requested) -> \(after)") + XCTAssertFalse(result.repaired) + XCTAssertNil(result.failure) + XCTAssertEqual(result.unconfirmed, TextEntryUnconfirmedEvidence( + requested: requested, before: before, after: after, target: Self.otpFieldIdentity + )) + } } - func testSingleCharacterTheSummaryAlreadyContainedIsUnconfirmed() { - let evidence = Self.unconfirmedTextEntryEvidence( - requested: "6", - baseline: Self.otpObservation("0 of 6 digits"), - observed: Self.otpObservation("1 of 6 digits") + func testReplacementEvidenceRequiresTheSameReadableTarget() { + let otherField = TextEntryElementIdentity(identifier: "name-input", elementType: "TextField", frame: Self.otpFieldIdentity.frame) + let differentTarget = Self.replacementTextEntryResult( + requested: "123456", baseline: Self.otpObservation("0 of 6 digits"), + observed: Self.otpObservation("6 of 6 digits", identity: otherField) ) + XCTAssertEqual(differentTarget.verified, false) + XCTAssertNil(differentTarget.unconfirmed) + let unreadableBefore = Self.replacementTextEntryResult(requested: "123456", baseline: nil, observed: Self.otpObservation("6 of 6 digits")) + XCTAssertEqual(unreadableBefore.verified, false) + XCTAssertNil(unreadableBefore.unconfirmed) + let unreadableAfter = Self.replacementTextEntryResult(requested: "123456", baseline: Self.otpObservation(""), observed: nil) + XCTAssertNil(unreadableAfter.verified) + XCTAssertNil(unreadableAfter.unconfirmed) + } - XCTAssertEqual(evidence?.after, "1 of 6 digits") + func testReplacementSlowFirstReadGetsOneStabilityWindow() throws { + let startedAt = Date(timeIntervalSinceReferenceDate: 100) + let otherField = TextEntryElementIdentity(identifier: "other-input", elementType: "TextField", frame: Self.otpFieldIdentity.frame) + let cases: [(String, TextEntryObservation, Bool?, String?)] = [ + ("123456", Self.otpObservation("123456"), true, nil), + ("123456", Self.otpObservation("12 34 56"), nil, "12 34 56"), + ("123456", Self.otpObservation("12345"), false, nil), + ("123456", Self.otpObservation("123456", identity: otherField), false, nil), + ("hello\n", Self.otpObservation(""), nil, nil), + ("hello\n", Self.otpObservation("hllo"), nil, nil), + ] + for (requested, observed, verified, unconfirmedAfter) in cases { + var confirmation = ReplacementTextEntryConfirmation(requested: requested, baseline: Self.otpObservation(""), startedAt: startedAt) + XCTAssertNil(confirmation.observe(observed, at: startedAt.addingTimeInterval(1.1))) + XCTAssertNil(confirmation.observe(observed, at: startedAt.addingTimeInterval(1.15))) + let result = try XCTUnwrap(confirmation.observe(observed, at: startedAt.addingTimeInterval(1.31))) + XCTAssertEqual(result.verified, verified) + XCTAssertNil(result.failure) + XCTAssertEqual(result.unconfirmed?.after, unconfirmedAfter) + XCTAssertFalse(result.repaired) + } } - func testRequestTheBaselineContainedEchoesOnlyWithTheBaseline() { - XCTAssertTrue(Self.textEntryValueEchoes(observed: "66", expected: "6", baseline: "6")) - XCTAssertTrue(Self.textEntryValueEchoes(observed: "0 of 6 digits6", expected: "6", baseline: "0 of 6 digits")) - XCTAssertFalse(Self.textEntryValueEchoes(observed: "1 of 6 digits", expected: "6", baseline: "0 of 6 digits")) - XCTAssertFalse(Self.textEntryValueEchoes(observed: "6 of 6 digits", expected: "123456", baseline: "0 of 6 digits")) - XCTAssertTrue(Self.textEntryValueEchoes(observed: "abc123abc12", expected: "abc123", baseline: "abc123")) + func testReplacementSlowFirstReadDoesNotExtendForLaterChanges() throws { + let startedAt = Date(timeIntervalSinceReferenceDate: 100) + for after in ["123456", "12 34 56"] { + var confirmation = ReplacementTextEntryConfirmation(requested: "123456", baseline: Self.otpObservation(""), startedAt: startedAt) + XCTAssertNil(confirmation.observe(Self.otpObservation("123"), at: startedAt.addingTimeInterval(1.1))) + let result = try XCTUnwrap(confirmation.observe(Self.otpObservation(after), at: startedAt.addingTimeInterval(1.31))) + XCTAssertNil(result.verified) + XCTAssertEqual(result.failure, .commitNotObserved) + XCTAssertNil(result.unconfirmed) + XCTAssertFalse(result.repaired) + } } - func testEveryOtherReplacementMismatchStaysAFailure() { - let baseline = Self.otpObservation("0 of 6 digits") - let otherField = TextEntryElementIdentity( - identifier: "name-input", - elementType: "TextField", - frame: Self.otpFieldIdentity.frame - ) - let cases: [(String, String, TextEntryObservation?, TextEntryObservation?)] = [ - ("value never moved", "123456", baseline, Self.otpObservation("0 of 6 digits")), - ("dropped characters echo the request", "123456", baseline, Self.otpObservation("12456")), - ( - "dropped characters echo a stale baseline plus the request", "abc123", - Self.otpObservation("abc123"), Self.otpObservation("abc123abc12") - ), - ("another element took the entry", "123456", baseline, Self.otpObservation("6 of 6 digits", identity: otherField)), - ("unreadable before the entry", "123456", nil, Self.otpObservation("6 of 6 digits")), - ("unreadable after the entry", "123456", baseline, nil), - ("nothing formats the empty value", "", baseline, Self.otpObservation("6 of 6 digits")), - ("a bare submit key types no text", "\n", baseline, Self.otpObservation("6 of 6 digits")), - ] - for (name, requested, before, after) in cases { - XCTAssertNil( - Self.unconfirmedTextEntryEvidence(requested: requested, baseline: before, observed: after), - name + func testReplacementReadBackWaitsForSettlingBeforeDeadline() throws { + let startedAt = Date(timeIntervalSinceReferenceDate: 100) + for after in ["123456", "12 34 56"] { + var confirmation = ReplacementTextEntryConfirmation(requested: "123456", baseline: Self.otpObservation(""), startedAt: startedAt) + let observed = Self.otpObservation(after) + XCTAssertNil(confirmation.observe(observed, at: startedAt.addingTimeInterval(0.1))) + let result = try XCTUnwrap(confirmation.observe(observed, at: startedAt.addingTimeInterval(0.31))) + XCTAssertEqual(result.verified, after == "123456" ? true : nil) + XCTAssertEqual(result.unconfirmed?.after, after == "123456" ? nil : after) + XCTAssertNil(result.failure) + } + } + + func testReplacementUnchangedValueWaitsForTheDeadline() throws { + let startedAt = Date(timeIntervalSinceReferenceDate: 100) + let observed = Self.otpObservation("") + var confirmation = ReplacementTextEntryConfirmation(requested: "123456", baseline: observed, startedAt: startedAt) + XCTAssertNil(confirmation.observe(observed, at: startedAt.addingTimeInterval(0.1))) + XCTAssertNil(confirmation.observe(observed, at: startedAt.addingTimeInterval(0.31))) + let result = try XCTUnwrap(confirmation.observe(observed, at: startedAt.addingTimeInterval(1.1))) + XCTAssertEqual(result.verified, false) + XCTAssertNil(result.failure) + XCTAssertFalse(result.repaired) + } + + func testReplacementSubmitDoesNotTreatMultilineContentAsASubmitKey() { + for after in ["", "hllo", "submitted"] { + let submitted = Self.replacementTextEntryResult( + requested: "hello\n", baseline: Self.otpObservation(""), observed: Self.otpObservation(after) ) + XCTAssertNil(submitted.verified) + XCTAssertNil(submitted.unconfirmed) + XCTAssertNil(submitted.failure) + XCTAssertFalse(submitted.repaired) + } + let textView = TextEntryElementIdentity(identifier: "message", elementType: elementTypeName(.textView), frame: Self.otpFieldIdentity.frame) + for (after, expected) in [("hello", false), ("hello\n", true)] { + let result = Self.replacementTextEntryResult( + requested: "hello\n", baseline: Self.otpObservation("", identity: textView), + observed: Self.otpObservation(after, identity: textView) + ) + XCTAssertEqual(result.verified, expected) + XCTAssertNil(result.unconfirmed) } } @@ -118,6 +195,74 @@ extension RunnerTests { } #if os(iOS) + @MainActor + func testFillWithAppWriteBackFailsWithoutRetyping() throws { + app.launchArguments = [ + "--agent-device-text-entry-regression", "--agent-device-text-entry-app-owned-value", + "--agent-device-text-entry-acknowledge-window", "10", + ] + app.launch() + addTeardownBlock { [self] in + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let field = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + let frame = field.frame + let text = "ada@example" + let command = try JSONDecoder().decode(Command.self, from: JSONSerialization.data(withJSONObject: [ + "command": "type", "textEntryMode": "replace", "text": text, + "x": frame.midX, "y": frame.midY, "appBundleId": "com.callstack.agentdevice.runner", + ])) + let failures = currentXCTestFailureCount() + let response = executeTypeCommand(activeApp: app, command: command) + XCTAssertFalse(didRecordXCTestFailure(since: failures)) + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_ENTRY_MISMATCH") + XCTAssertEqual(field.value as? String, "a") + let counters = app.staticTexts["agent-device-text-entry-write-backs"].label.split(separator: " ") + XCTAssertTrue(counters.contains("total-edits=11")) + XCTAssertTrue(counters.contains("write-backs=10")) + } + + @MainActor + func testFillIntoTemplateCurrencyFieldReportsUnconfirmedWithoutRetyping() throws { + let field = try assertFillIntoNormalizingFieldIsUnconfirmed( + fixtureFlag: "--agent-device-text-entry-currency-value", + commandId: "fill-currency", + text: "1000", + expectedBefore: "$0.00", + expectedAfter: "$10.00" + ) + XCTAssertEqual(field.value as? String, "$10.00") + XCTAssertEqual(app.staticTexts["agent-device-text-entry-events"].label, "4") + } + + @MainActor + func testFillThatClearsOnSubmitIsUnverifiedAndSubmitsOnce() throws { + app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-submit-clears"] + app.launch() + addTeardownBlock { [self] in + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let field = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + let frame = field.frame + let command = try JSONDecoder().decode(Command.self, from: JSONSerialization.data(withJSONObject: [ + "command": "type", "textEntryMode": "replace", "text": "hello\n", + "x": frame.midX, "y": frame.midY, "appBundleId": "com.callstack.agentdevice.runner", + ])) + let failures = currentXCTestFailureCount() + let response = executeTypeCommand(activeApp: app, command: command) + XCTAssertFalse(didRecordXCTestFailure(since: failures)) + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.message, "typed") + XCTAssertNil(response.data?.verification) + XCTAssertEqual(field.value as? String, "") + XCTAssertEqual(app.staticTexts["agent-device-text-entry-events"].label, "1") + } + /// An OTP field that announces "6 of 6 digits" instead of the digits it holds. Every digit /// arrives, so the fill succeeds with evidence instead of failing on a summary it compared /// against the code, and the code is typed once rather than retyped by a repair. @@ -133,13 +278,57 @@ extension RunnerTests { try assertDigitCountFillIsUnconfirmed(code: "6", summary: "1 of 6 digits") } + @MainActor + func testFillIntoDigitGroupingFieldReportsUnconfirmedEvidenceAndDoesNotRepair() throws { + let textField = try assertFillIntoNormalizingFieldIsUnconfirmed( + fixtureFlag: "--agent-device-text-entry-digit-grouping-value", + commandId: "fill-digit-grouping", + text: "000629177", + expectedBefore: "", + expectedAfter: "00 062 91 77" + ) + // The field is left holding the digits the request carried, grouped: neither cleared out nor + // doubled by a retype. + let deadline = Date().addingTimeInterval(appExistenceTimeout) + var observed: String? + while Date() < deadline { + observed = textField.value as? String + if observed == "00 062 91 77" { break } + Thread.sleep(forTimeInterval: 0.25) + } + XCTAssertEqual(observed, "00 062 91 77") + } + /// Fills the digit-count fixture field with `code` and asserts unconfirmed evidence and one entry. @MainActor private func assertDigitCountFillIsUnconfirmed(code: String, summary: String) throws { - app.launchArguments = [ - "--agent-device-text-entry-regression", - "--agent-device-text-entry-digit-count-value", - ] + try assertFillIntoNormalizingFieldIsUnconfirmed( + fixtureFlag: "--agent-device-text-entry-digit-count-value", + commandId: "fill-digit-count", + text: code, + expectedBefore: "0 of 6 digits", + expectedAfter: summary + ) + // The slots an OTP screen renders next to its input carry the digits the field really holds. + XCTAssertEqual(app.staticTexts["agent-device-text-entry-digit-slots"].label, code) + } + + /// Launches the `--agent-device-text-entry-regression` field chosen by `fixtureFlag`, fills + /// `text` through the daemon's `type`/replace command addressed at the field's center, and + /// asserts the fill succeeded once with target-bound unconfirmed evidence carrying + /// `expectedBefore`/`expectedAfter`. The field is returned for the caller's own after-entry + /// oracle, which is the only thing that differs between the normalizing fixtures: a digit-count + /// field's own value is a summary, a grouping field's is the formatted text. + @MainActor + @discardableResult + private func assertFillIntoNormalizingFieldIsUnconfirmed( + fixtureFlag: String, + commandId: String, + text: String, + expectedBefore: String, + expectedAfter: String + ) throws -> XCUIElement { + app.launchArguments = ["--agent-device-text-entry-regression", fixtureFlag] app.launch() addTeardownBlock { [self] in invalidateCachedTarget(reason: "unit_test_cleanup") @@ -152,8 +341,8 @@ extension RunnerTests { Command.self, from: JSONSerialization.data(withJSONObject: [ "command": "type", - "commandId": "fill-digit-count", - "text": code, + "commandId": commandId, + "text": text, "textEntryMode": "replace", "x": frame.midX, "y": frame.midY, @@ -168,13 +357,13 @@ extension RunnerTests { XCTAssertTrue(response.ok, String(describing: response.error)) XCTAssertEqual(response.data?.message, "typed") XCTAssertEqual(response.data?.verification, "unconfirmed") - XCTAssertEqual(response.data?.requested, code) - XCTAssertEqual(response.data?.before, "0 of 6 digits") - XCTAssertEqual(response.data?.after, summary) + XCTAssertEqual(response.data?.requested, text) + XCTAssertEqual(response.data?.before, expectedBefore) + XCTAssertEqual(response.data?.after, expectedAfter) XCTAssertEqual(response.data?.target?.resourceId, "agent-device-hardware-keyboard-input") XCTAssertEqual(response.data?.target?.className, "TextField") XCTAssertEqual(response.data?.target?.packageName, "com.callstack.agentdevice.runner") - XCTAssertEqual(app.staticTexts["agent-device-text-entry-digit-slots"].label, code) + return textField } #endif #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 586347eabc..5fd2d952bf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -128,6 +128,21 @@ extension RunnerTests { } } + // A synthesized replacement has no element-bound baseline for unconfirmed evidence. + func testSynthesizedReplacementCommitStillRefusesACompletingFormattedValue() { + let clock = CommitWaitClock() + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( + expectedText: "000629177", + placeholder: nil, + stallBudget: 2, + ceiling: 10, + now: clock.read, + observe: { "00 062 91 77" }, + waitForNextObservation: { clock.advance(1) } + ) + XCTAssertEqual(outcome, .notObserved) + } + // The non-failure counterpart: replacement mode must still tolerate real commit lag (the value // converges to an exact match over a few polls), not just instant matches. func testSynthesizedReplacementCommitToleratesLagUntilExactMatch() { diff --git a/packages/contracts/src/fill-evidence.ts b/packages/contracts/src/fill-evidence.ts index 9726d1bdb5..f4efa9c883 100644 --- a/packages/contracts/src/fill-evidence.ts +++ b/packages/contracts/src/fill-evidence.ts @@ -17,11 +17,10 @@ export type FillVerificationTarget = { }; /** - * Target-bound evidence that a fill moved a field's content from `before` to `after` without raw - * equality with `requested` being reachable, because app-owned formatting prevents it or the field's - * accessibility value does not echo the typed text. Bound to the - * {@link FillVerificationTarget} it was collected against so another field, or the same field after - * it re-laid out, cannot borrow this evidence. + * Target-bound read-back evidence when a fill could not confirm the requested text. The observed + * value may reflect app-owned formatting or altered input; this evidence makes no correctness + * claim. Bound to the {@link FillVerificationTarget} it was collected against so a different + * field cannot borrow the observation. */ export type FillUnconfirmedVerification = { verification: 'unconfirmed'; diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 0d8d8cb13f..9029fb8888 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -302,7 +302,7 @@ export const INTERACTION_COMMAND_OUTPUT_SCHEMAS = { fill: { type: 'object', description: - 'Fill response. Returns target-bound unconfirmed evidence when the exact field changed but its value cannot confirm the text: Android app-owned formatting, or an Apple field whose accessibility value does not echo the typed text.', + 'Fill response. Target-bound unconfirmed evidence reports an observed value different from the requested text. It does not establish that formatting preserved the input; assert the expected value or resulting screen before continuing.', oneOf: [confirmedFillResponseSchema, unconfirmedFillResponseSchema], }, longpress: interactionResponseDataSchema({ diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 54c607af99..742a03b8c0 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -60,7 +60,7 @@ const interactionCommandDescriptions = { 'Activate a UI target by snapshot ref, selector, or coordinates. Prefer a ref or selector after a snapshot; use coordinates only when semantic targeting is unavailable. This can change app state; use settle or verify to confirm the result without a follow-up snapshot.', press: 'Short-press a UI target by snapshot ref, selector, or coordinates. Use longpress instead when the target requires a context-menu or hold gesture.', - fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. Pass an empty text to clear the field. Prefer refs or selectors after snapshot; use recordAs to keep sensitive text out of a recorded replay while sending it to the live app.', + fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. Pass an empty text to clear the field. Inspect unconfirmed evidence and assert the expected value or resulting screen before continuing. Prefer refs or selectors after snapshot; use recordAs to keep sensitive text out of a recorded replay while sending it to the live app.', longpress: 'Hold a UI target by snapshot ref, selector, or coordinates to open a context menu or perform another hold gesture. Set durationMs when the default hold duration is unsuitable.', hover: diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index a51204a5f4..25c371a630 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -183,7 +183,7 @@ Stabilizers: Text-entry quirks: iOS Allow Paste cannot be exercised under XCUITest; prefill with clipboard write "some text" instead and test the system prompt manually. Android Gboard handwriting/stylus UI can capture text in an IME-owned input instead of the app field. If fill reports that input was captured by the keyboard/IME, use the diagnostic targetInput/actualInput details, inspect keyboard status/get if needed, and switch or disable handwriting outside the command plan before retrying. Do not keep retrying fill/type against the same field while the IME owns focus. If the exact target changes but app-owned formatting prevents raw equality, fill succeeds with verification: "unconfirmed" plus target-bound requested/before/after evidence; inspect that evidence instead of retrying the same mutation. - iOS accessibility-driven fill into a field whose accessibility value does not echo the typed text (an OTP input announcing "6 of 6 digits") succeeds with the same verification: "unconfirmed" evidence when that field's value moved off its post-clear baseline. Dropped characters and stale prefixes still fail, and the coordinate-driven fill route still requires an exact match. Confirm the code through the slots or the next screen; do not retry the fill or switch to per-character typing. + iOS accessibility-driven fill types the replacement once and never clears and retypes a read-back mismatch. Exact settled text confirms the entry. A value containing only characters in order from the post-clear value plus the request fails with TEXT_ENTRY_MISMATCH, including plain dropped characters and unchanged values; legitimate trimming or masks can also hit this limit ("5" into "$0.00" displaying "$0.05"). Other settled values return verification: "unconfirmed" with requested/before/after/target evidence, including grouping ("00 062 91 77" for "000629177"), currency ("$10.00" for "1000"), and OTP summaries. Missing, reordered, duplicated or stale text can also be unconfirmed, so assert the expected value with a value selector, get/is, or the resulting screen before continuing. For single-line fills ending in a submit key, any non-matching value observed on the same target succeeds unverified, including partial or cleared text; this exception skips dropped-character checks because the submit may change the field. Assert the resulting screen. Multiline fields compare newlines as content. Unreadable or removed fields remain unverified. The coordinate-driven fill route still requires an exact match. Only retry fill with --delay-ms 80 after TEXT_ENTRY_MISMATCH and inspection shows the field needs correction. React Native internals: If the question is about React Native performance, profiling, props, state, hooks, render causes, slow components, or rerenders, use help react-devtools instead of inferring from screenshots or logs.`, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8561afd7ee..0ef37baefb 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -474,7 +474,8 @@ On iOS, if `fill` reports `TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED`, the text is lo Use plain `fill` or `type` first for ordinary login and form fields. Use `--delay-ms` on `type` or `fill` only when a debounced search field or search-as-you-type input actually misses characters, or when the app must receive incremental updates. Delayed typing intentionally prefers paced character entry over clipboard-style fallbacks so the target field receives each incremental update. On Android, `fill` also verifies text and treats IME-owned capture as a terminal failure instead of retrying against the wrong field. -When the filled field changed but its value cannot confirm the text, `fill` succeeds with `verification: "unconfirmed"` plus target-bound `requested`/`before`/`after` evidence instead of failing. On Android that is app-owned formatting. On iOS's accessibility-driven `fill` route it is a field whose value moved off its post-clear baseline to one that does not echo the typed text, such as an OTP input announcing `6 of 6 digits`; dropped characters and stale prefixes still fail, and the coordinate-driven route still requires an exact match. Verify the result through the screen, for example the OTP slots or the next screen, instead of retrying the same fill. +When `fill` returns `verification: "unconfirmed"`, inspect its target-bound `requested`/`before`/`after` evidence and assert the expected field value or resulting screen before continuing. This can happen with app-owned formatting on Android or a different settled accessibility value on iOS, such as `00 062 91 77` after filling `000629177`, `$10.00` after filling `1000`, or an OTP input announcing `6 of 6 digits`. Unconfirmed evidence does not establish correctness: missing, reordered, duplicated, or stale text can also produce it. +On iOS's accessibility-driven route, `fill` types the replacement once and never clears and retypes a read-back mismatch. An exact settled read-back confirms the text. A value that only drops characters from the post-clear value plus your request fails with `TEXT_ENTRY_MISMATCH`; this includes unchanged values and plain dropped-character cases, but can also reject legitimate filtering, trimming, or a mask (`5` into `$0.00` may display `$0.05`). Only retry `fill --delay-ms 80` after `TEXT_ENTRY_MISMATCH` and inspection shows the field needs correction. For a single-line fill ending in a submit key, any non-matching value observed on the same target succeeds unverified, including partial or cleared text. This exception skips dropped-character checks because submitting may change the field; assert the resulting screen. Multiline fields compare newlines as content. An unreadable field or one removed after delivery also remains unverified. The coordinate-driven route used when accessibility is degraded still requires an exact match. For a digit-grouping field, you can assert its display value with `agent-device wait 'id="field-phone" value="00 062 91 77"'`. Android text entry is owned by `agent-device`: provider-native injection when available, then chunk-safe ASCII shell input. Do not switch to raw `adb`, clipboard, or paste as an agent fallback. If non-ASCII is unsupported in the current backend, report the tool/device gap. `click --button secondary` is the desktop context-menu flow on macOS. `click --button middle` returns an unsupported-operation error on macOS.