feat(playground): add device options configuration for Android/iOS (#1485)

* feat(playground): add device options configuration for Android/iOS

This commit implements device-specific configuration options in the
playground UI, allowing users to customize device behavior such as
keyboard handling and IME strategy.

Changes:
- Add device options state management with localStorage persistence
- Create UI controls for Android-specific options (imeStrategy,
  autoDismissKeyboard, keyboardDismissStrategy, alwaysRefreshScreenInfo)
- Create UI controls for iOS-specific options (autoDismissKeyboard)
- Extend execution pipeline to pass deviceOptions from frontend to backend
- Update agent.interface.options on the server side when deviceOptions
  are received
- Optimize parameter flattening to avoid delete operator performance issues

Technical implementation:
- Frontend: Store device options in Zustand with localStorage sync
- SDK: Include deviceOptions in remote execution adapter payload
- Server: Update agent.interface.options to apply settings globally
- This ensures all actions (including those called by aiAct) use the
  updated options

Fixes #1282

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

* feat(playground): add dynamic device type detection for iOS/web playground

The universal playground app now detects the device type from the
connected server's /interface-info API and displays device-specific
configuration options accordingly.

This ensures that iOS playground users can see and configure iOS
device options (autoDismissKeyboard), while web users see no
device-specific options.

Related to #1282

* fix(ios): improve keyboard dismissal to prevent accidental UI interactions

The previous implementation used a swipe down gesture at a fixed screen
position (1/3 from top) which could accidentally click on search results
or other UI elements that appeared after text input.

Changes:
- Use WDA's dismissKeyboard API as the primary method (more reliable)
- Fall back to safer swipe gesture (from bottom up) if API fails
- Increase wait time from 300ms to 500ms for UI stability
- Update autoDismissKeyboard documentation to reflect default behavior

Technical details:
- WDA API tries common keyboard button names: return, done, go, search, etc.
- Swipe fallback uses safer coordinates: from 90% height to 50% height
- This prevents accidental taps on UI elements in the upper portion of screen

Related to #1282

* feat(screenshot-viewer): add screenshot viewer component with styles and functionality

* fix(tests): enhance keyboard dismissal tests to simulate failure scenarios

---------

Co-authored-by: Claude <[email protected]>

Author: quanru

apps/android-playground/src/components/playground-panel/index.tsx

@@ -77,6 +77,7 @@ export default function PlaygroundPanel() {
             enableScrollToBottom: true,
             serverMode: true,
             showEnvConfigReminder: true,
+            deviceType: 'android',
           }}
           branding={{
             title: 'Android Playground',

apps/playground/src/App.tsx

@@ -23,6 +23,9 @@ export default function App() {
   const [serverOnline, setServerOnline] = useState(false);
   const [isUserOperating, setIsUserOperating] = useState(false);
   const [isNarrowScreen, setIsNarrowScreen] = useState(false);
+  const [deviceType, setDeviceType] = useState<'web' | 'android' | 'ios'>(
+    'web',
+  );
 
   // Create PlaygroundSDK and storage provider
   const playgroundSDK = useMemo(() => {
@@ -50,6 +53,21 @@ export default function App() {
       try {
         const online = await playgroundSDK.checkStatus();
         setServerOnline(online);
+
+        // Get device type from server if online
+        if (online) {
+          try {
+            const interfaceInfo = await playgroundSDK.getInterfaceInfo();
+            if (interfaceInfo?.type) {
+              const type = interfaceInfo.type.toLowerCase();
+              if (type === 'android' || type === 'ios' || type === 'web') {
+                setDeviceType(type as 'web' | 'android' | 'ios');
+              }
+            }
+          } catch (error) {
+            console.warn('Failed to get interface info:', error);
+          }
+        }
       } catch (error) {
         console.error('Failed to check server status:', error);
         setServerOnline(false);
@@ -145,6 +163,7 @@ export default function App() {
                       enableScrollToBottom: true,
                       serverMode: true,
                       showEnvConfigReminder: true,
+                      deviceType,
                     }}
                     branding={{
                       title: 'Playground',

packages/ios/src/device.ts

@@ -80,7 +80,7 @@ export class IOSDevice implements AbstractInterface {
             .boolean()
             .optional()
             .describe(
-              'If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.',
+              'Whether to dismiss the keyboard after input. Defaults to true if not specified. Set to false to keep the keyboard visible after input.',
             ),
           mode: z
             .enum(['replace', 'clear', 'append'])
@@ -763,32 +763,42 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
 
   async hideKeyboard(keyNames?: string[]): Promise<boolean> {
     try {
-      // If keyNames are provided, use them instead of manual swipe down
-      if (keyNames && keyNames.length > 0) {
+      // Always try WDA's dismissKeyboard API first (most reliable)
+      // Use common keyboard button names if not specified
+      const dismissKeys =
+        keyNames && keyNames.length > 0
+          ? keyNames
+          : ['return', 'done', 'go', 'search', 'next', 'send'];
+
+      debugDevice(
+        `Attempting to dismiss keyboard using WDA API with keys: ${dismissKeys.join(', ')}`,
+      );
+
+      try {
+        await this.wdaBackend.dismissKeyboard(dismissKeys);
+        debugDevice('Successfully dismissed keyboard using WDA API');
+        await sleep(500); // Wait longer to ensure UI is stable
+        return true;
+      } catch (wdaError) {
         debugDevice(
-          `Using keyNames to dismiss keyboard: ${keyNames.join(', ')}`,
+          `WDA dismissKeyboard failed, falling back to swipe gesture: ${wdaError}`,
         );
-        await this.wdaBackend.dismissKeyboard(keyNames);
-        debugDevice('Dismissed keyboard using provided keyNames');
-        await sleep(300);
-        return true;
       }
 
-      // Default behavior: Get window size for swipe coordinates
+      // Fallback: Use swipe gesture if WDA API fails
+      // Use safer coordinates: swipe up from bottom of screen
       const windowSize = await this.wdaBackend.getWindowSize();
-
-      // Calculate swipe coordinates at one-third position of the screen
       const centerX = Math.round(windowSize.width / 2);
-      const startY = Math.round(windowSize.height * 0.33); // Start at one-third from top
-      const endY = Math.round(windowSize.height * 0.33 + 10); // Swipe down
+      const startY = Math.round(windowSize.height * 0.9); // Start near bottom
+      const endY = Math.round(windowSize.height * 0.5); // Swipe up to middle
 
-      // Perform swipe down gesture to dismiss keyboard
-      await this.swipe(centerX, startY, centerX, endY, 50);
+      // Perform swipe up gesture to dismiss keyboard
+      await this.swipe(centerX, startY, centerX, endY, 300);
       debugDevice(
-        'Dismissed keyboard with swipe down gesture at screen one-third position',
+        'Dismissed keyboard with swipe up gesture from bottom of screen',
       );
 
-      await sleep(300);
+      await sleep(500); // Wait longer to ensure UI is stable
       return true;
     } catch (error) {
       debugDevice(`Failed to hide keyboard: ${error}`);

packages/ios/tests/unit-test/device.test.ts

@@ -424,7 +424,10 @@ describe('IOSDevice', () => {
     });
 
     it('should handle keyboard dismissal with default strategy', async () => {
-      // Mock getWindowSize and swipe methods since hideKeyboard uses swipe gesture by default
+      // Mock dismissKeyboard to fail so it falls back to swipe gesture
+      mockWdaClient.dismissKeyboard = vi
+        .fn()
+        .mockRejectedValue(new Error('dismissKeyboard not available'));
       mockWdaClient.getWindowSize = vi
         .fn()
         .mockResolvedValue({ width: 375, height: 812 });
@@ -436,7 +439,10 @@ describe('IOSDevice', () => {
     });
 
     it('should handle keyboard dismissal failure', async () => {
-      // Mock swipe to throw an error to simulate failure
+      // Mock both dismissKeyboard and swipe to fail to simulate total failure
+      mockWdaClient.dismissKeyboard = vi
+        .fn()
+        .mockRejectedValue(new Error('dismissKeyboard failed'));
       mockWdaClient.getWindowSize = vi
         .fn()
         .mockResolvedValue({ width: 375, height: 812 });
@@ -454,6 +460,9 @@ describe('IOSDevice', () => {
         ...mockWdaClient,
         createSession: vi.fn().mockResolvedValue({ sessionId: 'test-session' }),
         typeText: vi.fn().mockResolvedValue(undefined),
+        dismissKeyboard: vi
+          .fn()
+          .mockRejectedValue(new Error('dismissKeyboard not available')),
         getWindowSize: vi.fn().mockResolvedValue({ width: 375, height: 812 }),
         getScreenScale: vi.fn().mockResolvedValue(2),
         swipe: vi.fn().mockResolvedValue(undefined),

packages/playground/src/adapters/remote-execution.ts

@@ -191,6 +191,7 @@ export class RemoteExecutionAdapter extends BasePlaygroundAdapter {
       { key: 'deepThink', value: options.deepThink },
       { key: 'screenshotIncluded', value: options.screenshotIncluded },
       { key: 'domIncluded', value: options.domIncluded },
+      { key: 'deviceOptions', value: options.deviceOptions },
       { key: 'params', value: value.params },
     ] as const;
 

packages/playground/src/common.ts

@@ -51,7 +51,13 @@ export async function parseStructuredParams(
       ? Object.keys((schema as { shape: Record<string, unknown> }).shape)
       : [];
 
-  const paramObj: Record<string, unknown> = { ...options };
+  // Start with options and merge deviceOptions into the same level
+  // Destructure to exclude deviceOptions from the final object
+  const { deviceOptions: _, ...optionsWithoutDeviceOptions } = options;
+  const paramObj: Record<string, unknown> = {
+    ...optionsWithoutDeviceOptions,
+    ...(options.deviceOptions || {}),
+  };
 
   keys.forEach((key) => {
     if (
@@ -186,10 +192,19 @@ export async function executeAction(
           })
         : undefined;
 
-      return await activeAgent.callActionInActionSpace(action.name, {
+      // Flatten deviceOptions into the params
+      // Destructure to exclude deviceOptions from the final object
+      const { deviceOptions: _, ...optionsWithoutDeviceOptions } = options;
+      const actionParams = {
         locate: detailedLocateParam,
-        ...options,
-      });
+        ...optionsWithoutDeviceOptions,
+        ...(options.deviceOptions || {}),
+      };
+
+      return await activeAgent.callActionInActionSpace(
+        action.name,
+        actionParams,
+      );
     }
   } else {
     const prompt = value.prompt;

packages/playground/src/server.ts

@@ -315,6 +315,7 @@ class PlaygroundServer {
         deepThink,
         screenshotIncluded,
         domIncluded,
+        deviceOptions,
       } = req.body;
 
       if (!type) {
@@ -323,6 +324,18 @@ class PlaygroundServer {
         });
       }
 
+      // Update device options if provided
+      if (
+        deviceOptions &&
+        this.agent.interface &&
+        'options' in this.agent.interface
+      ) {
+        this.agent.interface.options = {
+          ...(this.agent.interface.options || {}),
+          ...deviceOptions,
+        };
+      }
+
       // Check if another task is running
       if (this.currentTaskId) {
         return res.status(409).json({
@@ -376,6 +389,7 @@ class PlaygroundServer {
             deepThink,
             screenshotIncluded,
             domIncluded,
+            deviceOptions,
           },
         );
       } catch (error: unknown) {

packages/playground/src/types.ts

@@ -23,12 +23,20 @@ export interface ServerResponse {
   error?: string;
 }
 
+export interface DeviceOptions {
+  imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii';
+  autoDismissKeyboard?: boolean;
+  keyboardDismissStrategy?: 'esc-first' | 'back-first';
+  alwaysRefreshScreenInfo?: boolean;
+}
+
 export interface ExecutionOptions {
   deepThink?: boolean;
   screenshotIncluded?: boolean;
   domIncluded?: boolean | 'visible-only';
   context?: any;
   requestId?: string;
+  deviceOptions?: DeviceOptions;
 }
 
 // Extended web types for playground