feat(playground): add device options configuration for Android/iOS

This commit implements device-specific configuration options in the
playground UI, allowing users to customize device behavior such as
keyboard handling and IME strategy.

Changes:
- Add device options state management with localStorage persistence
- Create UI controls for Android-specific options (imeStrategy,
  autoDismissKeyboard, keyboardDismissStrategy, alwaysRefreshScreenInfo)
- Create UI controls for iOS-specific options (autoDismissKeyboard)
- Extend execution pipeline to pass deviceOptions from frontend to backend
- Update agent.interface.options on the server side when deviceOptions
  are received
- Optimize parameter flattening to avoid delete operator performance issues

Technical implementation:
- Frontend: Store device options in Zustand with localStorage sync
- SDK: Include deviceOptions in remote execution adapter payload
- Server: Update agent.interface.options to apply settings globally
- This ensures all actions (including those called by aiAct) use the
  updated options

Fixes #1282

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: quanru

apps/android-playground/src/components/playground-panel/index.tsx

@@ -77,6 +77,7 @@ export default function PlaygroundPanel() {
             enableScrollToBottom: true,
             serverMode: true,
             showEnvConfigReminder: true,
+            deviceType: 'android',
           }}
           branding={{
             title: 'Android Playground',

packages/playground/src/adapters/remote-execution.ts

@@ -191,6 +191,7 @@ export class RemoteExecutionAdapter extends BasePlaygroundAdapter {
       { key: 'deepThink', value: options.deepThink },
       { key: 'screenshotIncluded', value: options.screenshotIncluded },
       { key: 'domIncluded', value: options.domIncluded },
+      { key: 'deviceOptions', value: options.deviceOptions },
       { key: 'params', value: value.params },
     ] as const;
 

packages/playground/src/common.ts

@@ -51,7 +51,13 @@ export async function parseStructuredParams(
       ? Object.keys((schema as { shape: Record<string, unknown> }).shape)
       : [];
 
-  const paramObj: Record<string, unknown> = { ...options };
+  // Start with options and merge deviceOptions into the same level
+  // Destructure to exclude deviceOptions from the final object
+  const { deviceOptions: _, ...optionsWithoutDeviceOptions } = options;
+  const paramObj: Record<string, unknown> = {
+    ...optionsWithoutDeviceOptions,
+    ...(options.deviceOptions || {}),
+  };
 
   keys.forEach((key) => {
     if (
@@ -186,10 +192,19 @@ export async function executeAction(
           })
         : undefined;
 
-      return await activeAgent.callActionInActionSpace(action.name, {
+      // Flatten deviceOptions into the params
+      // Destructure to exclude deviceOptions from the final object
+      const { deviceOptions: _, ...optionsWithoutDeviceOptions } = options;
+      const actionParams = {
         locate: detailedLocateParam,
-        ...options,
-      });
+        ...optionsWithoutDeviceOptions,
+        ...(options.deviceOptions || {}),
+      };
+
+      return await activeAgent.callActionInActionSpace(
+        action.name,
+        actionParams,
+      );
     }
   } else {
     const prompt = value.prompt;

packages/playground/src/server.ts

@@ -315,6 +315,7 @@ class PlaygroundServer {
         deepThink,
         screenshotIncluded,
         domIncluded,
+        deviceOptions,
       } = req.body;
 
       if (!type) {
@@ -323,6 +324,18 @@ class PlaygroundServer {
         });
       }
 
+      // Update device options if provided
+      if (
+        deviceOptions &&
+        this.agent.interface &&
+        'options' in this.agent.interface
+      ) {
+        this.agent.interface.options = {
+          ...(this.agent.interface.options || {}),
+          ...deviceOptions,
+        };
+      }
+
       // Check if another task is running
       if (this.currentTaskId) {
         return res.status(409).json({
@@ -376,6 +389,7 @@ class PlaygroundServer {
             deepThink,
             screenshotIncluded,
             domIncluded,
+            deviceOptions,
           },
         );
       } catch (error: unknown) {

packages/playground/src/types.ts

@@ -23,12 +23,20 @@ export interface ServerResponse {
   error?: string;
 }
 
+export interface DeviceOptions {
+  imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii';
+  autoDismissKeyboard?: boolean;
+  keyboardDismissStrategy?: 'esc-first' | 'back-first';
+  alwaysRefreshScreenInfo?: boolean;
+}
+
 export interface ExecutionOptions {
   deepThink?: boolean;
   screenshotIncluded?: boolean;
   domIncluded?: boolean | 'visible-only';
   context?: any;
   requestId?: string;
+  deviceOptions?: DeviceOptions;
 }
 
 // Extended web types for playground

packages/visualizer/src/component/config-selector/index.tsx

@@ -2,9 +2,14 @@ import { Checkbox, Dropdown, type MenuProps, Radio } from 'antd';
 import type React from 'react';
 import SettingOutlined from '../../icons/setting.svg';
 import { useEnvConfig } from '../../store/store';
+import type { DeviceType } from '../../types';
 import {
+  alwaysRefreshScreenInfoTip,
+  autoDismissKeyboardTip,
   deepThinkTip,
   domIncludedTip,
+  imeStrategyTip,
+  keyboardDismissStrategyTip,
   screenshotIncludedTip,
   trackingTip,
 } from '../../utils/constants';
@@ -14,13 +19,15 @@ interface ConfigSelectorProps {
   enableTracking: boolean;
   showDataExtractionOptions: boolean;
   hideDomAndScreenshotOptions?: boolean; // Hide domIncluded and screenshotIncluded options
+  deviceType?: DeviceType;
 }
 
 export const ConfigSelector: React.FC<ConfigSelectorProps> = ({
   showDeepThinkOption = false,
   enableTracking = false,
   showDataExtractionOptions = false,
   hideDomAndScreenshotOptions = false,
+  deviceType,
 }) => {
   const forceSameTabNavigation = useEnvConfig(
     (state) => state.forceSameTabNavigation,
@@ -37,7 +44,36 @@ export const ConfigSelector: React.FC<ConfigSelectorProps> = ({
   const domIncluded = useEnvConfig((state) => state.domIncluded);
   const setDomIncluded = useEnvConfig((state) => state.setDomIncluded);
 
-  if (!enableTracking && !showDeepThinkOption && !showDataExtractionOptions) {
+  // Device-specific configuration
+  const imeStrategy = useEnvConfig((state) => state.imeStrategy);
+  const setImeStrategy = useEnvConfig((state) => state.setImeStrategy);
+  const autoDismissKeyboard = useEnvConfig(
+    (state) => state.autoDismissKeyboard,
+  );
+  const setAutoDismissKeyboard = useEnvConfig(
+    (state) => state.setAutoDismissKeyboard,
+  );
+  const keyboardDismissStrategy = useEnvConfig(
+    (state) => state.keyboardDismissStrategy,
+  );
+  const setKeyboardDismissStrategy = useEnvConfig(
+    (state) => state.setKeyboardDismissStrategy,
+  );
+  const alwaysRefreshScreenInfo = useEnvConfig(
+    (state) => state.alwaysRefreshScreenInfo,
+  );
+  const setAlwaysRefreshScreenInfo = useEnvConfig(
+    (state) => state.setAlwaysRefreshScreenInfo,
+  );
+
+  const hasDeviceOptions = deviceType === 'android' || deviceType === 'ios';
+
+  if (
+    !enableTracking &&
+    !showDeepThinkOption &&
+    !showDataExtractionOptions &&
+    !hasDeviceOptions
+  ) {
     return null;
   }
 
@@ -120,6 +156,86 @@ export const ConfigSelector: React.FC<ConfigSelectorProps> = ({
       });
     }
 
+    // Android-specific options
+    if (deviceType === 'android') {
+      items.push({
+        label: (
+          <div style={{ padding: '4px 0' }}>
+            <div style={{ marginBottom: '4px', fontSize: '14px' }}>
+              {imeStrategyTip}
+            </div>
+            <Radio.Group
+              size="small"
+              value={imeStrategy}
+              onChange={(e) => setImeStrategy(e.target.value)}
+            >
+              <Radio value="always-yadb">Always YADB</Radio>
+              <Radio value="yadb-for-non-ascii">YADB for non-ASCII</Radio>
+            </Radio.Group>
+          </div>
+        ),
+        key: 'ime-strategy-config',
+      });
+
+      items.push({
+        label: (
+          <Checkbox
+            onChange={(e) => setAutoDismissKeyboard(e.target.checked)}
+            checked={autoDismissKeyboard}
+          >
+            {autoDismissKeyboardTip}
+          </Checkbox>
+        ),
+        key: 'auto-dismiss-keyboard-config',
+      });
+
+      items.push({
+        label: (
+          <div style={{ padding: '4px 0' }}>
+            <div style={{ marginBottom: '4px', fontSize: '14px' }}>
+              {keyboardDismissStrategyTip}
+            </div>
+            <Radio.Group
+              size="small"
+              value={keyboardDismissStrategy}
+              onChange={(e) => setKeyboardDismissStrategy(e.target.value)}
+            >
+              <Radio value="esc-first">ESC first</Radio>
+              <Radio value="back-first">Back first</Radio>
+            </Radio.Group>
+          </div>
+        ),
+        key: 'keyboard-dismiss-strategy-config',
+      });
+
+      items.push({
+        label: (
+          <Checkbox
+            onChange={(e) => setAlwaysRefreshScreenInfo(e.target.checked)}
+            checked={alwaysRefreshScreenInfo}
+          >
+            {alwaysRefreshScreenInfoTip}
+          </Checkbox>
+        ),
+        key: 'always-refresh-screen-info-config',
+      });
+    }
+
+    // iOS-specific options
+    if (deviceType === 'ios') {
+      items.push({
+        label: (
+          <Checkbox
+            onChange={(e) => setAutoDismissKeyboard(e.target.checked)}
+            checked={autoDismissKeyboard}
+          >
+            {autoDismissKeyboardTip}
+          </Checkbox>
+        ),
+        key: 'auto-dismiss-keyboard-config',
+      });
+    }
+
     return items;
   }
 };

packages/visualizer/src/component/prompt-input/index.tsx

@@ -12,7 +12,7 @@ import React, {
 } from 'react';
 import type { HistoryItem } from '../../store/history';
 import { useHistoryStore } from '../../store/history';
-import type { RunType } from '../../types';
+import type { DeviceType, RunType } from '../../types';
 import type { ServiceModeType } from '../../types';
 import {
   type FormParams,
@@ -57,6 +57,7 @@ interface PromptInputProps {
   clearPromptAfterRun?: boolean;
   hideDomAndScreenshotOptions?: boolean; // Hide domIncluded and screenshotIncluded options
   actionSpace: DeviceAction<any>[]; // Required actionSpace for dynamic parameter detection
+  deviceType?: DeviceType;
 }
 
 export const PromptInput: React.FC<PromptInputProps> = ({
@@ -72,6 +73,7 @@ export const PromptInput: React.FC<PromptInputProps> = ({
   clearPromptAfterRun = true,
   actionSpace,
   hideDomAndScreenshotOptions = false,
+  deviceType,
 }) => {
   const [hoveringSettings, setHoveringSettings] = useState(false);
   const [promptValue, setPromptValue] = useState('');
@@ -202,12 +204,14 @@ export const PromptInput: React.FC<PromptInputProps> = ({
     const hasDeepThink = showDeepThinkOption;
     const hasDataExtraction =
       showDataExtractionOptions && !hideDomAndScreenshotOptions;
-    return hasTracking || hasDeepThink || hasDataExtraction;
+    const hasDeviceOptions = deviceType === 'android' || deviceType === 'ios';
+    return hasTracking || hasDeepThink || hasDataExtraction || hasDeviceOptions;
   }, [
     serviceMode,
     showDeepThinkOption,
     showDataExtractionOptions,
     hideDomAndScreenshotOptions,
+    deviceType,
   ]);
 
   // Get available methods for dropdown (filtered by actionSpace)
@@ -1026,6 +1030,7 @@ export const PromptInput: React.FC<PromptInputProps> = ({
                 showDeepThinkOption={showDeepThinkOption}
                 showDataExtractionOptions={showDataExtractionOptions}
                 hideDomAndScreenshotOptions={hideDomAndScreenshotOptions}
+                deviceType={deviceType}
               />
             </div>
           )}

packages/visualizer/src/component/universal-playground/index.tsx

@@ -184,6 +184,7 @@ export function UniversalPlayground({
     showContextPreview && componentConfig.showContextPreview !== false;
   const layout = componentConfig.layout || 'vertical';
   const showVersionInfo = componentConfig.showVersionInfo !== false;
+  const deviceType = componentConfig.deviceType;
 
   return (
     <div className={`playground-container ${layout}-mode ${className}`.trim()}>
@@ -374,6 +375,7 @@ export function UniversalPlayground({
             onRun={handleFormRun}
             onStop={handleStop}
             actionSpace={actionSpace}
+            deviceType={deviceType}
           />
         </div>
 

packages/visualizer/src/hooks/usePlaygroundExecution.ts

@@ -30,7 +30,15 @@ export function usePlaygroundExecution(
   interruptedFlagRef: React.MutableRefObject<Record<number, boolean>>,
 ) {
   // Get execution options from environment config
-  const { deepThink, screenshotIncluded, domIncluded } = useEnvConfig();
+  const {
+    deepThink,
+    screenshotIncluded,
+    domIncluded,
+    imeStrategy,
+    autoDismissKeyboard,
+    keyboardDismissStrategy,
+    alwaysRefreshScreenInfo,
+  } = useEnvConfig();
   // Handle form submission and execution
   const handleRun = useCallback(
     async (value: FormValue) => {
@@ -109,11 +117,19 @@ export function usePlaygroundExecution(
         }
 
         // Execute the action using the SDK
+        const deviceOptionsToSend = {
+          imeStrategy,
+          autoDismissKeyboard,
+          keyboardDismissStrategy,
+          alwaysRefreshScreenInfo,
+        };
+
         result.result = await playgroundSDK.executeAction(actionType, value, {
           requestId: thisRunningId.toString(),
           deepThink,
           screenshotIncluded,
           domIncluded,
+          deviceOptions: deviceOptionsToSend,
         });
 
         // For some adapters, result might already include dump and reportHTML
@@ -213,6 +229,10 @@ export function usePlaygroundExecution(
       deepThink,
       screenshotIncluded,
       domIncluded,
+      imeStrategy,
+      autoDismissKeyboard,
+      keyboardDismissStrategy,
+      alwaysRefreshScreenInfo,
     ],
   );