Add gemini-3.5-flash model support

README.md

@@ -2,7 +2,7 @@
 
 ## Quick Start
 
-This section will guide you through setting up and running the Computer Use Preview model, either the Gemini Developer API or Vertex AI. Follow these steps to get started.
+This section will guide you through setting up and running Gemini Computer Use, using either the Gemini Developer API or Vertex AI. Follow these steps to get started.
 
 ### 1. Installation
 
@@ -131,7 +131,8 @@ python main.py --query="Go to Google and type 'Hello World' into the search bar"
 
 You can choose the model to use by specifying the ```--model <model name>``` flag. Available options on Gemini Developer API and Vertex AI Client:
 
-- `gemini-2.5-computer-use-preview-10-2025`: This is the default model.
+- `gemini-3.5-flash`: This is the default model.
+- `gemini-2.5-computer-use-preview-10-2025`: An earlier computer use preview model.
 - `gemini-3-flash-preview`: The preview version of Gemini 3 Flash.
 
 ## Agent CLI
@@ -146,7 +147,7 @@ The `main.py` script is the command-line interface (CLI) for running the browser
 | `--env` | The computer use environment to use. Must be one of the following: `playwright`, or `browserbase` | No | N/A | All |
 | `--initial_url` | The initial URL to load when the browser starts. | No | https://www.google.com | All |
 | `--highlight_mouse` | If specified, the agent will attempt to highlight the mouse cursor's position in the screenshots. This is useful for visual debugging. | No | False (not highlighted) | `playwright` |
-| `--model` | The model to use. See the "Available Models" section for more information. | No | `gemini-2.5-computer-use-preview-10-2025` | All |
+| `--model` | The model to use. See the "Available Models" section for more information. | No | `gemini-3.5-flash` | All |
 
 ### Environment Variables
 

agent.py

@@ -31,7 +31,14 @@
 from computers import EnvState, Computer
 
 MAX_RECENT_TURN_WITH_SCREENSHOTS = 3
-PREDEFINED_COMPUTER_USE_FUNCTIONS = [
+LEGACY_COMPUTER_USE_MODELS = [
+    "gemini-2.5-computer-use-preview-10-2025",
+    "gemini-3-flash-preview",
+    "gemini-3.1-pro-preview",
+]
+
+# Legacy predefined functions, which are used in gemini-2.5-computer-use-preview-10-2025, gemini-3-flash-preview and gemini-3.1-pro-preview.
+LEGACY_PREDEFINED_COMPUTER_USE_FUNCTIONS = [
     "open_web_browser",
     "click_at",
     "hover_at",
@@ -47,6 +54,30 @@
     "drag_and_drop",
 ]
 
+# Predefined functions which are used in gemini-3.5-flash and future models.
+PREDEFINED_COMPUTER_USE_FUNCTIONS = [
+    "click",
+    "double_click",
+    "triple_click",
+    "middle_click",
+    "right_click",
+    "mouse_down",
+    "mouse_up",
+    "move",
+    "type",
+    "drag_and_drop",
+    "wait",
+    "press_key",
+    "key_down",
+    "key_up",
+    "hotkey",
+    "take_screenshot",
+    "scroll",
+    "go_back",
+    "navigate",
+    "go_forward",
+]
+
 
 console = Console()
 
@@ -87,6 +118,9 @@ def __init__(
                 ],
             )
         ]
+        self._use_legacy_computer_use_function_call = (
+            model_name in LEGACY_COMPUTER_USE_MODELS
+        )
 
         # Exclude any predefined functions here.
         excluded_predefined_functions = []
@@ -113,13 +147,133 @@ def __init__(
                 ),
                 types.Tool(function_declarations=custom_functions),
             ],
-            thinking_config=types.ThinkingConfig(
-                include_thoughts=True
-            ),
+            thinking_config=types.ThinkingConfig(include_thoughts=True),
         )
 
-    def handle_action(self, action: types.FunctionCall) -> FunctionResponseT:
+    def handle_action(
+        self, action: types.FunctionCall, use_legacy_actions: bool
+    ) -> FunctionResponseT:
         """Handles the action and returns the environment state."""
+        if use_legacy_actions:
+            return self.handle_legacy_action(action)
+
+        if action.name == "open_web_browser":
+            return self._browser_computer.open_web_browser()
+        elif action.name == "click":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.click_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "double_click":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.double_click_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "triple_click":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.triple_click_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "middle_click":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.middle_click_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "right_click":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.right_click_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "mouse_down":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.mouse_down(
+                x=x,
+                y=y,
+            )
+        elif action.name == "mouse_up":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.mouse_up(
+                x=x,
+                y=y,
+            )
+        elif action.name == "move":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.hover_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "type":
+            press_enter = action.args.get("press_enter", False)
+            return self._browser_computer.type_text(
+                text=action.args["text"],
+                press_enter=press_enter,
+            )
+        elif action.name == "scroll":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            magnitude = action.args.get("magnitude", 800)
+            direction = action.args["direction"]
+
+            if direction in ("up", "down"):
+                magnitude = self.denormalize_y(magnitude)
+            elif direction in ("left", "right"):
+                magnitude = self.denormalize_x(magnitude)
+            else:
+                raise ValueError("Unknown direction: ", direction)
+            return self._browser_computer.scroll_at(
+                x=x, y=y, direction=direction, magnitude=magnitude
+            )
+        elif action.name == "wait":
+            wait_seconds = int(action.args.get("seconds", 1))
+            return self._browser_computer.wait(wait_seconds)
+        elif action.name == "go_back":
+            return self._browser_computer.go_back()
+        elif action.name == "go_forward":
+            return self._browser_computer.go_forward()
+        elif action.name == "navigate":
+            return self._browser_computer.navigate(action.args["url"])
+        elif action.name == "hotkey":
+            return self._browser_computer.key_combination(action.args["keys"])
+        elif action.name == "press_key":
+            return self._browser_computer.press_key(action.args["key"])
+        elif action.name == "key_down":
+            return self._browser_computer.key_down(action.args["key"])
+        elif action.name == "key_up":
+            return self._browser_computer.key_up(action.args["key"])
+        elif action.name == "take_screenshot":
+            return self._browser_computer.take_screenshot()
+        elif action.name == "drag_and_drop":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            destination_x = self.denormalize_x(action.args["destination_x"])
+            destination_y = self.denormalize_y(action.args["destination_y"])
+            return self._browser_computer.drag_and_drop(
+                x=x,
+                y=y,
+                destination_x=destination_x,
+                destination_y=destination_y,
+            )
+        # Handle the custom function declarations here.
+        elif action.name == multiply_numbers.__name__:
+            return multiply_numbers(x=action.args["x"], y=action.args["y"])
+        else:
+            raise ValueError(f"Unsupported function: {action}")
+
+    def handle_legacy_action(self, action: types.FunctionCall) -> FunctionResponseT:
+        """Handles the action defined in the legacy models, and returns the environment state."""
         if action.name == "open_web_browser":
             return self._browser_computer.open_web_browser()
         elif action.name == "click_at":
@@ -167,6 +321,7 @@ def handle_action(self, action: types.FunctionCall) -> FunctionResponseT:
             )
         elif action.name == "wait_5_seconds":
             return self._browser_computer.wait_5_seconds()
+
         elif action.name == "go_back":
             return self._browser_computer.go_back()
         elif action.name == "go_forward":
@@ -264,8 +419,13 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]:
                 return "COMPLETE"
 
         if not response.candidates:
-            if response.prompt_feedback and response.prompt_feedback.block_reason == types.BlockReason.SAFETY:
-                raise ValueError(f"Response was blocked due to safety. Feedback: {response.prompt_feedback}")
+            if (
+                response.prompt_feedback
+                and response.prompt_feedback.block_reason == types.BlockReason.SAFETY
+            ):
+                raise ValueError(
+                    f"Response was blocked due to safety. Feedback: {response.prompt_feedback}"
+                )
             print("Response has no candidates!")
             print(response)
             raise ValueError("Empty response")
@@ -328,9 +488,13 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]:
                 with console.status(
                     "Sending command to Computer...", spinner_style=None
                 ):
-                    fc_result = self.handle_action(function_call)
+                    fc_result = self.handle_action(
+                        function_call, self._use_legacy_computer_use_function_call
+                    )
             else:
-                fc_result = self.handle_action(function_call)
+                fc_result = self.handle_action(
+                    function_call, self._use_legacy_computer_use_function_call
+                )
             if isinstance(fc_result, EnvState):
                 function_responses.append(
                     FunctionResponse(
@@ -371,7 +535,7 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]:
                         part.function_response
                         and part.function_response.parts
                         and part.function_response.name
-                        in PREDEFINED_COMPUTER_USE_FUNCTIONS
+                        in (PREDEFINED_COMPUTER_USE_FUNCTIONS + LEGACY_PREDEFINED_COMPUTER_USE_FUNCTIONS)
                     ):
                         has_screenshot = True
                         break
@@ -385,7 +549,7 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]:
                                 part.function_response
                                 and part.function_response.parts
                                 and part.function_response.name
-                                in PREDEFINED_COMPUTER_USE_FUNCTIONS
+                                in (PREDEFINED_COMPUTER_USE_FUNCTIONS + LEGACY_PREDEFINED_COMPUTER_USE_FUNCTIONS)
                             ):
                                 part.function_response.parts = None
 

computers/computer.py

@@ -39,6 +39,62 @@ def click_at(self, x: int, y: int) -> EnvState:
 
         The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
         """
+
+    @abc.abstractmethod
+    def double_click_at(self, x: int, y: int) -> EnvState:
+        """Double clicks at a specific x, y  coordinate on the webpage.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def triple_click_at(self, x: int, y: int) -> EnvState:
+        """Triple clicks at a specific x, y  coordinate on the webpage.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def middle_click_at(self, x: int, y: int) -> EnvState:
+        """Middle clicks at a specific x, y  coordinate on the webpage.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def right_click_at(self, x: int, y: int) -> EnvState:
+        """Right clicks at a specific x, y  coordinate on the webpage.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def mouse_down(self, x: int, y: int) -> EnvState:
+        """Mouse down at a specific x, y  coordinate on the webpage.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def mouse_up(self, x: int, y: int) -> EnvState:
+        """Mouse up at a specific x, y  coordinate on the webpage.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def type_text(self, text: str, press_enter: bool) -> EnvState:
+        """Type text.
+
+        set `press_enter` to True to let system automatically presses ENTER after typing.
+        """
+
+    @abc.abstractmethod
+    def wait(self, seconds: int) -> EnvState:
+        """Waits for a duration to allow unfinished webpage processes to complete.
+
+        set `seconds` to wait for a specific duration.
+        """
 
     @abc.abstractmethod
     def hover_at(self, x: int, y: int) -> EnvState:
@@ -112,6 +168,23 @@ def navigate(self, url: str) -> EnvState:
     def key_combination(self, keys: list[str]) -> EnvState:
         """Presses keyboard keys and combinations, such as "control+c" or "enter"."""
 
+    @abc.abstractmethod
+    def press_key(self, key:str) -> EnvState:
+        """Presses a keyboard key."""
+
+    @abc.abstractmethod
+    def key_down(self, key:str) -> EnvState:
+        """Presses down a keyboard key."""
+
+    @abc.abstractmethod
+    def key_up(self, key: str) -> EnvState:
+        """Releases a keyboard key."""
+
+    @abc.abstractmethod
+    def take_screenshot(self) -> EnvState:
+        """Take screenshot of the current webpage."""
+
+
     @abc.abstractmethod
     def drag_and_drop(
         self, x: int, y: int, destination_x: int, destination_y: int