diff --git a/README.md b/README.md index d9254e9..bd064c9 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Quick Start -This section will guide you through setting up and running the Computer Use Preview model, either the Gemini Developer API or Vertex AI. Follow these steps to get started. +This section will guide you through setting up and running Gemini Computer Use, using either the Gemini Developer API or Vertex AI. Follow these steps to get started. ### 1. Installation @@ -131,7 +131,8 @@ python main.py --query="Go to Google and type 'Hello World' into the search bar" You can choose the model to use by specifying the ```--model ``` flag. Available options on Gemini Developer API and Vertex AI Client: -- `gemini-2.5-computer-use-preview-10-2025`: This is the default model. +- `gemini-3.5-flash`: This is the default model. +- `gemini-2.5-computer-use-preview-10-2025`: An earlier computer use preview model. - `gemini-3-flash-preview`: The preview version of Gemini 3 Flash. ## Agent CLI @@ -146,7 +147,7 @@ The `main.py` script is the command-line interface (CLI) for running the browser | `--env` | The computer use environment to use. Must be one of the following: `playwright`, or `browserbase` | No | N/A | All | | `--initial_url` | The initial URL to load when the browser starts. | No | https://www.google.com | All | | `--highlight_mouse` | If specified, the agent will attempt to highlight the mouse cursor's position in the screenshots. This is useful for visual debugging. | No | False (not highlighted) | `playwright` | -| `--model` | The model to use. See the "Available Models" section for more information. | No | `gemini-2.5-computer-use-preview-10-2025` | All | +| `--model` | The model to use. See the "Available Models" section for more information. | No | `gemini-3.5-flash` | All | ### Environment Variables diff --git a/agent.py b/agent.py index d263e3a..8657da1 100644 --- a/agent.py +++ b/agent.py @@ -31,7 +31,14 @@ from computers import EnvState, Computer MAX_RECENT_TURN_WITH_SCREENSHOTS = 3 -PREDEFINED_COMPUTER_USE_FUNCTIONS = [ +LEGACY_COMPUTER_USE_MODELS = [ + "gemini-2.5-computer-use-preview-10-2025", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", +] + +# Legacy predefined functions, which are used in gemini-2.5-computer-use-preview-10-2025, gemini-3-flash-preview and gemini-3.1-pro-preview. +LEGACY_PREDEFINED_COMPUTER_USE_FUNCTIONS = [ "open_web_browser", "click_at", "hover_at", @@ -47,6 +54,30 @@ "drag_and_drop", ] +# Predefined functions which are used in gemini-3.5-flash and future models. +PREDEFINED_COMPUTER_USE_FUNCTIONS = [ + "click", + "double_click", + "triple_click", + "middle_click", + "right_click", + "mouse_down", + "mouse_up", + "move", + "type", + "drag_and_drop", + "wait", + "press_key", + "key_down", + "key_up", + "hotkey", + "take_screenshot", + "scroll", + "go_back", + "navigate", + "go_forward", +] + console = Console() @@ -87,6 +118,9 @@ def __init__( ], ) ] + self._use_legacy_computer_use_function_call = ( + model_name in LEGACY_COMPUTER_USE_MODELS + ) # Exclude any predefined functions here. excluded_predefined_functions = [] @@ -113,13 +147,133 @@ def __init__( ), types.Tool(function_declarations=custom_functions), ], - thinking_config=types.ThinkingConfig( - include_thoughts=True - ), + thinking_config=types.ThinkingConfig(include_thoughts=True), ) - def handle_action(self, action: types.FunctionCall) -> FunctionResponseT: + def handle_action( + self, action: types.FunctionCall, use_legacy_actions: bool + ) -> FunctionResponseT: """Handles the action and returns the environment state.""" + if use_legacy_actions: + return self.handle_legacy_action(action) + + if action.name == "open_web_browser": + return self._browser_computer.open_web_browser() + elif action.name == "click": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.click_at( + x=x, + y=y, + ) + elif action.name == "double_click": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.double_click_at( + x=x, + y=y, + ) + elif action.name == "triple_click": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.triple_click_at( + x=x, + y=y, + ) + elif action.name == "middle_click": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.middle_click_at( + x=x, + y=y, + ) + elif action.name == "right_click": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.right_click_at( + x=x, + y=y, + ) + elif action.name == "mouse_down": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.mouse_down( + x=x, + y=y, + ) + elif action.name == "mouse_up": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.mouse_up( + x=x, + y=y, + ) + elif action.name == "move": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + return self._browser_computer.hover_at( + x=x, + y=y, + ) + elif action.name == "type": + press_enter = action.args.get("press_enter", False) + return self._browser_computer.type_text( + text=action.args["text"], + press_enter=press_enter, + ) + elif action.name == "scroll": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + magnitude = action.args.get("magnitude", 800) + direction = action.args["direction"] + + if direction in ("up", "down"): + magnitude = self.denormalize_y(magnitude) + elif direction in ("left", "right"): + magnitude = self.denormalize_x(magnitude) + else: + raise ValueError("Unknown direction: ", direction) + return self._browser_computer.scroll_at( + x=x, y=y, direction=direction, magnitude=magnitude + ) + elif action.name == "wait": + wait_seconds = int(action.args.get("seconds", 1)) + return self._browser_computer.wait(wait_seconds) + elif action.name == "go_back": + return self._browser_computer.go_back() + elif action.name == "go_forward": + return self._browser_computer.go_forward() + elif action.name == "navigate": + return self._browser_computer.navigate(action.args["url"]) + elif action.name == "hotkey": + return self._browser_computer.key_combination(action.args["keys"]) + elif action.name == "press_key": + return self._browser_computer.press_key(action.args["key"]) + elif action.name == "key_down": + return self._browser_computer.key_down(action.args["key"]) + elif action.name == "key_up": + return self._browser_computer.key_up(action.args["key"]) + elif action.name == "take_screenshot": + return self._browser_computer.take_screenshot() + elif action.name == "drag_and_drop": + x = self.denormalize_x(action.args["x"]) + y = self.denormalize_y(action.args["y"]) + destination_x = self.denormalize_x(action.args["destination_x"]) + destination_y = self.denormalize_y(action.args["destination_y"]) + return self._browser_computer.drag_and_drop( + x=x, + y=y, + destination_x=destination_x, + destination_y=destination_y, + ) + # Handle the custom function declarations here. + elif action.name == multiply_numbers.__name__: + return multiply_numbers(x=action.args["x"], y=action.args["y"]) + else: + raise ValueError(f"Unsupported function: {action}") + + def handle_legacy_action(self, action: types.FunctionCall) -> FunctionResponseT: + """Handles the action defined in the legacy models, and returns the environment state.""" if action.name == "open_web_browser": return self._browser_computer.open_web_browser() elif action.name == "click_at": @@ -167,6 +321,7 @@ def handle_action(self, action: types.FunctionCall) -> FunctionResponseT: ) elif action.name == "wait_5_seconds": return self._browser_computer.wait_5_seconds() + elif action.name == "go_back": return self._browser_computer.go_back() elif action.name == "go_forward": @@ -264,8 +419,13 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]: return "COMPLETE" if not response.candidates: - if response.prompt_feedback and response.prompt_feedback.block_reason == types.BlockReason.SAFETY: - raise ValueError(f"Response was blocked due to safety. Feedback: {response.prompt_feedback}") + if ( + response.prompt_feedback + and response.prompt_feedback.block_reason == types.BlockReason.SAFETY + ): + raise ValueError( + f"Response was blocked due to safety. Feedback: {response.prompt_feedback}" + ) print("Response has no candidates!") print(response) raise ValueError("Empty response") @@ -328,9 +488,13 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]: with console.status( "Sending command to Computer...", spinner_style=None ): - fc_result = self.handle_action(function_call) + fc_result = self.handle_action( + function_call, self._use_legacy_computer_use_function_call + ) else: - fc_result = self.handle_action(function_call) + fc_result = self.handle_action( + function_call, self._use_legacy_computer_use_function_call + ) if isinstance(fc_result, EnvState): function_responses.append( FunctionResponse( @@ -371,7 +535,7 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]: part.function_response and part.function_response.parts and part.function_response.name - in PREDEFINED_COMPUTER_USE_FUNCTIONS + in (PREDEFINED_COMPUTER_USE_FUNCTIONS + LEGACY_PREDEFINED_COMPUTER_USE_FUNCTIONS) ): has_screenshot = True break @@ -385,7 +549,7 @@ def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]: part.function_response and part.function_response.parts and part.function_response.name - in PREDEFINED_COMPUTER_USE_FUNCTIONS + in (PREDEFINED_COMPUTER_USE_FUNCTIONS + LEGACY_PREDEFINED_COMPUTER_USE_FUNCTIONS) ): part.function_response.parts = None diff --git a/computers/computer.py b/computers/computer.py index 0ca3698..c9cfdb2 100644 --- a/computers/computer.py +++ b/computers/computer.py @@ -39,6 +39,62 @@ def click_at(self, x: int, y: int) -> EnvState: The 'x' and 'y' values are absolute values, scaled to the height and width of the screen. """ + + @abc.abstractmethod + def double_click_at(self, x: int, y: int) -> EnvState: + """Double clicks at a specific x, y coordinate on the webpage. + + The 'x' and 'y' values are absolute values, scaled to the height and width of the screen. + """ + + @abc.abstractmethod + def triple_click_at(self, x: int, y: int) -> EnvState: + """Triple clicks at a specific x, y coordinate on the webpage. + + The 'x' and 'y' values are absolute values, scaled to the height and width of the screen. + """ + + @abc.abstractmethod + def middle_click_at(self, x: int, y: int) -> EnvState: + """Middle clicks at a specific x, y coordinate on the webpage. + + The 'x' and 'y' values are absolute values, scaled to the height and width of the screen. + """ + + @abc.abstractmethod + def right_click_at(self, x: int, y: int) -> EnvState: + """Right clicks at a specific x, y coordinate on the webpage. + + The 'x' and 'y' values are absolute values, scaled to the height and width of the screen. + """ + + @abc.abstractmethod + def mouse_down(self, x: int, y: int) -> EnvState: + """Mouse down at a specific x, y coordinate on the webpage. + + The 'x' and 'y' values are absolute values, scaled to the height and width of the screen. + """ + + @abc.abstractmethod + def mouse_up(self, x: int, y: int) -> EnvState: + """Mouse up at a specific x, y coordinate on the webpage. + + The 'x' and 'y' values are absolute values, scaled to the height and width of the screen. + """ + + @abc.abstractmethod + def type_text(self, text: str, press_enter: bool) -> EnvState: + """Type text. + + set `press_enter` to True to let system automatically presses ENTER after typing. + """ + + @abc.abstractmethod + def wait(self, seconds: int) -> EnvState: + """Waits for a duration to allow unfinished webpage processes to complete. + + set `seconds` to wait for a specific duration. + """ @abc.abstractmethod def hover_at(self, x: int, y: int) -> EnvState: @@ -112,6 +168,23 @@ def navigate(self, url: str) -> EnvState: def key_combination(self, keys: list[str]) -> EnvState: """Presses keyboard keys and combinations, such as "control+c" or "enter".""" + @abc.abstractmethod + def press_key(self, key:str) -> EnvState: + """Presses a keyboard key.""" + + @abc.abstractmethod + def key_down(self, key:str) -> EnvState: + """Presses down a keyboard key.""" + + @abc.abstractmethod + def key_up(self, key: str) -> EnvState: + """Releases a keyboard key.""" + + @abc.abstractmethod + def take_screenshot(self) -> EnvState: + """Take screenshot of the current webpage.""" + + @abc.abstractmethod def drag_and_drop( self, x: int, y: int, destination_x: int, destination_y: int diff --git a/computers/playwright/playwright.py b/computers/playwright/playwright.py index e7a53c3..dcb2401 100644 --- a/computers/playwright/playwright.py +++ b/computers/playwright/playwright.py @@ -150,13 +150,65 @@ def __exit__(self, exc_type, exc_val, exc_tb): def open_web_browser(self) -> EnvState: return self.current_state() - def click_at(self, x: int, y: int): + def click_at(self, x: int, y: int) -> EnvState: self.highlight_mouse(x, y) self._page.mouse.click(x, y) self._page.wait_for_load_state() return self.current_state() + + def double_click_at(self, x: int, y: int) -> EnvState: + self.highlight_mouse(x, y) + self._page.mouse.dblclick(x, y) + self._page.wait_for_load_state() + return self.current_state() + + def triple_click_at(self, x: int, y: int) -> EnvState: + self.highlight_mouse(x, y) + self._page.mouse.click(x, y, click_count=3) + self._page.wait_for_load_state() + return self.current_state() + + def middle_click_at(self, x: int, y: int) -> EnvState: + self.highlight_mouse(x, y) + self._page.mouse.click(x, y, button="middle") + self._page.wait_for_load_state() + return self.current_state() + + def right_click_at(self, x: int, y: int) -> EnvState: + self.highlight_mouse(x, y) + self._page.mouse.click(x, y, button="right") + self._page.wait_for_load_state() + return self.current_state() + + def mouse_down(self, x: int, y: int) -> EnvState: + self.highlight_mouse(x, y) + self._page.mouse.move(x, y) + self._page.mouse.down() + self._page.wait_for_load_state() + return self.current_state() + + def mouse_up(self, x: int, y: int) -> EnvState: + self.highlight_mouse(x, y) + self._page.mouse.move(x, y) + self._page.mouse.up() + self._page.wait_for_load_state() + return self.current_state() + + def type_text(self, text: str, press_enter: bool = False) -> EnvState: + self._page.keyboard.type(text) + self._page.wait_for_load_state() - def hover_at(self, x: int, y: int): + if press_enter: + self.key_combination(["Enter"]) + self._page.wait_for_load_state() + return self.current_state() + + def wait(self, seconds: int = 1) -> EnvState: + self._page.wait_for_timeout(seconds * 1000) + return self.current_state() + + + def hover_at(self, x: int, y: int) -> EnvState: self.highlight_mouse(x, y) self._page.mouse.move(x, y) self._page.wait_for_load_state() @@ -282,6 +334,25 @@ def key_combination(self, keys: list[str]) -> EnvState: for key in reversed(keys[:-1]): self._page.keyboard.up(key) + self._page.wait_for_load_state() + return self.current_state() + + def press_key(self, key: str) -> EnvState: + return self.key_combination([key]) + + def key_down(self, key: str) -> EnvState: + key = PLAYWRIGHT_KEY_MAP.get(key.lower(), key) + self._page.keyboard.down(key) + self._page.wait_for_load_state() + return self.current_state() + + def key_up(self, key: str) -> EnvState: + key = PLAYWRIGHT_KEY_MAP.get(key.lower(), key) + self._page.keyboard.up(key) + self._page.wait_for_load_state() + return self.current_state() + + def take_screenshot(self) -> EnvState: return self.current_state() def drag_and_drop( diff --git a/main.py b/main.py index 848722d..1ccf502 100644 --- a/main.py +++ b/main.py @@ -54,7 +54,7 @@ def main() -> int: ) parser.add_argument( "--model", - default='gemini-2.5-computer-use-preview-10-2025', + default='gemini-3.5-flash', help="Set which main model to use.", ) args = parser.parse_args() diff --git a/requirements.txt b/requirements.txt index 321adef..0b617dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ termcolor==3.1.0 pydantic==2.12.0 -google-genai>=1.40.0 +google-genai>=2.7.0 playwright==1.55.0 browserbase==1.4.0 rich diff --git a/test_agent.py b/test_agent.py index c001493..a932b36 100644 --- a/test_agent.py +++ b/test_agent.py @@ -37,35 +37,35 @@ def test_multiply_numbers(self): def test_handle_action_open_web_browser(self): action = types.FunctionCall(name="open_web_browser", args={}) - self.agent.handle_action(action) + self.agent.handle_action(action, use_legacy_actions=True) self.mock_browser_computer.open_web_browser.assert_called_once() def test_handle_action_click_at(self): action = types.FunctionCall(name="click_at", args={"x": 100, "y": 200}) - self.agent.handle_action(action) + self.agent.handle_action(action, use_legacy_actions=True) self.mock_browser_computer.click_at.assert_called_once_with(x=100, y=200) def test_handle_action_type_text_at(self): action = types.FunctionCall(name="type_text_at", args={"x": 100, "y": 200, "text": "hello"}) - self.agent.handle_action(action) + self.agent.handle_action(action, use_legacy_actions=True) self.mock_browser_computer.type_text_at.assert_called_once_with( x=100, y=200, text="hello", press_enter=False, clear_before_typing=True ) def test_handle_action_scroll_document(self): action = types.FunctionCall(name="scroll_document", args={"direction": "down"}) - self.agent.handle_action(action) + self.agent.handle_action(action, use_legacy_actions=True) self.mock_browser_computer.scroll_document.assert_called_once_with("down") def test_handle_action_navigate(self): action = types.FunctionCall(name="navigate", args={"url": "https://example.com"}) - self.agent.handle_action(action) + self.agent.handle_action(action, use_legacy_actions=True) self.mock_browser_computer.navigate.assert_called_once_with("https://example.com") def test_handle_action_unknown_function(self): action = types.FunctionCall(name="unknown_function", args={}) with self.assertRaises(ValueError): - self.agent.handle_action(action) + self.agent.handle_action(action, use_legacy_actions=True) def test_denormalize_x(self): self.assertEqual(self.agent.denormalize_x(500), 500) @@ -103,7 +103,7 @@ def test_run_one_iteration_with_function_call(self, mock_handle_action, mock_get result = self.agent.run_one_iteration() self.assertEqual(result, "CONTINUE") - mock_handle_action.assert_called_once_with(function_call) + mock_handle_action.assert_called_once_with(function_call, False) self.assertEqual(len(self.agent._contents), 3)