diff --git a/backend/app/agent/factory/browser.py b/backend/app/agent/factory/browser.py index 6eb501da0..e73f48641 100644 --- a/backend/app/agent/factory/browser.py +++ b/backend/app/agent/factory/browser.py @@ -12,6 +12,7 @@ # limitations under the License. # ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +import asyncio import platform import threading import uuid @@ -148,6 +149,114 @@ def get_occupied_ports(self) -> list[int]: _cdp_pool_manager = CdpBrowserPoolManager() +class ExtensionTabPoolManager: + """Manages tab allocations within a shared extension proxy connection + for parallel agent tasks.""" + + def __init__(self): + self._occupied_tabs: dict[int, str] = {} # tabId -> session_id + self._session_to_tab: dict[str, int] = {} + self._session_to_task: dict[str, str | None] = {} + self._extension_proxy = None + self._lock = threading.Lock() + + def set_extension_proxy(self, proxy): + """Set the shared ExtensionProxyWrapper instance.""" + self._extension_proxy = proxy + + async def acquire_tab( + self, + session_id: str, + task_id: str | None = None, + url: str = "about:blank", + ) -> int | None: + """Create a new tab and assign it to a session.""" + if not self._extension_proxy: + logger.error("ExtensionTabPoolManager: no extension proxy set") + return None + try: + result = await self._extension_proxy.open_tab(url) + tab_id = result.get("tabId") + if tab_id is not None: + with self._lock: + self._occupied_tabs[tab_id] = session_id + self._session_to_tab[session_id] = tab_id + self._session_to_task[session_id] = task_id + logger.info( + f"Acquired tab {tab_id} for session {session_id}. " + f"Total tabs: {len(self._occupied_tabs)}" + ) + return tab_id + except Exception as e: + logger.error(f"Failed to acquire tab: {e}") + return None + + async def release_tab(self, tab_id: int, session_id: str): + """Close a tab and release it from the session.""" + with self._lock: + if ( + tab_id in self._occupied_tabs + and self._occupied_tabs[tab_id] == session_id + ): + del self._occupied_tabs[tab_id] + self._session_to_tab.pop(session_id, None) + self._session_to_task.pop(session_id, None) + else: + logger.warning( + f"Tab {tab_id} not occupied by session {session_id}" + ) + return + try: + if self._extension_proxy: + await self._extension_proxy.close_tab(tab_id) + logger.info( + f"Released tab {tab_id} from session {session_id}. " + f"Remaining tabs: {len(self._occupied_tabs)}" + ) + except Exception as e: + logger.error(f"Failed to close tab {tab_id}: {e}") + + async def release_by_task(self, task_id: str) -> list[int]: + """Release all tabs associated with a task_id.""" + released = [] + with self._lock: + sessions = [ + s for s, t in self._session_to_task.items() if t == task_id + ] + for session_id in sessions: + tab_id = self._session_to_tab.get(session_id) + if tab_id is not None: + released.append((tab_id, session_id)) + self._occupied_tabs.pop(tab_id, None) + self._session_to_tab.pop(session_id, None) + self._session_to_task.pop(session_id, None) + + for tab_id, _ in released: + try: + if self._extension_proxy: + await self._extension_proxy.close_tab(tab_id) + except Exception as e: + logger.error(f"Failed to close tab {tab_id}: {e}") + + if released: + logger.info(f"Released {len(released)} tab(s) for task {task_id}") + return [t for t, _ in released] + + +# Global extension tab pool manager instance +_tab_pool_manager = ExtensionTabPoolManager() + + +def _find_extension_proxy_browser( + cdp_browsers: list[dict], +) -> dict | None: + """Find the first extension proxy browser in the list.""" + for browser in cdp_browsers: + if browser.get("isExtensionProxy", False): + return browser + return None + + def browser_agent(options: Chat): working_directory = get_working_directory(options) logger.info( @@ -164,57 +273,114 @@ def browser_agent(options: Chat): toolkit_session_id = str(uuid.uuid4())[:8] selected_port = None selected_is_external = False + use_extension_proxy = False + extension_proxy_port = 8765 + # Check for extension proxy browser first + extension_proxy_browser = None if options.cdp_browsers: - selected_browser = _cdp_pool_manager.acquire_browser( - options.cdp_browsers, toolkit_session_id, options.task_id + extension_proxy_browser = _find_extension_proxy_browser( + options.cdp_browsers ) - if selected_browser: - selected_port = _get_browser_port(selected_browser) - selected_is_external = selected_browser.get("isExternal", False) - logger.info( - f"Acquired CDP browser from pool (initial): " - f"port={selected_port}, isExternal={selected_is_external}, " - f"session_id={toolkit_session_id}" + + if extension_proxy_browser: + # Extension proxy mode: use plugin exclusively, no CDP fallback + use_extension_proxy = True + extension_proxy_port = int(extension_proxy_browser.get("port", 8765)) + selected_is_external = True + logger.info( + f"Using extension proxy mode (exclusive): " + f"port={extension_proxy_port}, session_id={toolkit_session_id}" + ) + else: + # No extension proxy — use CDP browsers + cdp_only_browsers = [ + b + for b in (options.cdp_browsers or []) + if not b.get("isExtensionProxy", False) + ] + if cdp_only_browsers: + selected_browser = _cdp_pool_manager.acquire_browser( + cdp_only_browsers, toolkit_session_id, options.task_id ) + if selected_browser: + selected_port = _get_browser_port(selected_browser) + selected_is_external = selected_browser.get( + "isExternal", False + ) + logger.info( + f"Acquired CDP browser from pool (initial): " + f"port={selected_port}, " + f"isExternal={selected_is_external}, " + f"session_id={toolkit_session_id}" + ) + else: + selected_port = _get_browser_port(cdp_only_browsers[0]) + selected_is_external = cdp_only_browsers[0].get( + "isExternal", False + ) + logger.warning( + f"No available browsers in pool (initial), " + f"using first: port={selected_port}, " + f"session_id={toolkit_session_id}" + ) else: - selected_port = _get_browser_port(options.cdp_browsers[0]) - selected_is_external = options.cdp_browsers[0].get( - "isExternal", False - ) - logger.warning( - f"No available browsers in pool (initial), using first: " - f"port={selected_port}, session_id={toolkit_session_id}" + selected_port = env("browser_port", "9222") + + enabled_tools = [ + "browser_click", + "browser_type", + "browser_back", + "browser_forward", + "browser_select", + "browser_console_exec", + "browser_console_view", + "browser_switch_tab", + "browser_enter", + "browser_visit_page", + "browser_scroll", + "browser_sheet_read", + "browser_sheet_input", + "browser_get_page_snapshot", + "browser_open", + ] + + if use_extension_proxy: + web_toolkit_custom = HybridBrowserToolkit( + options.project_id, + cdp_keep_current_page=True, + headless=False, + browser_log_to_file=True, + stealth=True, + session_id=toolkit_session_id, + extension_proxy_mode=True, + extension_proxy_port=extension_proxy_port, + enabled_tools=enabled_tools, + ) + # Inject pre-started global wrapper if available + from app.service.extension_proxy_service import ( + get_extension_proxy_wrapper, + ) + + global_wrapper = get_extension_proxy_wrapper() + if global_wrapper is not None: + web_toolkit_custom._extension_proxy_wrapper = global_wrapper + _tab_pool_manager.set_extension_proxy(global_wrapper) + logger.info( + "Injected pre-started global ExtensionProxyWrapper " + "into toolkit" ) else: - selected_port = env("browser_port", "9222") - - web_toolkit_custom = HybridBrowserToolkit( - options.project_id, - cdp_keep_current_page=True, - headless=False, - browser_log_to_file=True, - stealth=True, - session_id=toolkit_session_id, - cdp_url=f"http://localhost:{selected_port}", - enabled_tools=[ - "browser_click", - "browser_type", - "browser_back", - "browser_forward", - "browser_select", - "browser_console_exec", - "browser_console_view", - "browser_switch_tab", - "browser_enter", - "browser_visit_page", - "browser_scroll", - "browser_sheet_read", - "browser_sheet_input", - "browser_get_page_snapshot", - "browser_open", - ], - ) + web_toolkit_custom = HybridBrowserToolkit( + options.project_id, + cdp_keep_current_page=True, + headless=False, + browser_log_to_file=True, + stealth=True, + session_id=toolkit_session_id, + cdp_url=f"http://localhost:{selected_port}", + enabled_tools=enabled_tools, + ) # Save reference before registering for toolkits_to_register_agent web_toolkit_for_agent_registration = web_toolkit_custom @@ -324,44 +490,98 @@ def browser_agent(options: Chat): enable_snapshot_clean=True, ) - # Attach CDP management callbacks and info to the agent - def acquire_cdp_for_agent(agent_instance): - """Acquire a CDP browser from pool for a cloned agent.""" - if not options.cdp_browsers: - return - session_id = str(uuid.uuid4())[:8] - selected = _cdp_pool_manager.acquire_browser( - options.cdp_browsers, session_id, options.task_id - ) - if selected: - agent_instance._cdp_port = _get_browser_port(selected) - else: - agent_instance._cdp_port = _get_browser_port( - options.cdp_browsers[0] + if use_extension_proxy: + # Extension proxy mode: tab-based parallelism + def acquire_tab_for_agent(agent_instance): + """Acquire a new tab from extension for a cloned agent.""" + session_id = str(uuid.uuid4())[:8] + try: + loop = asyncio.get_event_loop() + tab_id = loop.run_until_complete( + _tab_pool_manager.acquire_tab(session_id, options.task_id) + ) + except RuntimeError: + # If no event loop, create one + tab_id = asyncio.run( + _tab_pool_manager.acquire_tab(session_id, options.task_id) + ) + agent_instance._extension_tab_id = tab_id + agent_instance._cdp_session_id = session_id + agent_instance._cdp_port = extension_proxy_port + logger.info( + f"Acquired tab {tab_id} for cloned agent " + f"{agent_instance.agent_id}, session={session_id}" ) - agent_instance._cdp_session_id = session_id - logger.info( - f"Acquired CDP for cloned agent {agent_instance.agent_id}: " - f"port={agent_instance._cdp_port}, session={session_id}" - ) - def release_cdp_from_agent(agent_instance): - """Release CDP browser back to pool.""" - port = getattr(agent_instance, "_cdp_port", None) - session_id = getattr(agent_instance, "_cdp_session_id", None) - if port is not None and session_id is not None: - _cdp_pool_manager.release_browser(port, session_id) + def release_tab_from_agent(agent_instance): + """Release tab back to pool.""" + tab_id = getattr(agent_instance, "_extension_tab_id", None) + session_id = getattr(agent_instance, "_cdp_session_id", None) + if tab_id is not None and session_id is not None: + try: + loop = asyncio.get_event_loop() + loop.run_until_complete( + _tab_pool_manager.release_tab(tab_id, session_id) + ) + except RuntimeError: + asyncio.run( + _tab_pool_manager.release_tab(tab_id, session_id) + ) + logger.info( + f"Released tab {tab_id} for agent " + f"{agent_instance.agent_id}" + ) + + agent._cdp_acquire_callback = acquire_tab_for_agent + agent._cdp_release_callback = release_tab_from_agent + agent._cdp_port = extension_proxy_port + agent._cdp_session_id = toolkit_session_id + agent._cdp_task_id = options.task_id + agent._cdp_options = options + agent._browser_toolkit = web_toolkit_for_agent_registration + agent._use_extension_proxy = True + else: + # Standard CDP mode: port-based parallelism + def acquire_cdp_for_agent(agent_instance): + """Acquire a CDP browser from pool for a cloned agent.""" + cdp_only = [ + b + for b in (options.cdp_browsers or []) + if not b.get("isExtensionProxy", False) + ] + if not cdp_only: + return + session_id = str(uuid.uuid4())[:8] + selected = _cdp_pool_manager.acquire_browser( + cdp_only, session_id, options.task_id + ) + if selected: + agent_instance._cdp_port = _get_browser_port(selected) + else: + agent_instance._cdp_port = _get_browser_port(cdp_only[0]) + agent_instance._cdp_session_id = session_id logger.info( - f"Released CDP for agent {agent_instance.agent_id}: " - f"port={port}, session={session_id}" + f"Acquired CDP for cloned agent {agent_instance.agent_id}: " + f"port={agent_instance._cdp_port}, session={session_id}" ) - agent._cdp_acquire_callback = acquire_cdp_for_agent - agent._cdp_release_callback = release_cdp_from_agent - agent._cdp_port = selected_port - agent._cdp_session_id = toolkit_session_id - agent._cdp_task_id = options.task_id - agent._cdp_options = options - agent._browser_toolkit = web_toolkit_for_agent_registration + def release_cdp_from_agent(agent_instance): + """Release CDP browser back to pool.""" + port = getattr(agent_instance, "_cdp_port", None) + session_id = getattr(agent_instance, "_cdp_session_id", None) + if port is not None and session_id is not None: + _cdp_pool_manager.release_browser(port, session_id) + logger.info( + f"Released CDP for agent {agent_instance.agent_id}: " + f"port={port}, session={session_id}" + ) + + agent._cdp_acquire_callback = acquire_cdp_for_agent + agent._cdp_release_callback = release_cdp_from_agent + agent._cdp_port = selected_port + agent._cdp_session_id = toolkit_session_id + agent._cdp_task_id = options.task_id + agent._cdp_options = options + agent._browser_toolkit = web_toolkit_for_agent_registration return agent diff --git a/backend/app/agent/listen_chat_agent.py b/backend/app/agent/listen_chat_agent.py index 7fbaddc6a..c821cc9ae 100644 --- a/backend/app/agent/listen_chat_agent.py +++ b/backend/app/agent/listen_chat_agent.py @@ -709,7 +709,10 @@ def clone(self, with_memory: bool = False) -> ChatAgent: if has_cdp and hasattr(self, "_cdp_options"): options = self._cdp_options cdp_browsers = getattr(options, "cdp_browsers", []) - if cdp_browsers and hasattr(self, "_browser_toolkit"): + cdp_only_browsers = [ + b for b in cdp_browsers if not b.get("isExtensionProxy", False) + ] + if cdp_only_browsers and hasattr(self, "_browser_toolkit"): need_cdp_clone = True import uuid as _uuid @@ -717,7 +720,7 @@ def clone(self, with_memory: bool = False) -> ChatAgent: new_cdp_session = str(_uuid.uuid4())[:8] selected = _cdp_pool_manager.acquire_browser( - cdp_browsers, + cdp_only_browsers, new_cdp_session, getattr(self, "_cdp_task_id", None), ) @@ -726,7 +729,7 @@ def clone(self, with_memory: bool = False) -> ChatAgent: if selected: new_cdp_port = _get_browser_port(selected) else: - new_cdp_port = _get_browser_port(cdp_browsers[0]) + new_cdp_port = _get_browser_port(cdp_only_browsers[0]) if need_cdp_clone: # Temporarily override the browser toolkit's CDP URL. diff --git a/backend/app/agent/toolkit/hybrid_browser_toolkit.py b/backend/app/agent/toolkit/hybrid_browser_toolkit.py index 74209fe42..51f73597e 100644 --- a/backend/app/agent/toolkit/hybrid_browser_toolkit.py +++ b/backend/app/agent/toolkit/hybrid_browser_toolkit.py @@ -460,6 +460,9 @@ def __init__( cdp_url: str | None = "http://localhost:9222", cdp_keep_current_page: bool = False, full_visual_mode: bool = False, + extension_proxy_mode: bool = False, + extension_proxy_host: str = "localhost", + extension_proxy_port: int = 8765, ) -> None: logger.info( f"[HybridBrowserToolkit] Initializing with api_task_id: {api_task_id}" @@ -489,6 +492,13 @@ def __init__( logger.debug( f"[HybridBrowserToolkit] Calling super().__init__ with session_id: {session_id}" ) + + if extension_proxy_mode: + # In extension proxy mode, we don't connect via CDP + # Set cdp_url to None to prevent CDP connection attempts + cdp_url = None + connect_over_cdp = False + super().__init__( headless=headless, user_data_dir=user_data_dir, @@ -510,9 +520,13 @@ def __init__( cdp_url=cdp_url, cdp_keep_current_page=cdp_keep_current_page, full_visual_mode=full_visual_mode, + extension_proxy_mode=extension_proxy_mode, + extension_proxy_host=extension_proxy_host, + extension_proxy_port=extension_proxy_port, ) logger.info( f"[HybridBrowserToolkit] Initialization complete for api_task_id: {self.api_task_id}" + f"{' (extension proxy mode)' if extension_proxy_mode else ''}" ) async def _ensure_ws_wrapper(self): @@ -520,6 +534,45 @@ async def _ensure_ws_wrapper(self): logger.debug( f"[HybridBrowserToolkit] _ensure_ws_wrapper called for api_task_id: {getattr(self, 'api_task_id', 'NOT SET')}" ) + + # Extension proxy mode: use ExtensionProxyWrapper + if self._extension_proxy_mode: + if self._extension_proxy_wrapper is not None: + # Already have a shared wrapper (from clone or explicit set) + self._ws_wrapper = self._extension_proxy_wrapper + logger.info( + "[HybridBrowserToolkit] Using shared ExtensionProxyWrapper" + ) + return + + from camel.toolkits.hybrid_browser_toolkit.extension_proxy_wrapper import ( + ExtensionProxyWrapper, + ) + + session_id = self._ws_config.get("session_id", "default") + wrapper = ExtensionProxyWrapper( + config=self._ws_config, + host=self._extension_proxy_host, + port=self._extension_proxy_port, + ) + await wrapper.start() + logger.info( + f"[HybridBrowserToolkit] Extension proxy started on " + f"ws://{self._extension_proxy_host}:{self._extension_proxy_port}" + ) + + # Wait for Chrome extension to connect + connected = await wrapper.wait_for_connection(timeout=120.0) + if not connected: + raise RuntimeError( + "Timed out waiting for Chrome extension to connect" + ) + + self._extension_proxy_wrapper = wrapper + self._ws_wrapper = wrapper + return + + # Standard CDP mode global websocket_connection_pool # Get session ID from config or use default @@ -561,17 +614,46 @@ def clone_for_new_session( new_session_id = str(uuid.uuid4())[:8] # For cloned sessions, use the same user_data_dir to share login state - # This allows multiple agents to use the same browser profile without conflicts logger.info( f"Cloning session {new_session_id} with shared user_data_dir: {self._user_data_dir}" ) - # Use the same session_id to share the same browser instance - # This ensures all clones use the same WebSocket connection and browser + if self._extension_proxy_mode: + # Extension proxy mode: share the same wrapper connection + cloned = HybridBrowserToolkit( + self.api_task_id, + headless=self._headless, + user_data_dir=self._user_data_dir, + stealth=self._stealth, + cache_dir=f"{self._cache_dir.rstrip('/')}/_clone_{new_session_id}/", + enabled_tools=self.enabled_tools.copy(), + browser_log_to_file=self._browser_log_to_file, + log_dir=self.config_loader.get_toolkit_config().log_dir, + session_id=new_session_id, + default_start_url=self._default_start_url, + default_timeout=self._default_timeout, + short_timeout=self._short_timeout, + navigation_timeout=self._navigation_timeout, + network_idle_timeout=self._network_idle_timeout, + screenshot_timeout=self._screenshot_timeout, + page_stability_timeout=self._page_stability_timeout, + dom_content_loaded_timeout=self._dom_content_loaded_timeout, + viewport_limit=self._viewport_limit, + full_visual_mode=self._full_visual_mode, + extension_proxy_mode=True, + extension_proxy_host=self._extension_proxy_host, + extension_proxy_port=self._extension_proxy_port, + ) + # Share the existing ExtensionProxyWrapper connection + if self._extension_proxy_wrapper is not None: + cloned._extension_proxy_wrapper = self._extension_proxy_wrapper + return cloned + + # Standard CDP mode return HybridBrowserToolkit( self.api_task_id, headless=self._headless, - user_data_dir=self._user_data_dir, # Use the same user_data_dir + user_data_dir=self._user_data_dir, stealth=self._stealth, cache_dir=f"{self._cache_dir.rstrip('/')}/_clone_{new_session_id}/", enabled_tools=self.enabled_tools.copy(), diff --git a/backend/app/controller/chat_controller.py b/backend/app/controller/chat_controller.py index 87e42a70a..c487347ac 100644 --- a/backend/app/controller/chat_controller.py +++ b/backend/app/controller/chat_controller.py @@ -342,6 +342,7 @@ def improve(id: str, data: SupplementChat): data=ImprovePayload( question=data.question, attaches=data.attaches or [], + target=data.target, ), new_task_id=data.task_id, ) @@ -349,7 +350,7 @@ def improve(id: str, data: SupplementChat): ) chat_logger.info( "Improvement request queued with preserved context", - extra={"project_id": id}, + extra={"project_id": id, "target": data.target}, ) return Response(status_code=201) diff --git a/backend/app/controller/extension_proxy_controller.py b/backend/app/controller/extension_proxy_controller.py new file mode 100644 index 000000000..119cbf9ce --- /dev/null +++ b/backend/app/controller/extension_proxy_controller.py @@ -0,0 +1,81 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +import logging + +from fastapi import APIRouter +from pydantic import BaseModel + +from app.service.extension_proxy_service import ( + get_status, + start_extension_proxy, + stop_extension_proxy, +) + +logger = logging.getLogger("extension_proxy_controller") +router = APIRouter() + + +class StartProxyRequest(BaseModel): + host: str = "localhost" + port: int = 8765 + model_platform: str | None = None + model_type: str | None = None + api_key: str | None = None + api_url: str | None = None + extra_params: dict | None = None + + +@router.post("/extension-proxy/start", name="start extension proxy") +async def start_proxy(req: StartProxyRequest = None): + if req is None: + req = StartProxyRequest() + # Build model config if provided + model_config = None + if req.model_platform and req.model_type and req.api_key: + model_config = { + "model_platform": req.model_platform, + "model_type": req.model_type, + "api_key": req.api_key, + "api_url": req.api_url, + "extra_params": req.extra_params or {}, + } + + result = await start_extension_proxy( + host=req.host, + port=req.port, + model_config=model_config, + ) + return {"success": True, **result} + + +@router.post("/extension-proxy/stop", name="stop extension proxy") +async def stop_proxy(): + result = await stop_extension_proxy() + return {"success": True, **result} + + +@router.get("/extension-proxy/status", name="extension proxy status") +async def proxy_status(): + return {"status": get_status()} + + +@router.post( + "/extension-proxy/chat/clear", name="clear extension chat context" +) +async def clear_chat(): + from app.service.extension_chat_service import clear_chat_context + + await clear_chat_context() + return {"success": True} diff --git a/backend/app/model/chat.py b/backend/app/model/chat.py index 8f13a5aa4..4a7c60d16 100644 --- a/backend/app/model/chat.py +++ b/backend/app/model/chat.py @@ -78,6 +78,9 @@ class Chat(BaseModel): search_config: dict[str, str] | None = None # User identifier for user-specific skill configurations user_id: str | None = None + # Target agent for @mention routing: "browser", "dev", "doc", + # "media", "workforce", or None (default behavior) + target: str | None = None @field_validator("model_type") @classmethod @@ -141,6 +144,7 @@ class SupplementChat(BaseModel): question: str task_id: str | None = None attaches: list[str] = [] + target: str | None = None class HumanReply(BaseModel): diff --git a/backend/app/router.py b/backend/app/router.py index 2909eaae0..8e43e8acd 100644 --- a/backend/app/router.py +++ b/backend/app/router.py @@ -23,6 +23,7 @@ from app.controller import ( chat_controller, + extension_proxy_controller, health_controller, model_controller, task_controller, @@ -71,6 +72,11 @@ def register_routers(app: FastAPI, prefix: str = "") -> None: "tags": ["tool"], "description": "Tool installation and management", }, + { + "router": extension_proxy_controller.router, + "tags": ["extension-proxy"], + "description": "Extension proxy WebSocket server lifecycle", + }, ] app.include_router(health_controller.router, tags=["Health"]) diff --git a/backend/app/service/chat_service.py b/backend/app/service/chat_service.py index b0ea4e606..bda390eef 100644 --- a/backend/app/service/chat_service.py +++ b/backend/app/service/chat_service.py @@ -35,6 +35,7 @@ mcp_agent, multi_modal_agent, question_confirm_agent, + social_media_agent, task_summary_agent, ) from app.agent.listen_chat_agent import ListenChatAgent @@ -46,8 +47,10 @@ from app.model.chat import Chat, NewAgent, Status, TaskContent, sse_json from app.service.task import ( Action, + ActionAgentEndData, ActionDecomposeProgressData, ActionDecomposeTextData, + ActionEndData, ActionImproveData, ActionInstallMcpData, ActionNewAgent, @@ -284,6 +287,104 @@ def build_context_for_workforce( ) +# ================================================================ +# @mention direct agent routing +# ================================================================ + +# Maps @mention target names to (factory_fn, is_async) pairs +_AGENT_TARGET_MAP: dict[str, tuple] = { + "browser": (browser_agent, False), + "dev": (developer_agent, True), + "doc": (document_agent, True), + "media": (multi_modal_agent, False), + "social": (social_media_agent, True), +} + + +async def _create_persistent_agent( + target: str, options: Chat +) -> ListenChatAgent: + """Create a persistent agent by target name using existing factories.""" + if target not in _AGENT_TARGET_MAP: + raise ValueError( + f"Unknown agent target: {target}. " + f"Valid targets: {list(_AGENT_TARGET_MAP.keys())}" + ) + factory_fn, is_async = _AGENT_TARGET_MAP[target] + if is_async: + agent = await factory_fn(options) + else: + agent = await asyncio.to_thread(factory_fn, options) + logger.info( + f"[DIRECT-AGENT] Created persistent agent: {target}", + extra={ + "project_id": options.project_id, + "agent_name": getattr(agent, "agent_name", target), + "agent_id": getattr(agent, "agent_id", ""), + }, + ) + return agent + + +async def _run_direct_agent( + agent, + prompt: str, + question: str, + task_lock: TaskLock, +): + """Background task that runs a direct agent step. + + agent.astep() internally sends activate_agent and deactivate_agent + events via the queue, so step_solve's main loop can process them + in real-time alongside toolkit events. + + After completion, puts ActionEndData into the queue so step_solve + yields the 'end' SSE event. + """ + from camel.agents.chat_agent import ( + AsyncStreamingChatAgentResponse, + ) + + response_content = "" + try: + response = await agent.astep(prompt) + if isinstance(response, AsyncStreamingChatAgentResponse): + # Must consume the stream to trigger deactivation + # in _astream_chunks's finally block + async for chunk in response: + if chunk.msg and chunk.msg.content: + response_content += chunk.msg.content + else: + response_content = response.msg.content if response.msg else "" + except Exception as e: + logger.error( + f"[DIRECT-AGENT] Error executing agent: {e}", + exc_info=True, + ) + response_content = f"Error executing agent: {e}" + + # Save conversation history + task_lock.add_conversation("user", question) + task_lock.add_conversation("assistant", response_content) + + # Yield control so the agent's deactivate_agent event + # (scheduled via _schedule_async_task in _astream_chunks's + # finally block) fires before the end event. + # Without this, end arrives first and frontend ignores + # deactivate_agent because task is already FINISHED. + await asyncio.sleep(0.1) + + # Signal per-agent completion. step_solve's agent_end handler + # will emit the real "end" only when ALL agents are done. + await task_lock.put_queue( + ActionAgentEndData( + data=response_content, + agent_id=getattr(agent, "agent_id", ""), + agent_name=getattr(agent, "agent_name", ""), + ) + ) + + @sync_step async def step_solve(options: Chat, request: Request, task_lock: TaskLock): """Main task execution loop. Called when POST /chat endpoint @@ -469,6 +570,139 @@ async def step_solve(options: Chat, request: Request, task_lock: TaskLock): f"'{question[:100]}...'" ) + # --- @mention direct agent routing --- + target: str | None = None + if ( + hasattr(options, "target") + and options.target + and loop_iteration == 1 + ): + target = options.target + elif isinstance(item, ActionImproveData) and item.data.target: + target = item.data.target + + if ( + target + and target != "workforce" + and target in _AGENT_TARGET_MAP + ): + # Direct agent mode: keep the SAME task_id + # across turns (continuous chat in one chatStore). + # Do NOT update options.task_id here — the + # frontend reuses the existing chatStore. + + logger.info( + f"[DIRECT-AGENT] Routing to @{target}", + extra={ + "project_id": options.project_id, + "target": target, + "task_id": options.task_id, + }, + ) + + # Send confirmed event so frontend transitions + # from pending/splitting state. + # direct=True tells frontend to skip task + # splitting and go straight to RUNNING. + yield sse_json( + "confirmed", + {"question": question, "direct": True}, + ) + + # Ensure event loop is set for agent internals + set_main_event_loop(asyncio.get_running_loop()) + + # Get or create persistent agent + agent = task_lock.persistent_agents.get(target) + is_new_agent = agent is None + + logger.info( + f"[DIRECT-AGENT] persistent_agents" + f" keys: {list(task_lock.persistent_agents.keys())}," + f" is_new={is_new_agent}," + f" target={target}", + ) + + if is_new_agent: + # Factory internally sends create_agent + # via ActionCreateAgentData in the queue + agent = await _create_persistent_agent(target, options) + task_lock.persistent_agents[target] = agent + logger.info( + f"[DIRECT-AGENT] Created NEW " + f"agent: {agent.agent_name} " + f"(id={agent.agent_id})", + ) + else: + logger.info( + f"[DIRECT-AGENT] REUSING " + f"agent: {agent.agent_name} " + f"(id={agent.agent_id})", + ) + # Reused agent: factory won't send + # create_agent, so we must send it + # explicitly for the new chatStore + tool_names = [] + for t in getattr(agent, "tools", []): + fn_name = getattr(t, "get_function_name", None) + if fn_name: + tool_names.append(fn_name()) + yield sse_json( + "create_agent", + { + "agent_name": agent.agent_name, + "agent_id": agent.agent_id, + "tools": tool_names, + }, + ) + + # Each direct agent needs a unique process_task_id + # so toolkit events map to the correct agent panel. + agent.process_task_id = ( + f"{options.task_id}_{agent.agent_id}" + ) + + # New agents need prior conversation context + # injected into the prompt; reused agents + # already have it in CAMEL memory. + if is_new_agent: + conv_ctx = build_conversation_context( + task_lock, + header="=== Previous Conversation ===", + ) + prompt = f"{conv_ctx}\nUser: {question}" + else: + prompt = question + if attaches_to_use: + prompt += f"\n\nAttached files: {attaches_to_use}" + + # Launch agent in background task so step_solve's + # while loop can process queue events (activate, + # toolkit, deactivate) in real-time. + # agent.astep() internally sends activate_agent + # and deactivate_agent via the queue. + task = asyncio.create_task( + _run_direct_agent( + agent, + prompt, + question, + task_lock, + ) + ) + task_lock.add_background_task(task) + continue + + if target == "workforce": + logger.info( + "[DIRECT-AGENT] @workforce: " + "cleaning up persistent agents", + extra={ + "project_id": options.project_id, + }, + ) + await task_lock.cleanup_persistent_agents() + # --- end @mention routing --- + is_exceeded, total_length = check_conversation_history_length( task_lock ) @@ -1598,6 +1832,36 @@ def on_stream_text(chunk): }, ) + elif item.action == Action.agent_end: + # Per-agent completion for @mention direct chat. + # Only emit the real "end" when ALL agents finish. + agent_name = item.agent_name + agent_id = item.agent_id + remaining = len(task_lock.background_tasks) + logger.info( + f"[AGENT-END] Agent {agent_name} " + f"({agent_id}) finished. " + f"Remaining background tasks: {remaining}", + extra={ + "project_id": options.project_id, + }, + ) + + yield sse_json( + "agent_end", + { + "agent_id": agent_id, + "agent_name": agent_name, + "data": item.data or "", + }, + ) + + if remaining == 0: + # All agents done — trigger real end. + # Don't pass agent content as data; per-agent + # results are already sent via agent_end events. + await task_lock.put_queue(ActionEndData()) + elif item.action == Action.end: logger.info("=" * 80) logger.info( diff --git a/backend/app/service/extension_chat_service.py b/backend/app/service/extension_chat_service.py new file mode 100644 index 000000000..55b1fea70 --- /dev/null +++ b/backend/app/service/extension_chat_service.py @@ -0,0 +1,344 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +"""Lightweight chat service for Chrome extension. + +Runs a persistent browser agent that processes chat messages +received via the ExtensionProxyWrapper WebSocket. +Uses ChatAgent directly — no TaskLock, no SSE, no project infrastructure. +""" + +import asyncio +import logging +import os +from pathlib import Path +import platform +import uuid + +from camel.agents import ChatAgent +from camel.messages import BaseMessage +from camel.models import ModelFactory +from camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit_ts import ( + HybridBrowserToolkit, +) + +from app.agent.prompt import BROWSER_SYS_PROMPT +from app.agent.utils import NOW_STR + +logger = logging.getLogger("extension_chat_service") + +_chat_agent: ChatAgent | None = None +_chat_loop_task: asyncio.Task | None = None +_model_config: dict | None = None +_current_vision_mode: bool = False + + +def configure_model(config: dict): + """Store model config for agent creation. + + config keys: model_platform, model_type, api_key, api_url (optional), + extra_params (optional) + """ + global _model_config + _model_config = config + logger.info( + f"Extension chat model configured: " + f"{config.get('model_platform')}/{config.get('model_type')}" + ) + + +def _create_chat_agent(wrapper, full_visual_mode: bool = False) -> ChatAgent: + """Create a browser ChatAgent using the shared wrapper.""" + if not _model_config: + raise ValueError( + "Extension chat not configured. " + "Pass model_config when starting extension proxy." + ) + + # Setup camel LLM logging (same pattern as chat_controller) + log_dir = ( + Path.home() / ".eigent" / "qwe" / "extension_chat" / "camel_logs" + ) + log_dir.mkdir(parents=True, exist_ok=True) + os.environ["CAMEL_LOG_DIR"] = str(log_dir) + os.environ["CAMEL_MODEL_LOG_ENABLED"] = "true" + logger.info(f"CAMEL_LOG_DIR set to {log_dir}") + + extra_params = _model_config.get("extra_params", {}) + model_config_dict = {} + if extra_params: + model_config_dict.update(extra_params) + # Disable parallel tool calls for browser agent + model_config_dict["parallel_tool_calls"] = False + # Enable streaming for real-time token output + model_config_dict["stream"] = True + + model = ModelFactory.create( + model_platform=_model_config["model_platform"], + model_type=_model_config["model_type"], + api_key=_model_config["api_key"], + url=_model_config.get("api_url"), + model_config_dict=model_config_dict or None, + timeout=600, + ) + + # Use camel's HybridBrowserToolkit directly (not eigent's) + # to avoid @listen_toolkit / TaskLock dependency + session_id = str(uuid.uuid4())[:8] + + enabled_tools = [ + "browser_click", + "browser_type", + "browser_back", + "browser_forward", + "browser_console_exec", + "browser_console_view", + "browser_switch_tab", + "browser_enter", + "browser_visit_page", + "browser_scroll", + "browser_set_trigger", + ] + if full_visual_mode: + enabled_tools.append("browser_get_screenshot") + else: + enabled_tools.extend( + [ + "browser_select", + "browser_get_page_snapshot", + ] + ) + + browser_toolkit = HybridBrowserToolkit( + headless=False, + stealth=True, + session_id=session_id, + extension_proxy_mode=True, + extension_proxy_host=wrapper.host, + extension_proxy_port=wrapper.port, + full_visual_mode=full_visual_mode, + enabled_tools=enabled_tools, + ) + # Inject the shared wrapper + browser_toolkit._extension_proxy_wrapper = wrapper + + tools = browser_toolkit.get_tools() + + system_message = BROWSER_SYS_PROMPT.format( + platform_system=platform.system(), + platform_machine=platform.machine(), + working_directory="~", + now_str=NOW_STR, + external_browser_notice=( + "\n\n" + "You are connected to the user's Chrome browser via extension. " + "The browser is already open with active sessions and logged-in " + "websites.\n" + "\n" + "\n\n" + "You have a browser_set_trigger tool. When the user asks you to " + "wait for a condition (e.g., 'click the buy button when it becomes " + "available', 'do something at 12:00'), write a JavaScript arrow " + "function that returns true when the condition is met, and call " + "browser_set_trigger. You will be automatically notified when the " + "condition is met, then proceed with the requested action.\n" + "\n" + ), + ) + + agent = ChatAgent( + system_message=BaseMessage.make_assistant_message( + role_name="Browser Agent", + content=system_message, + ), + model=model, + tools=tools, + message_window_size=20, + step_timeout=None, + ) + logger.info( + f"Extension chat agent created (full_visual_mode={full_visual_mode})" + ) + return agent + + +async def _process_chat_message( + wrapper, message: str, full_vision_mode: bool = False +): + """Run one agent turn and stream results back via WebSocket.""" + global _chat_agent, _current_vision_mode + + # Recreate agent if vision mode changed + if _chat_agent is not None and full_vision_mode != _current_vision_mode: + logger.info( + f"Vision mode changed to {full_vision_mode}, recreating agent" + ) + _chat_agent = None + + if _chat_agent is None: + try: + _chat_agent = _create_chat_agent( + wrapper, full_visual_mode=full_vision_mode + ) + _current_vision_mode = full_vision_mode + except Exception as e: + logger.error(f"Failed to create chat agent: {e}") + await wrapper.send_chat_response("TASK_ERROR", {"error": str(e)}) + return + + await wrapper.send_chat_response("STREAM_START", {}) + + try: + user_msg = BaseMessage.make_user_message( + role_name="User", content=message + ) + response = await _chat_agent.astep(user_msg) + + # Stream mode: async iterate over partial responses + full_text = "" + seen_tool_calls = set() + + async for chunk in response: + # Send text delta + delta = chunk.msg.content if chunk.msg else "" + if delta: + full_text += delta + await wrapper.send_chat_response( + "STREAM_TEXT", {"text": delta} + ) + + # Send tool call info as actions (deduplicate) + tool_calls = chunk.info.get("tool_calls", []) + for tc in tool_calls: + tc_id = id(tc) + if tc_id in seen_tool_calls: + continue + seen_tool_calls.add(tc_id) + func_name = getattr(tc, "func_name", str(tc)) + await wrapper.send_chat_response( + "ACTION", + { + "action": func_name, + "detail": str(getattr(tc, "args", {}))[:200], + }, + ) + result = getattr(tc, "result", "") + success = not str(result).startswith("Error") + await wrapper.send_chat_response( + "ACTION_COMPLETE", + { + "success": success, + "result": str(result)[:500], + }, + ) + + await wrapper.send_chat_response("STREAM_END", {}) + # Don't re-send full_text — it was already streamed via + # STREAM_TEXT chunks. Sending it again in TASK_COMPLETE causes + # duplicate text in the UI. + await wrapper.send_chat_response("TASK_COMPLETE", {}) + + except Exception as e: + logger.error(f"Chat agent error: {e}", exc_info=True) + await wrapper.send_chat_response("TASK_ERROR", {"error": str(e)}) + + +async def _chat_loop(wrapper): + """Main loop: wait for chat messages, process them.""" + logger.info("Extension chat loop started") + while True: + try: + msg_data = await wrapper.wait_for_chat_message() + if msg_data is None: + continue + + msg_type = msg_data.get("type", "CHAT_MESSAGE") + + if msg_type == "CLEAR_CONTEXT": + await clear_chat_context() + continue + + if msg_type == "TRIGGER_FIRED": + description = msg_data.get("description", "unknown") + trigger_msg = ( + f"[TRIGGER FIRED] Your trigger has been activated: " + f"{description}. Please proceed with the next action." + ) + logger.info(f"Trigger fired: {description}") + await _process_chat_message( + wrapper, + trigger_msg, + full_vision_mode=_current_vision_mode, + ) + continue + + message = msg_data.get("message", "") + full_vision = msg_data.get("fullVisionMode", False) + if message.strip(): + # Prepend current page context + page_url = msg_data.get("url", "") + page_title = msg_data.get("pageTitle", "") + if page_url: + context = ( + f"[Current page: {page_title} | {page_url}]\n" + if page_title + else f"[Current page: {page_url}]\n" + ) + message = context + message + + logger.info( + f"Processing chat message " + f"(vision={full_vision}): {message[:100]}" + ) + await _process_chat_message( + wrapper, message, full_vision_mode=full_vision + ) + + except asyncio.CancelledError: + logger.info("Extension chat loop cancelled") + break + except Exception as e: + logger.error(f"Chat loop error: {e}", exc_info=True) + await asyncio.sleep(1) + + +async def start_chat_loop(wrapper): + """Start the chat processing loop.""" + global _chat_loop_task + if _chat_loop_task is not None and not _chat_loop_task.done(): + logger.info("Chat loop already running") + return + _chat_loop_task = asyncio.create_task(_chat_loop(wrapper)) + + +async def stop_chat_loop(): + """Stop the chat loop and clear the agent.""" + global _chat_loop_task, _chat_agent + if _chat_loop_task is not None: + _chat_loop_task.cancel() + try: + await _chat_loop_task + except asyncio.CancelledError: + pass + _chat_loop_task = None + _chat_agent = None + logger.info("Extension chat loop stopped") + + +async def clear_chat_context(): + """Reset the agent (clear conversation history).""" + global _chat_agent + if _chat_agent is not None: + _chat_agent.reset() + logger.info("Extension chat agent context cleared") diff --git a/backend/app/service/extension_proxy_service.py b/backend/app/service/extension_proxy_service.py new file mode 100644 index 000000000..0d74bda77 --- /dev/null +++ b/backend/app/service/extension_proxy_service.py @@ -0,0 +1,151 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +"""Global singleton service managing ExtensionProxyWrapper lifecycle. + +Start/stop the WebSocket server from the Extension settings page, +and provide the running wrapper instance to browser_agent(). +""" + +import asyncio +import logging + +logger = logging.getLogger("extension_proxy_service") + +_wrapper = None # ExtensionProxyWrapper | None +_wrapper_status = "stopped" # "stopped" | "waiting" | "connected" + + +async def start_extension_proxy( + host: str = "localhost", + port: int = 8765, + model_config: dict | None = None, +) -> dict: + """Start the WebSocket server and wait for extension connection. + + Args: + host: WebSocket server host. + port: WebSocket server port. + model_config: LLM model config for extension chat. + Keys: model_platform, model_type, api_key, api_url, extra_params + """ + global _wrapper, _wrapper_status + + if _wrapper is not None: + return {"status": get_status()} + + # Store model config for extension chat + if model_config: + from app.service.extension_chat_service import configure_model + + configure_model(model_config) + + from camel.toolkits.hybrid_browser_toolkit.extension_proxy_wrapper import ( + ExtensionProxyWrapper, + ) + + _wrapper = ExtensionProxyWrapper(config={}, host=host, port=port) + await _wrapper.start() + _wrapper_status = "waiting" + logger.info(f"Extension proxy server started on ws://{host}:{port}") + + asyncio.create_task(_watch_connection()) + return {"status": "waiting"} + + +async def _watch_connection(): + """Background task: update status when extension connects, monitor for disconnection.""" + global _wrapper_status + if _wrapper is None: + return + try: + connected = await _wrapper.wait_for_connection(timeout=600.0) + if connected: + _wrapper_status = "connected" + logger.info("Chrome extension connected to proxy") + + # Start chat loop for extension chat + from app.service.extension_chat_service import start_chat_loop + + await start_chat_loop(_wrapper) + + # Monitor for disconnection + while _wrapper is not None: + await asyncio.sleep(2) + if _wrapper is None: + break + if _wrapper._client is None: + _wrapper_status = "waiting" + logger.warning( + "Chrome extension disconnected, " + "waiting for reconnection..." + ) + # Wait for reconnection + try: + reconnected = await _wrapper.wait_for_connection( + timeout=600.0 + ) + if reconnected: + _wrapper_status = "connected" + logger.info( + "Chrome extension reconnected to proxy" + ) + else: + logger.warning( + "Timed out waiting for extension reconnection" + ) + except Exception as e: + logger.error(f"Error waiting for reconnection: {e}") + break + else: + logger.warning("Timed out waiting for extension connection") + except Exception as e: + logger.error(f"Error watching extension connection: {e}") + + +async def stop_extension_proxy() -> dict: + """Stop the WebSocket server and clear the singleton.""" + global _wrapper, _wrapper_status + + # Stop chat loop first + from app.service.extension_chat_service import stop_chat_loop + + await stop_chat_loop() + + if _wrapper is not None: + try: + await _wrapper.stop() + except Exception as e: + logger.error(f"Error stopping extension proxy: {e}") + _wrapper = None + _wrapper_status = "stopped" + return {"status": "stopped"} + + +def get_extension_proxy_wrapper(): + """Get the running wrapper instance, or None.""" + return _wrapper + + +def get_status() -> str: + """Get current status: stopped, waiting, or connected.""" + global _wrapper_status + if _wrapper is None: + _wrapper_status = "stopped" + elif _wrapper._client is not None: + _wrapper_status = "connected" + elif _wrapper_status == "connected": + # Was connected but client is gone now + _wrapper_status = "waiting" + return _wrapper_status diff --git a/backend/app/service/task.py b/backend/app/service/task.py index 604fbc717..53d640dd7 100644 --- a/backend/app/service/task.py +++ b/backend/app/service/task.py @@ -58,6 +58,7 @@ class Action(str, Enum): search_mcp = "search_mcp" # backend -> user install_mcp = "install_mcp" # backend -> user terminal = "terminal" # backend -> user + agent_end = "agent_end" # backend -> user (single agent finished) end = "end" # backend -> user stop = "stop" # user -> backend supplement = "supplement" # user -> backend @@ -76,6 +77,7 @@ class ImprovePayload(BaseModel): question: str attaches: list[str] = [] + target: str | None = None class ActionImproveData(BaseModel): @@ -223,8 +225,18 @@ class ActionStopData(BaseModel): action: Literal[Action.stop] = Action.stop +class ActionAgentEndData(BaseModel): + """Per-agent completion signal for @mention direct chat.""" + + action: Literal[Action.agent_end] = Action.agent_end + data: str | None = None + agent_id: str = "" + agent_name: str = "" + + class ActionEndData(BaseModel): action: Literal[Action.end] = Action.end + data: str | None = None class ActionTimeoutData(BaseModel): @@ -351,6 +363,8 @@ class TaskLock: """Track if summary has been generated for this project""" current_task_id: str | None """Current task ID to be used in SSE responses""" + persistent_agents: dict[str, Any] + """Persistent agents for @mention direct chat (key: agent type name)""" def __init__( self, id: str, queue: asyncio.Queue, human_input: dict @@ -369,6 +383,7 @@ def __init__( self.last_task_summary = "" self.question_agent = None self.current_task_id = None + self.persistent_agents = {} logger.info( "Task lock initialized", @@ -426,6 +441,35 @@ def add_background_task(self, task: asyncio.Task) -> None: self.background_tasks.add(task) task.add_done_callback(lambda t: self.background_tasks.discard(t)) + async def cleanup_persistent_agents(self): + r"""Release all persistent agents and their resources (e.g. CDP).""" + if not self.persistent_agents: + return + logger.info( + "Cleaning up persistent agents", + extra={ + "task_id": self.id, + "agents": list(self.persistent_agents.keys()), + }, + ) + for name, agent in self.persistent_agents.items(): + try: + if ( + hasattr(agent, "_cdp_release_callback") + and agent._cdp_release_callback + ): + agent._cdp_release_callback(agent) + logger.info( + f"Released CDP for persistent agent {name}", + extra={"task_id": self.id}, + ) + except Exception as e: + logger.warning( + f"Failed to release CDP for persistent agent {name}: {e}", + extra={"task_id": self.id}, + ) + self.persistent_agents.clear() + async def cleanup(self): r"""Cancel all background tasks and clean up resources""" logger.info( @@ -435,6 +479,10 @@ async def cleanup(self): "background_tasks_count": len(self.background_tasks), }, ) + + # Clean up persistent agents first + await self.cleanup_persistent_agents() + for task in list(self.background_tasks): if not task.done(): task.cancel() diff --git a/backend/app/utils/standard_env.py b/backend/app/utils/standard_env.py new file mode 100644 index 000000000..303ee76fe --- /dev/null +++ b/backend/app/utils/standard_env.py @@ -0,0 +1,372 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +""" +Standard Python Environment Management + +This module provides utilities for managing a pre-configured Python environment +with common packages (pandas, numpy, matplotlib, requests, openpyxl). + +The standard environment is created once at ~/.eigent/standard_env/ and copied +to working directories as needed. This avoids race conditions when multiple +agents try to create environments simultaneously. +""" + +import os +import platform +import shutil +import subprocess +import sys +import threading +from collections.abc import Callable +from pathlib import Path + +from utils import traceroot_wrapper as traceroot + +logger = traceroot.get_logger("standard_env") + +# Standard packages to install in the environment +STANDARD_PACKAGES = [ + "pandas", + "numpy", + "matplotlib", + "requests", + "openpyxl", +] + +# Lock for thread-safe environment operations +_env_lock = threading.Lock() + +# Cache for environment paths to avoid repeated checks +_env_cache: dict[str, str] = {} + + +def get_standard_env_base_path() -> Path: + """Get the base path for the standard environment.""" + return Path.home() / ".eigent" / "standard_env" + + +def get_standard_env_marker_file(env_path: Path) -> Path: + """Get the path to the marker file that indicates a complete environment.""" + return env_path / ".env_complete" + + +def _get_python_executable(env_path: Path) -> str: + """Get the Python executable path for an environment.""" + if platform.system() == "Windows": + return str(env_path / "Scripts" / "python.exe") + return str(env_path / "bin" / "python") + + +def _get_uv_path() -> str | None: + """Get the path to uv if available.""" + try: + result = subprocess.run( + ["which", "uv"] + if platform.system() != "Windows" + else ["where", "uv"], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0: + return result.stdout.strip().split("\n")[0] + except Exception: + pass + return None + + +def ensure_standard_environment( + update_callback: Callable[[str], None] | None = None, +) -> str | None: + """ + Ensure the standard environment exists with all required packages. + + Creates the environment at ~/.eigent/standard_env/ if it doesn't exist. + This is a one-time setup that can be done at application startup. + + Args: + update_callback: Optional callback for progress updates. + + Returns: + Path to the standard environment, or None if creation failed. + """ + env_path = get_standard_env_base_path() + marker_file = get_standard_env_marker_file(env_path) + + # Fast path: if marker exists, environment is ready + if marker_file.exists(): + logger.debug(f"Standard environment already exists at {env_path}") + return str(env_path) + + with _env_lock: + # Double-check after acquiring lock + if marker_file.exists(): + return str(env_path) + + if update_callback: + update_callback(f"Creating standard environment at {env_path}\n") + + logger.info(f"Creating standard environment at {env_path}") + + try: + # Remove incomplete environment if exists + if env_path.exists(): + logger.info("Removing incomplete standard environment") + shutil.rmtree(env_path) + + env_path.mkdir(parents=True, exist_ok=True) + + current_version = ( + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + uv_path = _get_uv_path() + + if uv_path: + # Use uv for faster environment creation + if update_callback: + update_callback("Using uv to create environment...\n") + + # Create venv with uv + subprocess.run( + [ + uv_path, + "venv", + "--python", + current_version, + str(env_path), + ], + check=True, + capture_output=True, + timeout=300, + ) + + python_path = _get_python_executable(env_path) + + # Install pip, setuptools, wheel + subprocess.run( + [ + uv_path, + "pip", + "install", + "--python", + python_path, + "pip", + "setuptools", + "wheel", + ], + check=True, + capture_output=True, + timeout=300, + ) + + # Install standard packages + if update_callback: + update_callback( + f"Installing packages: {', '.join(STANDARD_PACKAGES)}\n" + ) + + subprocess.run( + [uv_path, "pip", "install", "--python", python_path] + + STANDARD_PACKAGES, + check=True, + capture_output=True, + timeout=600, + ) + else: + # Fallback to standard venv + if update_callback: + update_callback( + "Using standard venv to create environment...\n" + ) + + import venv + + venv.create(str(env_path), with_pip=True) + + python_path = _get_python_executable(env_path) + + # Upgrade pip + subprocess.run( + [python_path, "-m", "pip", "install", "--upgrade", "pip"], + check=True, + capture_output=True, + timeout=300, + ) + + # Install standard packages + if update_callback: + update_callback( + f"Installing packages: {', '.join(STANDARD_PACKAGES)}\n" + ) + + subprocess.run( + [python_path, "-m", "pip", "install", "--upgrade"] + + STANDARD_PACKAGES, + check=True, + capture_output=True, + timeout=600, + ) + + # Create marker file to indicate completion + marker_file.touch() + + if update_callback: + update_callback("Standard environment created successfully!\n") + + logger.info( + f"Standard environment created successfully at {env_path}" + ) + return str(env_path) + + except Exception as e: + logger.error( + f"Failed to create standard environment: {e}", exc_info=True + ) + # Clean up on failure + if env_path.exists(): + try: + shutil.rmtree(env_path) + except Exception: + pass + return None + + +def copy_standard_env_to_working_dir( + working_dir: str, update_callback: Callable[[str], None] | None = None +) -> str | None: + """ + Copy the standard environment to a working directory. + + This function is thread-safe and will only copy once per working directory. + If the environment already exists in the working directory, it returns + immediately. + + Args: + working_dir: The working directory to copy the environment to. + update_callback: Optional callback for progress updates. + + Returns: + Path to the copied environment's Python executable, or None if failed. + """ + target_env_path = Path(working_dir) / ".venv" + target_marker = get_standard_env_marker_file(target_env_path) + + # Fast path: if environment already exists and is complete + if target_marker.exists(): + python_path = _get_python_executable(target_env_path) + if os.path.exists(python_path): + logger.debug(f"Using existing environment at {target_env_path}") + return python_path + + # Check cache + cache_key = str(target_env_path) + if cache_key in _env_cache: + cached_path = _env_cache[cache_key] + if os.path.exists(cached_path): + return cached_path + + with _env_lock: + # Double-check after acquiring lock + if target_marker.exists(): + python_path = _get_python_executable(target_env_path) + if os.path.exists(python_path): + _env_cache[cache_key] = python_path + return python_path + + # Ensure standard environment exists + standard_env_path = ensure_standard_environment(update_callback) + if not standard_env_path: + logger.error("Failed to ensure standard environment exists") + return None + + if update_callback: + update_callback(f"Copying environment to {working_dir}\n") + + logger.info(f"Copying standard environment to {target_env_path}") + + try: + # Ensure working directory exists + Path(working_dir).mkdir(parents=True, exist_ok=True) + + # Remove incomplete target if exists + if target_env_path.exists(): + shutil.rmtree(target_env_path) + + # Copy the environment + # Use copytree with symlinks=True to preserve symlinks (faster and smaller) + shutil.copytree( + standard_env_path, + target_env_path, + symlinks=True, + ignore_dangling_symlinks=True, + ) + + # Update pyvenv.cfg if it exists (important for venv relocatability) + pyvenv_cfg = target_env_path / "pyvenv.cfg" + if pyvenv_cfg.exists(): + # For most use cases, symlinked venvs work fine after copy + # But we can update the home path if needed + pass + + # Create marker file + target_marker.touch() + + python_path = _get_python_executable(target_env_path) + _env_cache[cache_key] = python_path + + if update_callback: + update_callback("Environment copied successfully!\n") + + logger.info(f"Environment copied to {target_env_path}") + return python_path + + except Exception as e: + logger.error(f"Failed to copy environment: {e}", exc_info=True) + # Clean up on failure + if target_env_path.exists(): + try: + shutil.rmtree(target_env_path) + except Exception: + pass + return None + + +def get_env_python_executable(working_dir: str) -> str | None: + """ + Get the Python executable for a working directory's environment. + + This assumes the environment has already been set up via + copy_standard_env_to_working_dir(). + + Args: + working_dir: The working directory. + + Returns: + Path to the Python executable, or None if not found. + """ + env_path = Path(working_dir) / ".venv" + python_path = _get_python_executable(env_path) + + if os.path.exists(python_path): + return python_path + + return None + + +def cleanup_env_cache(): + """Clear the environment path cache.""" + global _env_cache + with _env_lock: + _env_cache.clear() diff --git a/electron/main/index.ts b/electron/main/index.ts index 1b9d5da40..8d1b0e1b8 100644 --- a/electron/main/index.ts +++ b/electron/main/index.ts @@ -95,6 +95,7 @@ interface CdpBrowser { id: string; port: number; isExternal: boolean; + isExtensionProxy?: boolean; name?: string; addedAt: number; } @@ -161,7 +162,9 @@ async function runPoolHealthCheck(): Promise { // Probe a snapshot so add/remove IPC handlers can run safely in parallel. const snapshot = [...cdp_browser_pool]; const results = await Promise.all( - snapshot.map((b) => isCdpPortAlive(b.port)) + snapshot.map((b) => + b.isExtensionProxy ? Promise.resolve(true) : isCdpPortAlive(b.port) + ) ); const deadIds = snapshot .filter((_, idx) => !results[idx]) @@ -699,7 +702,13 @@ function registerIpcHandlers() { // Add browser to pool ipcMain.handle( 'add-cdp-browser', - (event, port: number, isExternal: boolean, name?: string) => { + ( + event, + port: number, + isExternal: boolean, + name?: string, + isExtensionProxy?: boolean + ) => { const existing = cdp_browser_pool.find((b) => b.port === port); if (existing) { log.warn( @@ -715,6 +724,7 @@ function registerIpcHandlers() { id: `cdp-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, port, isExternal, + isExtensionProxy: isExtensionProxy || false, name, addedAt: Date.now(), }; @@ -742,8 +752,8 @@ function registerIpcHandlers() { const removed = cdp_browser_pool.splice(index, 1)[0]; - // Close the browser via CDP (best-effort) - if (closeBrowser) { + // Close the browser via CDP (best-effort, skip for extension proxy) + if (closeBrowser && !removed.isExtensionProxy) { await closeBrowserViaCdp(removed.port); } diff --git a/electron/preload/index.ts b/electron/preload/index.ts index 910a670d7..2c301d843 100644 --- a/electron/preload/index.ts +++ b/electron/preload/index.ts @@ -162,8 +162,19 @@ contextBridge.exposeInMainWorld('electronAPI', { ipcRenderer.invoke('set-browser-port', port, isExternal), getBrowserPort: () => ipcRenderer.invoke('get-browser-port'), getCdpBrowsers: () => ipcRenderer.invoke('get-cdp-browsers'), - addCdpBrowser: (port: number, isExternal: boolean, name?: string) => - ipcRenderer.invoke('add-cdp-browser', port, isExternal, name), + addCdpBrowser: ( + port: number, + isExternal: boolean, + name?: string, + isExtensionProxy?: boolean + ) => + ipcRenderer.invoke( + 'add-cdp-browser', + port, + isExternal, + name, + isExtensionProxy + ), removeCdpBrowser: (browserId: string, closeBrowser?: boolean) => ipcRenderer.invoke('remove-cdp-browser', browserId, closeBrowser ?? true), launchCdpBrowser: () => ipcRenderer.invoke('launch-cdp-browser'), diff --git a/eslint.config.js b/eslint.config.js index fa8839466..fd85d7c3d 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -70,6 +70,8 @@ export default [ '**/.venv/**', // Prebuilt resources 'resources/prebuilt/**', + // Chrome extension (standalone, uses chrome/browser globals) + 'extensions/chrome_extension/**', ], }, diff --git a/extensions/chrome_extension/background.js b/extensions/chrome_extension/background.js new file mode 100644 index 000000000..b08d1ba82 --- /dev/null +++ b/extensions/chrome_extension/background.js @@ -0,0 +1,999 @@ +// WebSocket connection +let ws = null; +let isConnected = false; +let serverUrl = 'ws://localhost:8765'; +let fullVisionMode = false; + +// Auto-reconnect state +let intentionalDisconnect = false; +let reconnectAttempts = 0; +const maxReconnectAttempts = 5; +let reconnectTimer = null; + +function getReconnectDelay() { + return Math.min(1000 * Math.pow(2, reconnectAttempts), 30000); +} + +// Tab operation locks - prevent concurrent attach/detach races +const tabLocks = new Map(); + +async function withTabLock(tabId, fn) { + // Wait for any existing lock on this tab + while (tabLocks.has(tabId)) { + await tabLocks.get(tabId); + } + let resolve; + const lockPromise = new Promise((r) => { + resolve = r; + }); + tabLocks.set(tabId, lockPromise); + try { + return await fn(); + } finally { + tabLocks.delete(tabId); + resolve(); + } +} + +// Multi-tab state: Map +const attachedTabs = new Map(); + +// Restore settings from chrome.storage on startup +chrome.storage.local.get(['serverUrl', 'fullVisionMode'], (result) => { + if (result.serverUrl) serverUrl = result.serverUrl; + if (result.fullVisionMode !== undefined) + fullVisionMode = result.fullVisionMode; +}); + +// Listen for settings changes from sidepanel +chrome.storage.onChanged.addListener((changes, area) => { + if (area !== 'local') return; + if (changes.serverUrl) serverUrl = changes.serverUrl.newValue; + if (changes.fullVisionMode) fullVisionMode = changes.fullVisionMode.newValue; +}); + +// Open side panel when action button is clicked +chrome.action.onClicked.addListener((tab) => { + chrome.sidePanel.open({ windowId: tab.windowId }); +}); + +// Enable side panel for all URLs +chrome.sidePanel + .setPanelBehavior({ openPanelOnActionClick: true }) + .catch((error) => console.error('Error setting panel behavior:', error)); + +// Connect to Python backend +function connect(url) { + if (url) serverUrl = url; + + return new Promise((resolve, reject) => { + try { + ws = new WebSocket(serverUrl); + + ws.onopen = () => { + console.log('Connected to backend server'); + isConnected = true; + reconnectAttempts = 0; + intentionalDisconnect = false; + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + broadcastToPopup({ type: 'CONNECTION_STATUS', connected: true }); + resolve({ success: true }); + }; + + ws.onclose = () => { + console.log('Disconnected from backend server'); + const wasConnected = isConnected; + isConnected = false; + ws = null; + broadcastToPopup({ type: 'CONNECTION_STATUS', connected: false }); + + // Auto-reconnect if not intentional + if ( + !intentionalDisconnect && + wasConnected && + reconnectAttempts < maxReconnectAttempts + ) { + const delay = getReconnectDelay(); + reconnectAttempts++; + console.log( + `Auto-reconnecting in ${delay}ms (attempt ${reconnectAttempts}/${maxReconnectAttempts})` + ); + broadcastToPopup({ + type: 'CONNECTION_STATUS', + connected: false, + reconnecting: true, + attempt: reconnectAttempts, + }); + reconnectTimer = setTimeout(() => { + connect().catch(() => { + console.log('Reconnect attempt failed'); + }); + }, delay); + } else if (reconnectAttempts >= maxReconnectAttempts) { + console.log('Max reconnect attempts reached'); + broadcastToPopup({ + type: 'CONNECTION_STATUS', + connected: false, + reconnecting: false, + failed: true, + }); + reconnectAttempts = 0; + } + }; + + ws.onerror = (error) => { + console.error('WebSocket error:', error); + isConnected = false; + reject({ + success: false, + error: + 'Cannot connect to server. Make sure the Python backend is running.', + }); + }; + + ws.onmessage = async (event) => { + try { + const message = JSON.parse(event.data); + await handleServerMessage(message); + } catch (e) { + console.error('Error parsing message:', e); + } + }; + + // Timeout for connection + setTimeout(() => { + if (!isConnected) { + ws?.close(); + reject({ success: false, error: 'Connection timeout' }); + } + }, 5000); + } catch (error) { + reject({ success: false, error: error.message }); + } + }); +} + +// Disconnect from backend +function disconnect() { + intentionalDisconnect = true; + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + reconnectAttempts = 0; + if (ws) { + ws.close(); + ws = null; + } + isConnected = false; + detachAllDebuggers(); +} + +// Handle messages from server +async function handleServerMessage(message) { + console.log('Received from server:', message); + + switch (message.type) { + case 'LOG': + broadcastToPopup({ + type: 'LOG', + level: message.level || 'info', + message: message.message, + }); + break; + + case 'ACTION': + broadcastToPopup({ + type: 'ACTION', + action: message.action, + detail: message.detail, + }); + break; + + case 'ACTION_COMPLETE': + broadcastToPopup({ + type: 'ACTION_COMPLETE', + success: message.success, + result: message.result, + }); + break; + + case 'CDP_COMMAND': { + // Execute CDP command via chrome.debugger, routed by tabId + const targetTabId = message.tabId || getDefaultTabId(); + try { + // Check if we should highlight before this action + if (message.highlight && message.highlight.selector) { + await highlightElement( + message.highlight.selector, + message.highlight.duration || 1500, + targetTabId + ); + // Small delay to let user see the highlight + await new Promise((resolve) => setTimeout(resolve, 300)); + } + + const result = await executeCdpCommand( + message.method, + message.params || {}, + targetTabId + ); + + // Send result back to server with tabId + sendToServer({ + type: 'CDP_RESULT', + id: message.id, + result: result, + tabId: targetTabId, + }); + } catch (error) { + sendToServer({ + type: 'CDP_ERROR', + id: message.id, + error: error.message, + tabId: targetTabId, + }); + // Only show errors to UI (skip routine CDP noise) + broadcastToPopup({ + type: 'LOG', + level: 'error', + message: `Failed: ${message.method} - ${error.message}`, + }); + } + break; + } + + case 'TAB_CREATE': { + // Create a new tab, attach debugger, and respond + try { + const url = message.url || 'about:blank'; + const newTab = await chrome.tabs.create({ url, active: false }); + await attachDebugger(newTab.id); + await enableCdpDomains(newTab.id); + sendToServer({ + type: 'TAB_CREATED', + id: message.id, + tabId: newTab.id, + url: newTab.url || url, + }); + broadcastToPopup({ + type: 'LOG', + level: 'success', + message: `Created tab ${newTab.id}: ${url}`, + }); + } catch (error) { + sendToServer({ + type: 'TAB_CREATE_ERROR', + id: message.id, + error: error.message, + }); + broadcastToPopup({ + type: 'LOG', + level: 'error', + message: `Failed to create tab: ${error.message}`, + }); + } + break; + } + + case 'TAB_CLOSE': { + // Close a specific tab + try { + const tabIdToClose = message.tabId; + await detachDebuggerFromTab(tabIdToClose); + await chrome.tabs.remove(tabIdToClose); + sendToServer({ + type: 'TAB_CLOSED', + id: message.id, + tabId: tabIdToClose, + }); + broadcastToPopup({ + type: 'LOG', + level: 'info', + message: `Closed tab ${tabIdToClose}`, + }); + } catch (error) { + sendToServer({ + type: 'TAB_CLOSE_ERROR', + id: message.id, + error: error.message, + }); + } + break; + } + + case 'TASK_COMPLETE': + broadcastToPopup({ + type: 'TASK_COMPLETE', + result: message.result, + }); + // Don't detach all tabs on task complete - let server manage tab lifecycle + break; + + case 'TASK_ERROR': + broadcastToPopup({ + type: 'TASK_ERROR', + error: message.error, + }); + break; + + case 'STREAM_TEXT': + // Forward streaming text to popup + broadcastToPopup({ + type: 'STREAM_TEXT', + text: message.text, + }); + break; + + case 'STREAM_START': + broadcastToPopup({ + type: 'STREAM_START', + }); + break; + + case 'STREAM_END': + broadcastToPopup({ + type: 'STREAM_END', + }); + break; + + case 'DETACH': + if (message.tabId) { + detachDebuggerFromTab(message.tabId); + } else { + detachAllDebuggers(); + } + break; + + case 'DEBUG_RESULT': + // Forward debug result to popup + broadcastToPopup({ + type: 'DEBUG_RESULT', + success: message.success, + result: message.result, + error: message.error, + }); + break; + + case 'REQUEST_ATTACH': + // Server is requesting debugger attachment to active tab + handleAttachRequest(); + break; + + case 'HIGHLIGHT': { + // Highlight an element on the page + console.log('Received HIGHLIGHT message:', message); + const hlTabId = message.tabId || getDefaultTabId(); + try { + const highlightResult = await highlightElement( + message.selector, + message.duration || 2000, + hlTabId + ); + console.log('Highlight completed:', highlightResult); + sendToServer({ + type: 'HIGHLIGHT_RESULT', + id: message.id, + success: true, + result: highlightResult, + tabId: hlTabId, + }); + } catch (error) { + console.error('Highlight failed:', error); + sendToServer({ + type: 'HIGHLIGHT_RESULT', + id: message.id, + success: false, + error: error.message, + tabId: hlTabId, + }); + } + break; + } + } +} + +// Get the first attached tab as default (backward compatibility) +function getDefaultTabId() { + if (attachedTabs.size > 0) { + return attachedTabs.keys().next().value; + } + return null; +} + +// Enable CDP domains for a specific tab +async function enableCdpDomains(tabId) { + const tabState = attachedTabs.get(tabId); + if (tabState && tabState.cdpEnabled) { + return; + } + await executeCdpCommand('Page.enable', {}, tabId); + await executeCdpCommand('DOM.enable', {}, tabId); + await executeCdpCommand('Runtime.enable', {}, tabId); + if (tabState) { + tabState.cdpEnabled = true; + } +} + +// Highlight element on page with animation +async function highlightElement(selector, duration = 600, tabId = null) { + const targetTabId = tabId || getDefaultTabId(); + if (!targetTabId) return null; + + console.log( + 'highlightElement called with selector:', + selector, + 'tab:', + targetTabId + ); + + const highlightScript = ` + (function() { + const sel = ${JSON.stringify(selector)}; + console.log('[Agent Highlight] Looking for element:', sel); + let element = null; + let foundBy = ''; + + // Method 1: Use ARIA snapshot's getElementByRef if available + if (typeof __ariaSnapshot !== 'undefined' && __ariaSnapshot.getElementByRef) { + console.log('[Agent Highlight] Trying __ariaSnapshot.getElementByRef'); + try { + element = __ariaSnapshot.getElementByRef(sel, document.body); + if (element) foundBy = 'ariaSnapshot.getElementByRef'; + } catch (e) { + console.log('[Agent Highlight] ariaSnapshot error:', e); + } + } + + // Method 2: Walk DOM looking for _ariaRef property + if (!element) { + console.log('[Agent Highlight] Walking DOM for _ariaRef'); + const walker = document.createTreeWalker( + document.body, + NodeFilter.SHOW_ELEMENT, + null, + false + ); + let node; + let count = 0; + while (node = walker.nextNode()) { + count++; + if (node._ariaRef && node._ariaRef.ref === sel) { + element = node; + foundBy = '_ariaRef property'; + break; + } + } + console.log('[Agent Highlight] Walked', count, 'nodes'); + } + + // Method 3: Try data attributes + if (!element) { + const refNum = sel.replace(/^e/, ''); + const selectors = [ + '[data-ref="' + sel + '"]', + '[data-ref="' + refNum + '"]', + '[ref="' + sel + '"]', + '[aria-ref="' + sel + '"]' + ]; + for (const s of selectors) { + try { + element = document.querySelector(s); + if (element) { + foundBy = 'data attribute: ' + s; + break; + } + } catch (e) {} + } + } + + // Method 4: Try CSS selector directly + if (!element && (sel.includes('[') || sel.includes('.') || sel.includes('#') || sel.includes(' '))) { + try { + element = document.querySelector(sel); + if (element) foundBy = 'CSS selector'; + } catch (e) {} + } + + if (!element) { + console.log('[Agent Highlight] Element NOT found for:', sel); + return { found: false, selector: sel, message: 'Element not found' }; + } + + console.log('[Agent Highlight] Element FOUND via:', foundBy, element); + + // Get element position + const rect = element.getBoundingClientRect(); + console.log('[Agent Highlight] Element rect:', rect); + + // Add animation keyframes if not exists + if (!document.getElementById('__agent_highlight_styles__')) { + const style = document.createElement('style'); + style.id = '__agent_highlight_styles__'; + style.textContent = \` + @keyframes __agent_pulse__ { + 0% { box-shadow: 0 0 0 4px rgba(255, 68, 68, 1), 0 0 15px rgba(255, 68, 68, 0.7); } + 50% { box-shadow: 0 0 0 6px rgba(255, 68, 68, 0.7), 0 0 25px rgba(255, 68, 68, 0.5); } + 100% { box-shadow: 0 0 0 4px rgba(255, 68, 68, 1), 0 0 15px rgba(255, 68, 68, 0.7); } + } + @keyframes __agent_ripple__ { + 0% { transform: scale(0.8); opacity: 1; } + 100% { transform: scale(2); opacity: 0; } + } + \`; + document.head.appendChild(style); + console.log('[Agent Highlight] Added styles'); + } + + // Remove any existing overlays + const existing = document.getElementById('__agent_highlight_overlay__'); + if (existing) existing.remove(); + const existingRipple = document.getElementById('__agent_ripple__'); + if (existingRipple) existingRipple.remove(); + + // Create highlight overlay + const overlay = document.createElement('div'); + overlay.id = '__agent_highlight_overlay__'; + overlay.style.cssText = \` + position: fixed; + top: \${rect.top - 8}px; + left: \${rect.left - 8}px; + width: \${rect.width + 16}px; + height: \${rect.height + 16}px; + border: 4px solid #ff4444; + border-radius: 8px; + background: rgba(255, 68, 68, 0.15); + pointer-events: none; + z-index: 2147483647; + animation: __agent_pulse__ 0.2s ease-in-out infinite; + \`; + document.body.appendChild(overlay); + console.log('[Agent Highlight] Added overlay'); + + // Add ripple effect + const ripple = document.createElement('div'); + ripple.id = '__agent_ripple__'; + ripple.style.cssText = \` + position: fixed; + top: \${rect.top + rect.height/2 - 25}px; + left: \${rect.left + rect.width/2 - 25}px; + width: 50px; + height: 50px; + border: 3px solid #ff4444; + border-radius: 50%; + pointer-events: none; + z-index: 2147483646; + animation: __agent_ripple__ 0.3s ease-out forwards; + \`; + document.body.appendChild(ripple); + + // Auto remove after duration (fast) + const dur = ${duration}; + setTimeout(() => { + const ol = document.getElementById('__agent_highlight_overlay__'); + if (ol) { + ol.style.transition = 'opacity 0.1s ease'; + ol.style.opacity = '0'; + setTimeout(() => ol.remove(), 100); + } + }, dur); + + setTimeout(() => { + const rp = document.getElementById('__agent_ripple__'); + if (rp) rp.remove(); + }, 300); + + return { + found: true, + selector: sel, + foundBy: foundBy, + rect: { top: rect.top, left: rect.left, width: rect.width, height: rect.height } + }; + })(); + `; + + try { + const result = await executeCdpCommand( + 'Runtime.evaluate', + { + expression: highlightScript, + returnByValue: true, + }, + targetTabId + ); + console.log('Highlight CDP result:', result); + return result; + } catch (error) { + console.error('Highlight CDP error:', error); + return null; + } +} + +// Handle attach request from server +async function handleAttachRequest() { + try { + // Get current active tab + const [tab] = await chrome.tabs.query({ + active: true, + currentWindow: true, + }); + if (!tab) { + sendToServer({ + type: 'ATTACH_RESULT', + success: false, + error: 'No active tab found', + }); + return; + } + + // If current tab is a restricted page, navigate to about:blank first + if ( + tab.url && + (tab.url.startsWith('chrome://') || + tab.url.startsWith('chrome-extension://') || + tab.url.startsWith('edge://') || + tab.url.startsWith('about:')) + ) { + await chrome.tabs.update(tab.id, { url: 'about:blank' }); + await new Promise((resolve) => { + const listener = (id, changeInfo) => { + if (id === tab.id && changeInfo.status === 'complete') { + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + } + }; + chrome.tabs.onUpdated.addListener(listener); + setTimeout(() => { + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + }, 3000); + }); + } + + // Attach debugger if not already attached to this tab + if (!attachedTabs.has(tab.id) || !attachedTabs.get(tab.id).attached) { + await attachDebugger(tab.id); + await enableCdpDomains(tab.id); + } + + sendToServer({ + type: 'ATTACH_RESULT', + success: true, + tabId: tab.id, + url: tab.url, + }); + + broadcastToPopup({ + type: 'LOG', + level: 'success', + message: 'Debugger attached to: ' + tab.url, + }); + } catch (error) { + sendToServer({ + type: 'ATTACH_RESULT', + success: false, + error: error.message, + }); + broadcastToPopup({ + type: 'LOG', + level: 'error', + message: 'Failed to attach debugger: ' + error.message, + }); + } +} + +// Execute CDP command routed to a specific tab +async function executeCdpCommand(method, params, tabId) { + if (!tabId || !attachedTabs.has(tabId) || !attachedTabs.get(tabId).attached) { + throw new Error(`Debugger not attached to tab ${tabId}`); + } + + return new Promise((resolve, reject) => { + chrome.debugger.sendCommand({ tabId: tabId }, method, params, (result) => { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)); + } else { + resolve(result); + } + }); + }); +} + +// Attach debugger to tab (supports multiple concurrent attachments) +async function attachDebugger(tabId) { + return withTabLock(tabId, async () => { + // Already attached to this tab + if (attachedTabs.has(tabId) && attachedTabs.get(tabId).attached) { + return true; + } + + // Do NOT detach other tabs - support concurrent attachments + return new Promise((resolve, reject) => { + chrome.debugger.attach({ tabId: tabId }, '1.3', () => { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)); + } else { + attachedTabs.set(tabId, { attached: true, cdpEnabled: false }); + console.log( + 'Debugger attached to tab:', + tabId, + 'Total attached:', + attachedTabs.size + ); + resolve(true); + } + }); + }); + }); +} + +// Detach debugger from a specific tab +async function detachDebuggerFromTab(tabId) { + return withTabLock(tabId, async () => { + if (!attachedTabs.has(tabId)) return; + + return new Promise((resolve) => { + chrome.debugger.detach({ tabId: tabId }, () => { + if (chrome.runtime.lastError) { + console.log( + 'Detach error (may already be detached):', + chrome.runtime.lastError.message + ); + } + attachedTabs.delete(tabId); + console.log( + 'Debugger detached from tab:', + tabId, + 'Remaining:', + attachedTabs.size + ); + resolve(); + }); + }); + }); +} + +// Detach debugger from all tabs +async function detachAllDebuggers() { + const tabIds = [...attachedTabs.keys()]; + for (const tabId of tabIds) { + await detachDebuggerFromTab(tabId); + } + console.log('All debuggers detached'); +} + +// Send message to server +function sendToServer(message) { + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify(message)); + } +} + +// Broadcast message to popup +function broadcastToPopup(message) { + chrome.runtime.sendMessage(message).catch(() => { + // Popup might be closed, ignore error + }); +} + +// Listen for debugger events - forward from ALL attached tabs +chrome.debugger.onEvent.addListener((source, method, params) => { + if (attachedTabs.has(source.tabId)) { + sendToServer({ + type: 'CDP_EVENT', + method: method, + params: params, + tabId: source.tabId, + }); + } +}); + +// Handle debugger detach - remove specific tab from map +chrome.debugger.onDetach.addListener((source, reason) => { + if (attachedTabs.has(source.tabId)) { + console.log('Debugger detached from tab:', source.tabId, 'reason:', reason); + attachedTabs.delete(source.tabId); + sendToServer({ + type: 'TAB_DETACHED', + tabId: source.tabId, + reason: reason, + }); + broadcastToPopup({ + type: 'LOG', + level: 'info', + message: `Debugger detached from tab ${source.tabId}: ${reason}`, + }); + } +}); + +// Message handler from popup +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + console.log('Message from popup:', message); + + switch (message.type) { + case 'GET_STATUS': + sendResponse({ + connected: isConnected, + attachedTabs: attachedTabs.size, + }); + break; + + case 'CONNECT': + connect(message.serverUrl) + .then((result) => sendResponse(result)) + .catch((error) => sendResponse(error)); + return true; // Keep channel open for async response + + case 'DISCONNECT': + disconnect(); + sendResponse({ success: true }); + break; + + case 'EXECUTE_TASK': + if (message.fullVisionMode !== undefined) { + fullVisionMode = message.fullVisionMode; + } + executeTask(message.task, message.tabId, message.url) + .then(() => sendResponse({ success: true })) + .catch((error) => + sendResponse({ success: false, error: error.message }) + ); + return true; + + case 'STOP_TASK': + sendToServer({ type: 'STOP_TASK' }); + // Only detach the specific tab if provided, otherwise detach all + if (message.tabId && attachedTabs.has(message.tabId)) { + detachDebuggerFromTab(message.tabId); + } + sendResponse({ success: true }); + break; + + case 'CLEAR_CONTEXT': + sendToServer({ type: 'CLEAR_CONTEXT' }); + sendResponse({ success: true }); + break; + + case 'SET_FULL_VISION': + fullVisionMode = message.enabled; + chrome.storage.local.set({ fullVisionMode }); + console.log('Full vision mode:', fullVisionMode); + sendResponse({ success: true }); + break; + + case 'DEBUG_COMMAND': + executeDebugCommand(message.command, message.tabId, message.url) + .then(() => sendResponse({ success: true })) + .catch((error) => + sendResponse({ success: false, error: error.message }) + ); + return true; + } +}); + +// Execute debug command +async function executeDebugCommand(command, tabId, url) { + try { + // Attach debugger if not already attached + if (!attachedTabs.has(tabId) || !attachedTabs.get(tabId).attached) { + await attachDebugger(tabId); + await enableCdpDomains(tabId); + } + + // Send debug command to server + sendToServer({ + type: 'DEBUG_COMMAND', + command: command, + url: url, + tabId: tabId, + }); + } catch (error) { + broadcastToPopup({ + type: 'DEBUG_RESULT', + success: false, + error: error.message, + }); + throw error; + } +} + +// Navigation re-attach: re-enable CDP after page navigation +chrome.webNavigation.onCompleted.addListener(async (details) => { + // Only main frame, only tabs we have debugger attached to + if (details.frameId !== 0) return; + const tabState = attachedTabs.get(details.tabId); + if (!tabState || !tabState.attached) return; + + // Skip restricted pages + if ( + details.url && + (details.url.startsWith('chrome://') || + details.url.startsWith('chrome-extension://') || + details.url.startsWith('edge://')) + ) { + return; + } + + console.log( + 'Navigation completed on attached tab:', + details.tabId, + details.url + ); + // Small delay to let the page settle + await new Promise((r) => setTimeout(r, 500)); + + // Re-enable CDP domains (debugger stays attached, but domains may reset) + try { + tabState.cdpEnabled = false; + await enableCdpDomains(details.tabId); + console.log( + 'CDP domains re-enabled after navigation on tab:', + details.tabId + ); + } catch (e) { + console.log('Failed to re-enable CDP after navigation:', e.message); + } +}); + +// Execute task +async function executeTask(task, tabId, url) { + try { + // If current tab is a restricted page, navigate to about:blank first + if ( + url && + (url.startsWith('chrome://') || + url.startsWith('chrome-extension://') || + url.startsWith('edge://') || + url.startsWith('about:')) + ) { + await chrome.tabs.update(tabId, { url: 'about:blank' }); + // Wait for navigation to complete + await new Promise((resolve) => { + const listener = (id, changeInfo) => { + if (id === tabId && changeInfo.status === 'complete') { + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + } + }; + chrome.tabs.onUpdated.addListener(listener); + setTimeout(() => { + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + }, 3000); + }); + url = 'about:blank'; + } + + // Attach debugger to the tab (only if not already attached) + if (!attachedTabs.has(tabId) || !attachedTabs.get(tabId).attached) { + await attachDebugger(tabId); + + // Enable necessary CDP domains + await enableCdpDomains(tabId); + } + + // Send task to server + broadcastToPopup({ + type: 'LOG', + level: 'info', + message: 'Sending task to AI...', + }); + + sendToServer({ + type: 'START_TASK', + task: task, + url: url, + tabId: tabId, + fullVisionMode: fullVisionMode, + }); + } catch (error) { + broadcastToPopup({ + type: 'TASK_ERROR', + error: error.message, + }); + throw error; + } +} diff --git a/extensions/chrome_extension/icon.png b/extensions/chrome_extension/icon.png new file mode 100644 index 000000000..0e63daa25 Binary files /dev/null and b/extensions/chrome_extension/icon.png differ diff --git a/extensions/chrome_extension/icon.svg b/extensions/chrome_extension/icon.svg new file mode 100644 index 000000000..a289dbde1 --- /dev/null +++ b/extensions/chrome_extension/icon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/extensions/chrome_extension/manifest.json b/extensions/chrome_extension/manifest.json new file mode 100644 index 000000000..f7bfb5df7 --- /dev/null +++ b/extensions/chrome_extension/manifest.json @@ -0,0 +1,30 @@ +{ + "manifest_version": 3, + "name": "CAMEL Browser Agent", + "version": "1.0", + "description": "AI-powered browser automation using CAMEL framework", + "permissions": [ + "activeTab", + "tabs", + "debugger", + "scripting", + "sidePanel", + "webNavigation", + "storage" + ], + "host_permissions": [""], + "side_panel": { + "default_path": "sidepanel.html" + }, + "action": { + "default_title": "Open CAMEL Browser Agent" + }, + "background": { + "service_worker": "background.js" + }, + "icons": { + "16": "icon.png", + "48": "icon.png", + "128": "icon.png" + } +} diff --git a/extensions/chrome_extension/popup.css b/extensions/chrome_extension/popup.css new file mode 100644 index 000000000..d84d01eda --- /dev/null +++ b/extensions/chrome_extension/popup.css @@ -0,0 +1,572 @@ +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +:root { + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #f1f3f5; + --text-primary: #212529; + --text-secondary: #6c757d; + --text-tertiary: #adb5bd; + --border-color: #e9ecef; + --accent: #228be6; + --accent-light: #e7f5ff; + --success: #40c057; + --error: #fa5252; + --warning: #fab005; +} + +body { + width: 380px; + height: 520px; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', + sans-serif; + font-size: 14px; + color: var(--text-primary); + background: var(--bg-primary); + overflow: hidden; +} + +.app { + display: flex; + flex-direction: column; + height: 100%; +} + +/* Header */ +.header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid var(--border-color); + background: var(--bg-primary); +} + +.header-left { + display: flex; + align-items: center; + gap: 10px; +} + +.logo { + color: var(--accent); +} + +.title { + font-weight: 600; + font-size: 15px; +} + +.icon-btn { + width: 32px; + height: 32px; + border: none; + background: transparent; + border-radius: 8px; + cursor: pointer; + color: var(--text-secondary); + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s; +} + +.icon-btn:hover { + background: var(--bg-secondary); + color: var(--text-primary); +} + +/* Pages */ +.page { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.page.hidden { + display: none; +} + +/* Settings Page */ +.settings-content { + flex: 1; + padding: 20px 16px; + overflow-y: auto; +} + +.settings-content h2 { + font-size: 18px; + font-weight: 600; + margin-bottom: 20px; + color: var(--text-primary); +} + +.setting-group { + margin-bottom: 20px; +} + +.setting-group label { + display: block; + font-size: 12px; + font-weight: 500; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 8px; +} + +.connection-status { + display: flex; + align-items: center; + gap: 8px; + font-weight: 500; +} + +.status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--error); +} + +.status-dot.connected { + background: var(--success); +} + +.input { + width: 100%; + padding: 10px 12px; + border: 1px solid var(--border-color); + border-radius: 8px; + font-size: 14px; + transition: border-color 0.2s; +} + +.input:focus { + outline: none; + border-color: var(--accent); +} + +.current-page { + font-size: 13px; + color: var(--text-secondary); + word-break: break-all; + padding: 10px 12px; + background: var(--bg-secondary); + border-radius: 8px; +} + +.setting-actions { + margin-bottom: 20px; +} + +.about-text { + font-size: 13px; + color: var(--text-secondary); + line-height: 1.5; +} + +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + padding: 10px 16px; + border: none; + border-radius: 8px; + font-size: 14px; + font-weight: 500; + cursor: pointer; + transition: all 0.2s; +} + +.btn-primary { + background: var(--accent); + color: white; +} + +.btn-primary:hover { + background: #1c7ed6; +} + +.btn-primary:disabled { + background: var(--text-tertiary); + cursor: not-allowed; +} + +.btn-secondary { + background: var(--bg-secondary); + color: var(--text-primary); +} + +.btn-secondary:hover { + background: var(--bg-tertiary); +} + +.btn-full { + width: calc(100% - 32px); + margin: 16px; +} + +.btn-loader { + width: 14px; + height: 14px; + border: 2px solid rgba(255, 255, 255, 0.3); + border-top-color: white; + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +.btn-loader.hidden { + display: none; +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} + +/* Chat Page */ +.messages-container { + flex: 1; + overflow-y: auto; + padding: 16px; +} + +.welcome-message { + text-align: center; + padding: 40px 20px; +} + +.welcome-icon { + color: var(--accent); + margin-bottom: 16px; +} + +.welcome-message h3 { + font-size: 16px; + font-weight: 600; + margin-bottom: 8px; +} + +.welcome-message p { + font-size: 13px; + color: var(--text-secondary); + margin-bottom: 20px; +} + +.suggestions { + display: flex; + flex-wrap: wrap; + gap: 8px; + justify-content: center; +} + +.suggestion-chip { + padding: 8px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 16px; + font-size: 12px; + color: var(--text-secondary); + cursor: pointer; + transition: all 0.2s; +} + +.suggestion-chip:hover { + background: var(--accent-light); + border-color: var(--accent); + color: var(--accent); +} + +/* Message Bubbles */ +.message { + margin-bottom: 16px; + animation: fadeInUp 0.3s ease; +} + +@keyframes fadeInUp { + from { + opacity: 0; + transform: translateY(10px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.message-user { + display: flex; + justify-content: flex-end; +} + +.message-user .message-content { + background: var(--accent); + color: white; + border-radius: 16px 16px 4px 16px; + padding: 10px 14px; + max-width: 85%; +} + +.message-agent { + display: flex; + flex-direction: column; + align-items: flex-start; +} + +.message-agent .message-content { + background: var(--bg-secondary); + border-radius: 16px 16px 16px 4px; + padding: 12px 14px; + max-width: 85%; +} + +.message-text { + font-size: 14px; + line-height: 1.5; +} + +/* Action Steps */ +.action-steps { + margin-top: 12px; +} + +.action-step { + display: flex; + align-items: flex-start; + gap: 10px; + padding: 8px 0; + border-top: 1px solid var(--border-color); + animation: fadeIn 0.3s ease; +} + +@keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +} + +.action-icon { + width: 20px; + height: 20px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + margin-top: 2px; +} + +.action-icon.pending { + background: var(--bg-tertiary); + color: var(--text-tertiary); +} + +.action-icon.running { + background: var(--accent-light); + color: var(--accent); + animation: pulse 1.5s ease infinite; +} + +@keyframes pulse { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0.5; + } +} + +.action-icon.success { + background: #d3f9d8; + color: var(--success); +} + +.action-icon.error { + background: #ffe3e3; + color: var(--error); +} + +.action-icon svg { + width: 12px; + height: 12px; +} + +.action-info { + flex: 1; + min-width: 0; +} + +.action-name { + font-size: 13px; + font-weight: 500; + color: var(--text-primary); +} + +.action-detail { + font-size: 12px; + color: var(--text-secondary); + margin-top: 2px; + word-break: break-word; +} + +/* Typing Indicator */ +.typing-indicator { + display: flex; + gap: 4px; + padding: 12px 14px; + background: var(--bg-secondary); + border-radius: 16px; + width: fit-content; +} + +.typing-dot { + width: 8px; + height: 8px; + background: var(--text-tertiary); + border-radius: 50%; + animation: typingBounce 1.4s ease infinite; +} + +.typing-dot:nth-child(2) { + animation-delay: 0.2s; +} +.typing-dot:nth-child(3) { + animation-delay: 0.4s; +} + +@keyframes typingBounce { + 0%, + 60%, + 100% { + transform: translateY(0); + } + 30% { + transform: translateY(-4px); + } +} + +/* Input Area */ +.input-area { + padding: 12px 16px 16px; + border-top: 1px solid var(--border-color); + background: var(--bg-primary); +} + +.input-wrapper { + display: flex; + align-items: flex-end; + gap: 8px; + background: var(--bg-secondary); + border-radius: 20px; + padding: 4px 4px 4px 16px; + border: 1px solid var(--border-color); + transition: border-color 0.2s; +} + +.input-wrapper:focus-within { + border-color: var(--accent); +} + +#messageInput { + flex: 1; + border: none; + background: transparent; + font-size: 14px; + line-height: 1.5; + resize: none; + max-height: 100px; + padding: 8px 0; +} + +#messageInput:focus { + outline: none; +} + +#messageInput::placeholder { + color: var(--text-tertiary); +} + +.send-btn { + width: 36px; + height: 36px; + border: none; + background: var(--accent); + border-radius: 50%; + color: white; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s; + flex-shrink: 0; +} + +.send-btn:hover:not(:disabled) { + background: #1c7ed6; + transform: scale(1.05); +} + +.send-btn:disabled { + background: var(--text-tertiary); + cursor: not-allowed; +} + +.input-hint { + margin-top: 8px; + text-align: center; +} + +.status-hint { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 12px; + color: var(--text-tertiary); +} + +.hint-dot { + width: 6px; + height: 6px; + border-radius: 50%; + background: var(--error); +} + +.status-hint.connected .hint-dot { + background: var(--success); +} + +.status-hint.connected { + color: var(--success); +} + +/* Scrollbar */ +::-webkit-scrollbar { + width: 6px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: var(--border-color); + border-radius: 3px; +} + +::-webkit-scrollbar-thumb:hover { + background: var(--text-tertiary); +} diff --git a/extensions/chrome_extension/popup.html b/extensions/chrome_extension/popup.html new file mode 100644 index 000000000..2e4eeec6b --- /dev/null +++ b/extensions/chrome_extension/popup.html @@ -0,0 +1,128 @@ + + + + + + CAMEL Browser Agent + + + +
+ +
+
+ + CAMEL Browser Agent +
+
+ +
+
+ + + + + +
+ +
+
+
+ + + + + + +
+

Welcome to CAMEL Browser Agent

+

Describe what you want to do on this page, and I'll help automate it.

+
+ + + +
+
+
+ + +
+
+ + +
+
+ + + Not connected + +
+
+
+
+ + + + diff --git a/extensions/chrome_extension/popup.js b/extensions/chrome_extension/popup.js new file mode 100644 index 000000000..7b24b5d07 --- /dev/null +++ b/extensions/chrome_extension/popup.js @@ -0,0 +1,386 @@ +// State +let isConnected = false; +let currentTabId = null; +let currentTabUrl = ''; +let _conversationHistory = []; + +// DOM Elements +const settingsPage = document.getElementById('settingsPage'); +const chatPage = document.getElementById('chatPage'); +const settingsBtn = document.getElementById('settingsBtn'); +const backToChat = document.getElementById('backToChat'); +const connectBtn = document.getElementById('connectBtn'); +const connectionDot = document.getElementById('connectionDot'); +const connectionText = document.getElementById('connectionText'); +const serverUrlInput = document.getElementById('serverUrl'); +const currentPageUrl = document.getElementById('currentPageUrl'); +const messagesContainer = document.getElementById('messagesContainer'); +const messageInput = document.getElementById('messageInput'); +const sendBtn = document.getElementById('sendBtn'); +const statusHint = document.getElementById('statusHint'); + +// Initialize +document.addEventListener('DOMContentLoaded', async () => { + // Get current tab info + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tab) { + currentTabId = tab.id; + currentTabUrl = tab.url; + currentPageUrl.textContent = truncateUrl(tab.url, 50); + currentPageUrl.title = tab.url; + } + + // Check connection status + chrome.runtime.sendMessage({ type: 'GET_STATUS' }, (response) => { + if (response && response.connected) { + updateConnectionStatus(true); + } + }); + + // Listen for messages from background + chrome.runtime.onMessage.addListener(handleBackgroundMessage); + + // Setup event listeners + setupEventListeners(); +}); + +function setupEventListeners() { + // Settings toggle + settingsBtn.addEventListener('click', () => { + settingsPage.classList.remove('hidden'); + chatPage.classList.add('hidden'); + }); + + backToChat.addEventListener('click', () => { + chatPage.classList.remove('hidden'); + settingsPage.classList.add('hidden'); + }); + + // Connect button + connectBtn.addEventListener('click', handleConnect); + + // Send message + sendBtn.addEventListener('click', sendMessage); + messageInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + sendMessage(); + } + }); + + // Auto-resize textarea + messageInput.addEventListener('input', () => { + messageInput.style.height = 'auto'; + messageInput.style.height = Math.min(messageInput.scrollHeight, 100) + 'px'; + updateSendButton(); + }); + + // Suggestion chips + document.querySelectorAll('.suggestion-chip').forEach((chip) => { + chip.addEventListener('click', () => { + messageInput.value = chip.dataset.text; + updateSendButton(); + messageInput.focus(); + }); + }); +} + +async function handleConnect() { + const btnText = connectBtn.querySelector('.btn-text'); + const btnLoader = connectBtn.querySelector('.btn-loader'); + + if (isConnected) { + chrome.runtime.sendMessage({ type: 'DISCONNECT' }); + updateConnectionStatus(false); + } else { + btnText.textContent = 'Connecting...'; + btnLoader.classList.remove('hidden'); + connectBtn.disabled = true; + + chrome.runtime.sendMessage( + { + type: 'CONNECT', + serverUrl: serverUrlInput.value, + }, + (response) => { + btnLoader.classList.add('hidden'); + connectBtn.disabled = false; + + if (response && response.success) { + updateConnectionStatus(true); + } else { + updateConnectionStatus(false); + showError(response?.error || 'Connection failed'); + } + } + ); + } +} + +function updateConnectionStatus(connected) { + isConnected = connected; + + // Update settings page + connectionDot.classList.toggle('connected', connected); + connectionText.textContent = connected ? 'Connected' : 'Disconnected'; + + const btnText = connectBtn.querySelector('.btn-text'); + btnText.textContent = connected ? 'Disconnect' : 'Connect'; + + // Update chat page + statusHint.classList.toggle('connected', connected); + statusHint.innerHTML = connected + ? 'Connected' + : 'Not connected'; + + updateSendButton(); +} + +function updateSendButton() { + const hasText = messageInput.value.trim().length > 0; + sendBtn.disabled = !hasText || !isConnected; +} + +async function sendMessage() { + const text = messageInput.value.trim(); + if (!text || !isConnected) return; + + // Clear input + messageInput.value = ''; + messageInput.style.height = 'auto'; + updateSendButton(); + + // Hide welcome message + const welcome = messagesContainer.querySelector('.welcome-message'); + if (welcome) welcome.remove(); + + // Add user message + addMessage('user', text); + + // Add agent response placeholder + const agentMsgId = addAgentMessage(); + + // Send to background + chrome.runtime.sendMessage({ + type: 'EXECUTE_TASK', + task: text, + tabId: currentTabId, + url: currentTabUrl, + }); +} + +function addMessage(type, text) { + const msgDiv = document.createElement('div'); + msgDiv.className = `message message-${type}`; + msgDiv.innerHTML = ` +
+
${escapeHtml(text)}
+
+ `; + messagesContainer.appendChild(msgDiv); + scrollToBottom(); + return msgDiv; +} + +function addAgentMessage() { + const msgId = 'agent-msg-' + Date.now(); + const msgDiv = document.createElement('div'); + msgDiv.className = 'message message-agent'; + msgDiv.id = msgId; + msgDiv.innerHTML = ` +
+
+
+
+
+
+ +
+ `; + messagesContainer.appendChild(msgDiv); + scrollToBottom(); + return msgId; +} + +function addActionStep(msgId, action, status = 'running') { + const msgDiv = + document.getElementById(msgId) || + document.querySelector('.message-agent:last-child'); + if (!msgDiv) return; + + const stepsContainer = msgDiv.querySelector('.action-steps'); + const typingIndicator = msgDiv.querySelector('.typing-indicator'); + + // Hide typing indicator, show steps + if (typingIndicator) typingIndicator.style.display = 'none'; + stepsContainer.style.display = 'block'; + + const stepId = 'step-' + Date.now(); + const stepDiv = document.createElement('div'); + stepDiv.className = 'action-step'; + stepDiv.id = stepId; + stepDiv.innerHTML = ` +
+ ${getStatusIcon(status)} +
+
+
${escapeHtml(action.name || action)}
+ ${action.detail ? `
${escapeHtml(action.detail)}
` : ''} +
+ `; + stepsContainer.appendChild(stepDiv); + scrollToBottom(); + return stepId; +} + +function updateActionStep(stepId, status) { + const stepDiv = document.getElementById(stepId); + if (!stepDiv) return; + + const iconDiv = stepDiv.querySelector('.action-icon'); + iconDiv.className = `action-icon ${status}`; + iconDiv.innerHTML = getStatusIcon(status); +} + +function completeAgentMessage(msgId, text) { + const msgDiv = + document.getElementById(msgId) || + document.querySelector('.message-agent:last-child'); + if (!msgDiv) return; + + const typingIndicator = msgDiv.querySelector('.typing-indicator'); + if (typingIndicator) typingIndicator.remove(); + + if (text) { + const content = msgDiv.querySelector('.message-content'); + const textDiv = document.createElement('div'); + textDiv.className = 'message-text'; + textDiv.style.marginTop = '12px'; + textDiv.textContent = text; + content.appendChild(textDiv); + } + + scrollToBottom(); +} + +function getStatusIcon(status) { + switch (status) { + case 'running': + return ''; + case 'success': + return ''; + case 'error': + return ''; + default: + return ''; + } +} + +function handleBackgroundMessage(message) { + console.log('Message from background:', message); + + switch (message.type) { + case 'CONNECTION_STATUS': + updateConnectionStatus(message.connected); + break; + + case 'LOG': + handleLogMessage(message); + break; + + case 'ACTION': + const stepId = addActionStep( + null, + { + name: message.action, + detail: message.detail, + }, + 'running' + ); + // Store step ID for later update + window.currentStepId = stepId; + break; + + case 'ACTION_COMPLETE': + if (window.currentStepId) { + updateActionStep( + window.currentStepId, + message.success ? 'success' : 'error' + ); + } + break; + + case 'TASK_COMPLETE': + completeAgentMessage(null, message.result); + break; + + case 'TASK_ERROR': + if (window.currentStepId) { + updateActionStep(window.currentStepId, 'error'); + } + completeAgentMessage(null, 'Error: ' + message.error); + break; + } +} + +function handleLogMessage(message) { + const level = message.level || 'info'; + const text = message.message; + + // Parse action from log + if (text.includes('CDP command:') || text.includes('Executing:')) { + const actionName = text.split(':').pop().trim(); + const stepId = addActionStep(null, { name: actionName }, 'running'); + window.currentStepId = stepId; + } else if ( + text.includes('success') || + text.includes('Success') || + text.includes('Clicked') || + text.includes('Typed') + ) { + if (window.currentStepId) { + updateActionStep(window.currentStepId, 'success'); + } + // Add new step for the action + addActionStep(null, { name: text }, 'success'); + } else if (level === 'error') { + if (window.currentStepId) { + updateActionStep(window.currentStepId, 'error'); + } + addActionStep(null, { name: text }, 'error'); + } else if ( + text.includes('AI') || + text.includes('Analyzing') || + text.includes('Processing') + ) { + addActionStep(null, { name: text }, 'running'); + } +} + +function showError(message) { + // Add error message to chat + const errorDiv = document.createElement('div'); + errorDiv.className = 'message message-agent'; + errorDiv.innerHTML = ` +
+
${escapeHtml(message)}
+
+ `; + messagesContainer.appendChild(errorDiv); + scrollToBottom(); +} + +function scrollToBottom() { + messagesContainer.scrollTop = messagesContainer.scrollHeight; +} + +function truncateUrl(url, maxLen) { + if (url.length <= maxLen) return url; + return url.substring(0, maxLen - 3) + '...'; +} + +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} diff --git a/extensions/chrome_extension/sidepanel.css b/extensions/chrome_extension/sidepanel.css new file mode 100644 index 000000000..3899129bd --- /dev/null +++ b/extensions/chrome_extension/sidepanel.css @@ -0,0 +1,1067 @@ +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +:root { + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #f1f3f5; + --text-primary: #212529; + --text-secondary: #6c757d; + --text-tertiary: #adb5bd; + --border-color: #e9ecef; + --accent: #228be6; + --accent-light: #e7f5ff; + --success: #40c057; + --error: #fa5252; + --warning: #fab005; +} + +html, +body { + height: 100%; + font-family: + -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', + sans-serif; + font-size: 14px; + color: var(--text-primary); + background: var(--bg-primary); +} + +.app { + display: flex; + flex-direction: column; + height: 100%; + position: relative; +} + +/* Header */ +.header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid var(--border-color); + background: var(--bg-primary); + flex-shrink: 0; +} + +.header-left { + display: flex; + align-items: center; + gap: 10px; +} + +.header-right { + display: flex; + align-items: center; + gap: 4px; +} + +.logo { + color: var(--accent); +} + +.title { + font-weight: 600; + font-size: 15px; +} + +.icon-btn { + width: 32px; + height: 32px; + border: none; + background: transparent; + border-radius: 8px; + cursor: pointer; + color: var(--text-secondary); + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s; +} + +.icon-btn:hover { + background: var(--bg-secondary); + color: var(--text-primary); +} + +/* Settings Panel */ +.settings-panel { + position: absolute; + top: 0; + right: 0; + width: 100%; + max-width: 320px; + height: 100%; + background: var(--bg-primary); + box-shadow: -4px 0 20px rgba(0, 0, 0, 0.1); + z-index: 100; + display: flex; + flex-direction: column; + transform: translateX(0); + transition: transform 0.3s ease; +} + +.settings-panel.hidden { + transform: translateX(100%); +} + +.settings-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid var(--border-color); +} + +.settings-header h2 { + font-size: 16px; + font-weight: 600; +} + +.settings-content { + flex: 1; + padding: 16px; + overflow-y: auto; +} + +.setting-group { + margin-bottom: 20px; +} + +.setting-group label { + display: block; + font-size: 11px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 8px; +} + +.connection-row { + display: flex; + align-items: center; + gap: 8px; +} + +.status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--error); + flex-shrink: 0; +} + +.status-dot.connected { + background: var(--success); +} + +.connection-row span:not(.status-dot) { + flex: 1; + font-weight: 500; +} + +.input { + width: 100%; + padding: 10px 12px; + border: 1px solid var(--border-color); + border-radius: 8px; + font-size: 13px; + transition: border-color 0.2s; +} + +.input:focus { + outline: none; + border-color: var(--accent); +} + +.current-page { + font-size: 12px; + color: var(--text-secondary); + word-break: break-all; + padding: 10px 12px; + background: var(--bg-secondary); + border-radius: 8px; + line-height: 1.4; +} + +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + padding: 8px 14px; + border: none; + border-radius: 6px; + font-size: 13px; + font-weight: 500; + cursor: pointer; + transition: all 0.2s; +} + +.btn-sm { + padding: 6px 12px; + font-size: 12px; +} + +.btn-primary { + background: var(--accent); + color: white; +} + +.btn-primary:hover { + background: #1c7ed6; +} + +.btn-primary.connected { + background: var(--text-tertiary); +} + +/* Chat Area */ +.chat-area { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.messages-container { + flex: 1; + overflow-y: auto; + padding: 16px; +} + +.welcome-message { + text-align: center; + padding: 30px 16px; +} + +.welcome-icon { + color: var(--accent); + margin-bottom: 12px; +} + +.welcome-message h3 { + font-size: 15px; + font-weight: 600; + margin-bottom: 6px; +} + +.welcome-message p { + font-size: 13px; + color: var(--text-secondary); + margin-bottom: 16px; +} + +.suggestions { + display: flex; + flex-wrap: wrap; + gap: 6px; + justify-content: center; +} + +.suggestion-chip { + padding: 6px 10px; + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 14px; + font-size: 11px; + color: var(--text-secondary); + cursor: pointer; + transition: all 0.2s; +} + +.suggestion-chip:hover { + background: var(--accent-light); + border-color: var(--accent); + color: var(--accent); +} + +/* Message Bubbles */ +.message { + margin-bottom: 12px; + animation: fadeInUp 0.3s ease; +} + +@keyframes fadeInUp { + from { + opacity: 0; + transform: translateY(8px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.message-user { + display: flex; + justify-content: flex-end; +} + +.message-user .message-content { + background: var(--accent); + color: white; + border-radius: 14px 14px 4px 14px; + padding: 10px 12px; + max-width: 85%; +} + +.message-agent { + display: flex; + flex-direction: column; + align-items: flex-start; +} + +.message-agent .message-content { + background: var(--bg-secondary); + border-radius: 14px 14px 14px 4px; + padding: 12px; + max-width: 95%; + width: 100%; +} + +.message-text { + font-size: 13px; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-word; +} + +/* Streaming text */ +.streaming-text { + font-size: 13px; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-word; + display: none; + margin-top: 8px; + padding-top: 8px; + border-top: 1px solid var(--border-color); +} + +.streaming-text:not(:empty) { + display: block; +} + +/* Typing cursor animation for streaming */ +.streaming-text:not(.message-text)::after { + content: '▋'; + animation: blink 1s step-end infinite; + color: var(--accent); +} + +@keyframes blink { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0; + } +} + +/* Actions Container - Collapsible */ +.actions-container { + margin-top: 8px; + border: 1px solid var(--border-color); + border-radius: 8px; + background: var(--bg-primary); + display: none; /* Hidden by default, shown when first action is added */ +} + +.actions-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 10px; + background: var(--bg-tertiary); + cursor: pointer; + user-select: none; + transition: background 0.2s; +} + +.actions-header:hover { + background: var(--border-color); +} + +.actions-header-left { + display: flex; + align-items: center; + gap: 8px; +} + +.actions-status-icon { + width: 16px; + height: 16px; + display: flex; + align-items: center; + justify-content: center; +} + +.actions-status-icon.running { + color: var(--accent); + animation: spin 1s linear infinite; +} + +.actions-status-icon.success { + color: var(--success); +} + +.actions-status-icon.error { + color: var(--error); +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +.actions-title { + font-size: 12px; + font-weight: 500; + color: var(--text-primary); +} + +.actions-count { + font-size: 11px; + color: var(--text-tertiary); + margin-left: 4px; +} + +.actions-toggle { + width: 20px; + height: 20px; + display: flex; + align-items: center; + justify-content: center; + color: var(--text-tertiary); + transition: transform 0.2s; +} + +.actions-container.expanded .actions-toggle { + transform: rotate(180deg); +} + +/* Actions Body */ +.actions-body { + max-height: 0; + overflow: hidden; + transition: max-height 0.3s ease-out; +} + +.actions-container.expanded .actions-body { + max-height: 200px; + overflow-y: auto !important; + overflow-x: hidden; +} + +/* Scrollbar for actions list */ +.actions-body::-webkit-scrollbar { + width: 6px; +} + +.actions-body::-webkit-scrollbar-track { + background: var(--bg-tertiary); + border-radius: 3px; +} + +.actions-body::-webkit-scrollbar-thumb { + background: var(--text-tertiary); + border-radius: 3px; + min-height: 30px; +} + +.actions-body::-webkit-scrollbar-thumb:hover { + background: var(--text-secondary); +} + +.actions-list { + padding: 8px; + min-height: fit-content; +} + +/* Current Action Display (when collapsed) */ +.current-action-display { + padding: 8px 10px; + border-top: 1px solid var(--border-color); + position: relative; + overflow: hidden; + min-height: 32px; +} + +.current-action-display:empty { + display: none; +} + +.actions-container.expanded .current-action-display { + display: none; +} + +.current-action { + display: flex; + align-items: center; + gap: 8px; + animation: slideIn 0.3s ease; +} + +@keyframes slideIn { + from { + opacity: 0; + transform: translateY(-10px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.current-action-icon { + width: 16px; + height: 16px; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; +} + +.current-action-icon.running { + color: var(--accent); + animation: pulse 1.5s ease infinite; +} + +@keyframes pulse { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0.4; + } +} + +.current-action-icon.success { + color: var(--success); +} + +.current-action-icon.error { + color: var(--error); +} + +.current-action-icon svg { + width: 12px; + height: 12px; +} + +.current-action-text { + font-size: 11px; + color: var(--text-secondary); + flex: 1; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +/* Previous action fade effect */ +.previous-action { + position: absolute; + top: 0; + left: 0; + right: 0; + padding: 8px 10px; + display: flex; + align-items: center; + gap: 8px; + animation: fadeOut 0.3s ease forwards; + pointer-events: none; +} + +.previous-action .current-action-icon { + width: 16px; + height: 16px; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; +} + +.previous-action .current-action-text { + font-size: 11px; + color: var(--text-secondary); +} + +@keyframes fadeOut { + from { + opacity: 0.6; + transform: translateY(0); + } + to { + opacity: 0; + transform: translateY(10px); + } +} + +/* Action Step in expanded list */ +.action-step { + display: flex; + align-items: flex-start; + gap: 8px; + padding: 6px 0; + border-bottom: 1px solid var(--border-color); +} + +.action-step:last-child { + border-bottom: none; +} + +.action-icon { + width: 16px; + height: 16px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + margin-top: 1px; +} + +.action-icon.pending { + background: var(--bg-tertiary); + color: var(--text-tertiary); +} + +.action-icon.running { + background: var(--accent-light); + color: var(--accent); + animation: pulse 1.5s ease infinite; +} + +.action-icon.success { + background: #d3f9d8; + color: var(--success); +} + +.action-icon.error { + background: #ffe3e3; + color: var(--error); +} + +.action-icon svg { + width: 10px; + height: 10px; +} + +.action-info { + flex: 1; + min-width: 0; +} + +.action-name { + font-size: 11px; + font-weight: 500; + color: var(--text-primary); +} + +.action-time { + font-size: 10px; + color: var(--text-tertiary); + margin-left: 6px; +} + +/* Typing Indicator */ +.typing-indicator { + display: flex; + gap: 4px; + padding: 8px 0; +} + +.typing-dot { + width: 6px; + height: 6px; + background: var(--text-tertiary); + border-radius: 50%; + animation: typingBounce 1.4s ease infinite; +} + +.typing-dot:nth-child(2) { + animation-delay: 0.2s; +} +.typing-dot:nth-child(3) { + animation-delay: 0.4s; +} + +@keyframes typingBounce { + 0%, + 60%, + 100% { + transform: translateY(0); + } + 30% { + transform: translateY(-3px); + } +} + +/* Input Area */ +.input-area { + padding: 12px 16px 16px; + border-top: 1px solid var(--border-color); + background: var(--bg-primary); + flex-shrink: 0; +} + +.input-wrapper { + display: flex; + align-items: flex-end; + gap: 8px; + background: var(--bg-secondary); + border-radius: 18px; + padding: 4px 4px 4px 14px; + border: 1px solid var(--border-color); + transition: border-color 0.2s; +} + +.input-wrapper:focus-within { + border-color: var(--accent); +} + +#messageInput { + flex: 1; + border: none; + background: transparent; + font-size: 13px; + line-height: 1.4; + resize: none; + max-height: 80px; + padding: 6px 0; + font-family: inherit; +} + +#messageInput:focus { + outline: none; +} + +#messageInput::placeholder { + color: var(--text-tertiary); +} + +.send-btn { + width: 32px; + height: 32px; + border: none; + background: var(--accent); + border-radius: 50%; + color: white; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s; + flex-shrink: 0; +} + +.send-btn:hover:not(:disabled) { + background: #1c7ed6; + transform: scale(1.05); +} + +.send-btn:disabled { + background: var(--text-tertiary); + cursor: not-allowed; +} + +.send-btn svg { + width: 16px; + height: 16px; +} + +.send-btn.stop-mode { + background: var(--error); +} + +.send-btn.stop-mode:hover:not(:disabled) { + background: #e03131; +} + +.input-hint { + margin-top: 6px; + text-align: center; +} + +.status-hint { + display: inline-flex; + align-items: center; + gap: 5px; + font-size: 11px; + color: var(--text-tertiary); +} + +.hint-dot { + width: 5px; + height: 5px; + border-radius: 50%; + background: var(--error); +} + +.status-hint.connected .hint-dot { + background: var(--success); +} + +.status-hint.connected { + color: var(--success); +} + +/* Scrollbar */ +::-webkit-scrollbar { + width: 5px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: var(--border-color); + border-radius: 3px; +} + +::-webkit-scrollbar-thumb:hover { + background: var(--text-tertiary); +} + +/* Debug Mode Toggle */ +.debug-toggle-row { + display: flex; + align-items: center; + justify-content: space-between; + font-size: 13px; + color: var(--text-primary); +} + +.toggle-switch { + position: relative; + display: inline-block; + width: 44px; + height: 24px; +} + +.toggle-switch input { + opacity: 0; + width: 0; + height: 0; +} + +.toggle-slider { + position: absolute; + cursor: pointer; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: var(--border-color); + transition: 0.3s; + border-radius: 24px; +} + +.toggle-slider:before { + position: absolute; + content: ''; + height: 18px; + width: 18px; + left: 3px; + bottom: 3px; + background-color: white; + transition: 0.3s; + border-radius: 50%; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2); +} + +.toggle-switch input:checked + .toggle-slider { + background-color: var(--accent); +} + +.toggle-switch input:checked + .toggle-slider:before { + transform: translateX(20px); +} + +/* Debug Panel */ +.debug-panel { + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 50%; + background: var(--bg-primary); + border-top: 1px solid var(--border-color); + box-shadow: 0 -4px 20px rgba(0, 0, 0, 0.1); + z-index: 99; + display: flex; + flex-direction: column; + transform: translateY(0); + transition: transform 0.3s ease; +} + +.debug-panel.hidden { + transform: translateY(100%); +} + +.debug-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 12px; + background: var(--bg-tertiary); + border-bottom: 1px solid var(--border-color); + flex-shrink: 0; +} + +.debug-title { + font-size: 12px; + font-weight: 600; + color: var(--text-primary); + display: flex; + align-items: center; + gap: 6px; +} + +.debug-title::before { + content: ''; + display: inline-block; + width: 8px; + height: 8px; + background: var(--warning); + border-radius: 50%; +} + +.debug-content { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.debug-output { + flex: 1; + overflow-y: auto; + padding: 12px; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 11px; + line-height: 1.5; + background: #1e1e1e; + color: #d4d4d4; +} + +.debug-welcome { + color: var(--text-tertiary); + margin-bottom: 12px; +} + +.debug-welcome p { + margin-bottom: 8px; + color: #569cd6; +} + +.debug-welcome ul { + list-style: none; + padding-left: 0; +} + +.debug-welcome li { + margin-bottom: 4px; + color: #9cdcfe; +} + +.debug-welcome code { + background: #333; + padding: 1px 4px; + border-radius: 3px; + color: #ce9178; +} + +.debug-line { + margin-bottom: 4px; + white-space: pre-wrap; + word-break: break-all; +} + +.debug-line.command { + color: #4ec9b0; +} + +.debug-line.command::before { + content: '> '; + color: #6a9955; +} + +.debug-line.result { + color: #d4d4d4; + padding-left: 12px; +} + +.debug-line.error { + color: #f14c4c; +} + +.debug-line.success { + color: #4ec9b0; +} + +.debug-line.info { + color: #569cd6; +} + +.debug-input-area { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + background: var(--bg-secondary); + border-top: 1px solid var(--border-color); +} + +.debug-input { + flex: 1; + padding: 8px 12px; + border: 1px solid var(--border-color); + border-radius: 6px; + font-size: 12px; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + background: var(--bg-primary); +} + +.debug-input:focus { + outline: none; + border-color: var(--accent); +} + +.debug-send-btn { + width: 32px; + height: 32px; + border: none; + background: var(--warning); + border-radius: 6px; + color: white; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s; + flex-shrink: 0; +} + +.debug-send-btn:hover { + background: #e09f00; +} + +.debug-send-btn:disabled { + background: var(--text-tertiary); + cursor: not-allowed; +} diff --git a/extensions/chrome_extension/sidepanel.html b/extensions/chrome_extension/sidepanel.html new file mode 100644 index 000000000..19087e2b5 --- /dev/null +++ b/extensions/chrome_extension/sidepanel.html @@ -0,0 +1,186 @@ + + + + + + CAMEL Browser Agent + + + +
+ +
+
+ + CAMEL Browser Agent +
+
+ + +
+
+ + + + + + + + +
+ +
+
+
+ + + + + + +
+

Welcome to CAMEL Browser Agent

+

Describe what you want to do on this page.

+
+ + + + +
+
+
+ + +
+
+ + +
+
+ + + Not connected + +
+
+
+
+ + + + diff --git a/extensions/chrome_extension/sidepanel.js b/extensions/chrome_extension/sidepanel.js new file mode 100644 index 000000000..efffc5d55 --- /dev/null +++ b/extensions/chrome_extension/sidepanel.js @@ -0,0 +1,914 @@ +// State +let isConnected = false; +let currentTabId = null; +let currentTabUrl = ''; +let isDebugMode = false; +let fullVisionMode = false; +let isTaskRunning = false; + +// Message queue - queue messages while task is running +let messageQueue = []; + +// Output truncation +const MAX_OUTPUT_LENGTH = 120000; +function truncateOutput(text, maxLen = MAX_OUTPUT_LENGTH) { + if (typeof text !== 'string') return text; + if (text.length <= maxLen) return text; + return text.substring(0, maxLen) + '...[truncated]'; +} + +// DOM Elements +const settingsPanel = document.getElementById('settingsPanel'); +const settingsBtn = document.getElementById('settingsBtn'); +const closeSettings = document.getElementById('closeSettings'); +const clearBtn = document.getElementById('clearBtn'); +const connectBtn = document.getElementById('connectBtn'); +const connectionDot = document.getElementById('connectionDot'); +const connectionText = document.getElementById('connectionText'); +const serverUrlInput = document.getElementById('serverUrl'); +const currentPageUrl = document.getElementById('currentPageUrl'); +const messagesContainer = document.getElementById('messagesContainer'); +const messageInput = document.getElementById('messageInput'); +const sendBtn = document.getElementById('sendBtn'); +const statusHint = document.getElementById('statusHint'); +const debugModeToggle = document.getElementById('debugModeToggle'); +const debugPanel = document.getElementById('debugPanel'); +const closeDebug = document.getElementById('closeDebug'); +const debugInput = document.getElementById('debugInput'); +const debugSendBtn = document.getElementById('debugSendBtn'); +const debugOutput = document.getElementById('debugOutput'); +const fullVisionToggle = document.getElementById('fullVisionToggle'); + +// Initialize +document.addEventListener('DOMContentLoaded', async () => { + await updateCurrentTab(); + + // Restore settings from chrome.storage + chrome.storage.local.get( + ['serverUrl', 'fullVisionMode', 'debugMode'], + (result) => { + if (result.serverUrl && serverUrlInput) { + serverUrlInput.value = result.serverUrl; + } + if (result.fullVisionMode !== undefined && fullVisionToggle) { + fullVisionMode = result.fullVisionMode; + fullVisionToggle.checked = fullVisionMode; + } + if (result.debugMode !== undefined && debugModeToggle) { + isDebugMode = result.debugMode; + debugModeToggle.checked = isDebugMode; + if (isDebugMode) debugPanel.classList.remove('hidden'); + } + } + ); + + // Listen for tab changes + chrome.tabs.onActivated.addListener(updateCurrentTab); + chrome.tabs.onUpdated.addListener((tabId, changeInfo) => { + if (changeInfo.url) updateCurrentTab(); + }); + + // Check connection status + chrome.runtime.sendMessage({ type: 'GET_STATUS' }, (response) => { + if (response && response.connected) { + updateConnectionStatus(true); + } + }); + + // Listen for messages from background + chrome.runtime.onMessage.addListener(handleBackgroundMessage); + + // Setup event listeners + setupEventListeners(); +}); + +async function updateCurrentTab() { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tab) { + currentTabId = tab.id; + currentTabUrl = tab.url; + currentPageUrl.textContent = tab.url; + currentPageUrl.title = tab.url; + } +} + +function setupEventListeners() { + // Settings toggle + settingsBtn.addEventListener('click', () => { + settingsPanel.classList.remove('hidden'); + }); + + closeSettings.addEventListener('click', () => { + settingsPanel.classList.add('hidden'); + }); + + // Clear chat + clearBtn.addEventListener('click', clearChat); + + // Connect button + connectBtn.addEventListener('click', handleConnect); + + // Send message + sendBtn.addEventListener('click', sendMessage); + messageInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + sendMessage(); + } + }); + + // Auto-resize textarea + messageInput.addEventListener('input', () => { + messageInput.style.height = 'auto'; + messageInput.style.height = Math.min(messageInput.scrollHeight, 80) + 'px'; + updateSendButton(); + }); + + // Suggestion chips + document.querySelectorAll('.suggestion-chip').forEach((chip) => { + chip.addEventListener('click', () => { + messageInput.value = chip.dataset.text; + updateSendButton(); + messageInput.focus(); + }); + }); + + // Full vision mode toggle + fullVisionToggle.addEventListener('change', (e) => { + fullVisionMode = e.target.checked; + chrome.storage.local.set({ fullVisionMode }); + chrome.runtime.sendMessage({ + type: 'SET_FULL_VISION', + enabled: fullVisionMode, + }); + }); + + // Debug mode toggle + debugModeToggle.addEventListener('change', (e) => { + isDebugMode = e.target.checked; + chrome.storage.local.set({ debugMode: isDebugMode }); + if (isDebugMode) { + debugPanel.classList.remove('hidden'); + settingsPanel.classList.add('hidden'); + } else { + debugPanel.classList.add('hidden'); + } + }); + + // Close debug panel + closeDebug.addEventListener('click', () => { + debugPanel.classList.add('hidden'); + isDebugMode = false; + debugModeToggle.checked = false; + }); + + // Debug command input + debugSendBtn.addEventListener('click', sendDebugCommand); + debugInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter') { + e.preventDefault(); + sendDebugCommand(); + } + }); +} + +async function handleConnect() { + if (isConnected) { + chrome.runtime.sendMessage({ type: 'DISCONNECT' }); + updateConnectionStatus(false); + } else { + connectBtn.textContent = 'Connecting...'; + connectBtn.disabled = true; + + const urlToConnect = serverUrlInput.value; + chrome.storage.local.set({ serverUrl: urlToConnect }); + chrome.runtime.sendMessage( + { + type: 'CONNECT', + serverUrl: urlToConnect, + }, + (response) => { + connectBtn.disabled = false; + + if (response && response.success) { + updateConnectionStatus(true); + } else { + updateConnectionStatus(false); + showSystemMessage( + 'Connection failed: ' + (response?.error || 'Unknown error'), + 'error' + ); + } + } + ); + } +} + +function updateConnectionStatus(connected) { + isConnected = connected; + + // Update settings panel + connectionDot.classList.toggle('connected', connected); + connectionText.textContent = connected ? 'Connected' : 'Disconnected'; + connectBtn.textContent = connected ? 'Disconnect' : 'Connect'; + connectBtn.classList.toggle('connected', connected); + + // Update status hint + statusHint.classList.toggle('connected', connected); + statusHint.querySelector('.hint-text').textContent = connected + ? 'Connected' + : 'Not connected'; + + updateSendButton(); +} + +function updateSendButton() { + if (isTaskRunning) { + const hasText = messageInput.value.trim().length > 0; + // Keep input enabled for queueing + messageInput.disabled = false; + + if (hasText) { + // Show send icon (will queue the message) + sendBtn.classList.remove('stop-mode'); + sendBtn.innerHTML = ` + + + + + `; + sendBtn.disabled = !isConnected; + } else { + // Show stop icon when no text + sendBtn.disabled = false; + sendBtn.classList.add('stop-mode'); + sendBtn.innerHTML = ` + + + + `; + } + + // Show queue badge + updateQueueBadge(); + } else { + sendBtn.classList.remove('stop-mode'); + sendBtn.innerHTML = ` + + + + + `; + messageInput.disabled = false; + const hasText = messageInput.value.trim().length > 0; + sendBtn.disabled = !hasText || !isConnected; + updateQueueBadge(); + } +} + +function updateQueueBadge() { + let badge = document.getElementById('queueBadge'); + if (messageQueue.length > 0) { + if (!badge) { + badge = document.createElement('span'); + badge.id = 'queueBadge'; + badge.style.cssText = + 'position:absolute;top:-6px;right:-6px;background:var(--primary);color:white;border-radius:50%;width:18px;height:18px;font-size:11px;display:flex;align-items:center;justify-content:center;font-weight:600;'; + sendBtn.style.position = 'relative'; + sendBtn.appendChild(badge); + } + badge.textContent = messageQueue.length; + } else if (badge) { + badge.remove(); + } +} + +function setTaskRunning(running) { + isTaskRunning = running; + updateSendButton(); +} + +async function sendMessage() { + const text = messageInput.value.trim(); + + // If task is running + if (isTaskRunning) { + if (text) { + // Queue the message + messageQueue.push(text); + messageInput.value = ''; + messageInput.style.height = 'auto'; + showSystemMessage( + `Message queued (${messageQueue.length} in queue)`, + 'info' + ); + updateSendButton(); + return; + } else { + // No text = stop task (stop button) + chrome.runtime.sendMessage({ type: 'STOP_TASK', tabId: currentTabId }); + setTaskRunning(false); + messageQueue = []; // Clear queue on stop + completeAgentMessage('Task stopped by user.'); + updateSendButton(); + return; + } + } + + if (!text || !isConnected) return; + + await executeMessage(text); +} + +async function executeMessage(text) { + // Update current tab info before sending + await updateCurrentTab(); + + // Clear input + messageInput.value = ''; + messageInput.style.height = 'auto'; + + // Set task running + setTaskRunning(true); + + // Hide welcome message + const welcome = messagesContainer.querySelector('.welcome-message'); + if (welcome) welcome.remove(); + + // Add user message + addMessage('user', text); + + // Add agent response placeholder + addAgentMessage(); + + // Send to background + chrome.runtime.sendMessage({ + type: 'EXECUTE_TASK', + task: text, + tabId: currentTabId, + url: currentTabUrl, + fullVisionMode: fullVisionMode, + }); +} + +// Process next message in queue +function processMessageQueue() { + if (messageQueue.length > 0 && !isTaskRunning && isConnected) { + const nextMessage = messageQueue.shift(); + updateQueueBadge(); + executeMessage(nextMessage); + } +} + +function addMessage(type, text) { + const msgDiv = document.createElement('div'); + msgDiv.className = `message message-${type}`; + msgDiv.innerHTML = ` +
+
${escapeHtml(text)}
+
+ `; + messagesContainer.appendChild(msgDiv); + scrollToBottom(); + return msgDiv; +} + +function addAgentMessage() { + const msgId = 'agent-msg-' + Date.now(); + const msgDiv = document.createElement('div'); + msgDiv.className = 'message message-agent'; + msgDiv.id = msgId; + msgDiv.innerHTML = ` +
+
+
+
+
+
+
+
+
+
+ + + +
+ Running Actions + (0) +
+
+ + + +
+
+
+
+
+
+
+
+
+ `; + messagesContainer.appendChild(msgDiv); + + // Add click listener for expand/collapse + const header = msgDiv.querySelector('.actions-header'); + header.addEventListener('click', () => { + const container = header.closest('.actions-container'); + container.classList.toggle('expanded'); + }); + + // Store action count + msgDiv.actionCount = 0; + + scrollToBottom(); + return msgId; +} + +function addActionStep(action, status = 'running') { + const msgDiv = document.querySelector('.message-agent:last-child'); + if (!msgDiv) return null; + + const actionsContainer = msgDiv.querySelector('.actions-container'); + const actionsList = msgDiv.querySelector('.actions-list'); + const currentActionDisplay = msgDiv.querySelector('.current-action-display'); + const typingIndicator = msgDiv.querySelector('.typing-indicator'); + + // Hide typing indicator + if (typingIndicator) typingIndicator.style.display = 'none'; + + // Show actions container + if (actionsContainer) { + actionsContainer.style.display = 'block'; + } + + const stepId = 'step-' + Date.now(); + const actionName = escapeHtml( + typeof action === 'string' ? action : action.name + ); + const actionTime = new Date().toLocaleTimeString('en-US', { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); + + // Create action step for the expanded list + const stepDiv = document.createElement('div'); + stepDiv.className = 'action-step'; + stepDiv.id = stepId; + stepDiv.innerHTML = ` +
+ ${getStatusIcon(status)} +
+
+
${actionName}${actionTime}
+
+ `; + actionsList.appendChild(stepDiv); + + // Update action count + msgDiv.actionCount = (msgDiv.actionCount || 0) + 1; + const countSpan = msgDiv.querySelector('.actions-count'); + if (countSpan) { + countSpan.textContent = `(${msgDiv.actionCount})`; + } + + // Update current action display with animation (only when collapsed) + if (currentActionDisplay) { + // Get previous action if exists + const prevAction = currentActionDisplay.querySelector('.current-action'); + + if (prevAction) { + // Move previous to fade out + prevAction.classList.remove('current-action'); + prevAction.classList.add('previous-action'); + + // Remove after animation + setTimeout(() => prevAction.remove(), 300); + } + + // Add new current action + const newAction = document.createElement('div'); + newAction.className = 'current-action'; + newAction.innerHTML = ` +
+ ${getStatusIcon(status)} +
+ ${actionName} + `; + newAction.dataset.stepId = stepId; + currentActionDisplay.appendChild(newAction); + } + + // Update header status icon + updateActionsHeaderStatus(msgDiv, status); + + scrollToBottom(); + return stepId; +} + +function updateActionStep(stepId, status) { + const stepDiv = document.getElementById(stepId); + if (!stepDiv) return; + + // Update in actions list + const iconDiv = stepDiv.querySelector('.action-icon'); + iconDiv.className = `action-icon ${status}`; + iconDiv.innerHTML = getStatusIcon(status); + + // Update in current action display if this is the current one + const msgDiv = stepDiv.closest('.message-agent'); + if (msgDiv) { + const currentAction = msgDiv.querySelector( + `.current-action[data-step-id="${stepId}"]` + ); + if (currentAction) { + const currentIcon = currentAction.querySelector('.current-action-icon'); + if (currentIcon) { + currentIcon.className = `current-action-icon ${status}`; + currentIcon.innerHTML = getStatusIcon(status); + } + } + + // Update header status based on overall state + updateActionsHeaderStatus(msgDiv, status); + } +} + +// Update the actions header status icon +function updateActionsHeaderStatus(msgDiv, _latestStatus) { + const statusIcon = msgDiv.querySelector('.actions-status-icon'); + const titleSpan = msgDiv.querySelector('.actions-title'); + + if (!statusIcon) return; + + // Check if any action is still running + const runningActions = msgDiv.querySelectorAll('.action-icon.running'); + const hasRunning = runningActions.length > 0; + + // Check for errors + const errorActions = msgDiv.querySelectorAll('.action-icon.error'); + const hasError = errorActions.length > 0; + + let overallStatus = 'success'; + let title = 'Actions Complete'; + + if (hasRunning) { + overallStatus = 'running'; + title = 'Running Actions'; + } else if (hasError) { + overallStatus = 'error'; + title = 'Actions (with errors)'; + } + + statusIcon.className = `actions-status-icon ${overallStatus}`; + statusIcon.innerHTML = getHeaderStatusIcon(overallStatus); + + if (titleSpan) { + titleSpan.textContent = title; + } +} + +// Get status icon for header +function getHeaderStatusIcon(status) { + switch (status) { + case 'running': + return ''; + case 'success': + return ''; + case 'error': + return ''; + default: + return ''; + } +} + +// Append streaming text to the current agent message +function appendStreamingText(text) { + const msgDiv = document.querySelector('.message-agent:last-child'); + if (!msgDiv) return; + + const typingIndicator = msgDiv.querySelector('.typing-indicator'); + if (typingIndicator) typingIndicator.style.display = 'none'; + + let streamingDiv = msgDiv.querySelector('.streaming-text'); + if (!streamingDiv) { + const content = msgDiv.querySelector('.message-content'); + streamingDiv = document.createElement('div'); + streamingDiv.className = 'streaming-text'; + content.appendChild(streamingDiv); + } + + // Show the streaming div + streamingDiv.style.display = 'block'; + + // Append text with typing effect + streamingDiv.textContent += text; + scrollToBottom(); +} + +function completeAgentMessage(text) { + const msgDiv = document.querySelector('.message-agent:last-child'); + if (!msgDiv) return; + + const typingIndicator = msgDiv.querySelector('.typing-indicator'); + if (typingIndicator) typingIndicator.remove(); + + // Update header to show completion + const actionsContainer = msgDiv.querySelector('.actions-container'); + if (actionsContainer) { + // Mark all running actions as complete + const runningIcons = msgDiv.querySelectorAll('.action-icon.running'); + runningIcons.forEach((icon) => { + icon.className = 'action-icon success'; + icon.innerHTML = getStatusIcon('success'); + }); + + // Update current action icons + const runningCurrentIcons = msgDiv.querySelectorAll( + '.current-action-icon.running' + ); + runningCurrentIcons.forEach((icon) => { + icon.className = 'current-action-icon success'; + icon.innerHTML = getStatusIcon('success'); + }); + + // Update header status + updateActionsHeaderStatus(msgDiv, 'success'); + } + + // Check if we have streaming text that should become the final text + const streamingDiv = msgDiv.querySelector('.streaming-text'); + if (streamingDiv && streamingDiv.textContent) { + // Streaming text already contains the content, just style it + streamingDiv.className = 'message-text'; + if (actionsContainer && actionsContainer.style.display !== 'none') { + streamingDiv.style.marginTop = '8px'; + streamingDiv.style.paddingTop = '8px'; + streamingDiv.style.borderTop = '1px solid var(--border-color)'; + } + } else if (text) { + const content = msgDiv.querySelector('.message-content'); + + // Remove empty streaming div if exists + if (streamingDiv) streamingDiv.remove(); + + const textDiv = document.createElement('div'); + textDiv.className = 'message-text'; + if (actionsContainer && actionsContainer.style.display !== 'none') { + textDiv.style.marginTop = '8px'; + textDiv.style.paddingTop = '8px'; + textDiv.style.borderTop = '1px solid var(--border-color)'; + } + textDiv.textContent = text; + content.appendChild(textDiv); + } + + scrollToBottom(); +} + +function showSystemMessage(text, type = 'info') { + const msgDiv = document.createElement('div'); + msgDiv.className = 'message message-agent'; + const bgColor = + type === 'error' + ? '#ffe3e3' + : type === 'success' + ? '#d3f9d8' + : 'var(--bg-secondary)'; + const textColor = + type === 'error' + ? 'var(--error)' + : type === 'success' + ? 'var(--success)' + : 'var(--text-primary)'; + msgDiv.innerHTML = ` +
+
${escapeHtml(text)}
+
+ `; + messagesContainer.appendChild(msgDiv); + scrollToBottom(); +} + +function getStatusIcon(status) { + switch (status) { + case 'running': + return ''; + case 'success': + return ''; + case 'error': + return ''; + default: + return ''; + } +} + +function handleBackgroundMessage(message) { + console.log('Message from background:', message); + + switch (message.type) { + case 'CONNECTION_STATUS': + updateConnectionStatus(message.connected); + if (message.reconnecting) { + statusHint.querySelector('.hint-text').textContent = + `Reconnecting (attempt ${message.attempt})...`; + } else if (message.failed) { + statusHint.querySelector('.hint-text').textContent = + 'Reconnection failed'; + } + break; + + case 'LOG': + handleLogMessage(message); + break; + + case 'ACTION': + window.currentStepId = addActionStep( + { + name: message.action, + detail: message.detail, + }, + 'running' + ); + break; + + case 'ACTION_COMPLETE': + if (window.currentStepId) { + updateActionStep( + window.currentStepId, + message.success ? 'success' : 'error' + ); + } + break; + + case 'STREAM_TEXT': + // Handle streaming text from agent (with truncation) + appendStreamingText(truncateOutput(message.text)); + break; + + case 'STREAM_START': + // Clear any existing streaming text for new stream + const msgDiv = document.querySelector('.message-agent:last-child'); + if (msgDiv) { + const streamingDiv = msgDiv.querySelector('.streaming-text'); + if (streamingDiv) streamingDiv.textContent = ''; + } + break; + + case 'STREAM_END': + // Stream ended, finalize the message + completeAgentMessage(null); + break; + + case 'TASK_COMPLETE': + completeAgentMessage(message.result); + setTaskRunning(false); + // Process next queued message + setTimeout(processMessageQueue, 500); + break; + + case 'TASK_ERROR': + if (window.currentStepId) { + updateActionStep(window.currentStepId, 'error'); + } + completeAgentMessage('Error: ' + message.error); + setTaskRunning(false); + // Process next queued message + setTimeout(processMessageQueue, 500); + break; + + case 'DEBUG_RESULT': + handleDebugResult(message); + break; + } +} + +function handleLogMessage(message) { + const level = message.level || 'info'; + const text = message.message; + + // Parse action from log message + if (text.includes('Executing:')) { + const actionName = text.replace('Executing:', '').trim(); + window.currentStepId = addActionStep({ name: actionName }, 'running'); + } else if (text.includes('Completed:')) { + if (window.currentStepId) { + updateActionStep(window.currentStepId, 'success'); + } + } else if (text.includes('Failed:')) { + if (window.currentStepId) { + updateActionStep(window.currentStepId, 'error'); + } + addActionStep({ name: text }, 'error'); + } else if (level === 'success' && !text.includes('Debugger')) { + addActionStep({ name: text }, 'success'); + } else if (level === 'error') { + addActionStep({ name: text }, 'error'); + } else if ( + text.includes('AI') || + text.includes('Sending task') || + text.includes('Processing') || + text.includes('Analyzing') + ) { + window.currentStepId = addActionStep({ name: text }, 'running'); + } else if (text.includes('Attaching') || text.includes('attached')) { + addActionStep({ name: text }, level === 'success' ? 'success' : 'running'); + } +} + +function clearChat() { + messagesContainer.innerHTML = ` +
+
+ + + + + + +
+

Welcome to CAMEL Browser Agent

+

Describe what you want to do on this page.

+
+ + + + +
+
+ `; + + // Re-attach suggestion chip listeners + document.querySelectorAll('.suggestion-chip').forEach((chip) => { + chip.addEventListener('click', () => { + messageInput.value = chip.dataset.text; + updateSendButton(); + messageInput.focus(); + }); + }); + + // Notify backend to clear context + if (isConnected) { + chrome.runtime.sendMessage({ type: 'CLEAR_CONTEXT' }); + } +} + +function scrollToBottom() { + messagesContainer.scrollTop = messagesContainer.scrollHeight; +} + +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +// Debug mode functions +async function sendDebugCommand() { + const command = debugInput.value.trim(); + if (!command || !isConnected) return; + + // Clear input + debugInput.value = ''; + + // Add command to output + addDebugLine(command, 'command'); + + // Update current tab info + await updateCurrentTab(); + + // Send to background + chrome.runtime.sendMessage({ + type: 'DEBUG_COMMAND', + command: command, + tabId: currentTabId, + url: currentTabUrl, + }); +} + +function addDebugLine(text, type = 'result') { + const line = document.createElement('div'); + line.className = `debug-line ${type}`; + line.textContent = text; + debugOutput.appendChild(line); + debugOutput.scrollTop = debugOutput.scrollHeight; +} + +function handleDebugResult(message) { + if (message.success) { + if (message.result) { + // Format the result + let resultText = message.result; + if (typeof resultText === 'object') { + resultText = JSON.stringify(resultText, null, 2); + } + addDebugLine(resultText, 'success'); + } else { + addDebugLine('Command executed successfully', 'success'); + } + } else { + addDebugLine(`Error: ${message.error}`, 'error'); + } +} diff --git a/src/components/ChatBox/BottomBox/InputBox.tsx b/src/components/ChatBox/BottomBox/InputBox.tsx index be5892709..e01517e45 100644 --- a/src/components/ChatBox/BottomBox/InputBox.tsx +++ b/src/components/ChatBox/BottomBox/InputBox.tsx @@ -33,10 +33,15 @@ import { X, Zap, } from 'lucide-react'; -import { useCallback, useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { toast } from 'sonner'; import { ExpandedInputBox } from './ExpandedInputBox'; +import { + BUILTIN_AGENTS, + MentionAgent, + MentionDropdown, +} from './MentionDropdown'; // Module-level singleton to track which InputBox instance has the expanded dialog open // This prevents multiple dialogs from opening when Cmd+P is pressed @@ -78,6 +83,10 @@ export interface InputboxProps { allowDragDrop?: boolean; /** Use cloud model in dev */ useCloudModelInDev?: boolean; + /** Active @mention target (e.g. "browser") — shown as a tag in the input */ + mentionTarget?: string | null; + /** Callback when mention target changes */ + onMentionTargetChange?: (target: string | null) => void; /** Callback when trigger is being created (for placeholder) */ onTriggerCreating?: (triggerData: TriggerInput) => void; /** Callback when trigger is created successfully */ @@ -138,9 +147,11 @@ export const Inputbox = ({ textareaRef: externalTextareaRef, allowDragDrop = false, useCloudModelInDev = false, - onTriggerCreating, - onTriggerCreated, - hideExpandButton = false, + mentionTarget, + onMentionTargetChange, + onTriggerCreating: _onTriggerCreating, + onTriggerCreated: _onTriggerCreated, + hideExpandButton: _hideExpandButton = false, }: InputboxProps) => { const { t } = useTranslation(); const internalTextareaRef = useRef(null); @@ -152,12 +163,16 @@ export const Inputbox = ({ const [isRemainingOpen, setIsRemainingOpen] = useState(false); const hoverCloseTimerRef = useRef(null); const [isComposing, setIsComposing] = useState(false); + const [mentionState, setMentionState] = useState<{ + visible: boolean; + filter: string; + startIndex: number; + }>({ visible: false, filter: '', startIndex: -1 }); const [isExpandedDialogOpen, setIsExpandedDialogOpen] = useState(false); const [triggerDialogOpen, setTriggerDialogOpen] = useState(false); - const expandedTextareaRef = useRef(null); - const instanceIdRef = useRef( - `inputbox-${Math.random().toString(36).substr(2, 9)}` - ); + const _expandedTextareaRef = useRef(null); + const reactId = React.useId(); + const instanceIdRef = useRef(`inputbox-${reactId}`); // Handle dialog open/close with singleton tracking const handleExpandedDialogChange = useCallback((open: boolean) => { @@ -228,8 +243,56 @@ export const Inputbox = ({ const hasContent = value.trim().length > 0 || files.length > 0; const isActive = isFocused || hasContent; - const handleTextChange = (newValue: string) => { + const handleTextChange = useCallback( + (newValue: string, cursorPos?: number) => { + onChange?.(newValue); + + // Detect @ mention + const pos = cursorPos ?? newValue.length; + const textBeforeCursor = newValue.slice(0, pos); + const lastAtIndex = textBeforeCursor.lastIndexOf('@'); + + if ( + lastAtIndex >= 0 && + (lastAtIndex === 0 || + textBeforeCursor[lastAtIndex - 1] === ' ' || + textBeforeCursor[lastAtIndex - 1] === '\n') + ) { + const filterText = textBeforeCursor.slice(lastAtIndex + 1); + if (!filterText.includes(' ')) { + setMentionState({ + visible: true, + filter: filterText, + startIndex: lastAtIndex, + }); + return; + } + } + setMentionState({ visible: false, filter: '', startIndex: -1 }); + }, + [onChange] + ); + + const handleMentionSelect = (agent: MentionAgent) => { + // Remove the "@filter" text from the input and set the + // mention target as a rendered tag instead + const currentValue = value; + const before = currentValue.slice(0, mentionState.startIndex); + const afterFilterEnd = + mentionState.startIndex + 1 + mentionState.filter.length; + const after = currentValue.slice(afterFilterEnd); + const newValue = `${before}${after}`.trimStart(); onChange?.(newValue); + onMentionTargetChange?.(agent.id); + setMentionState({ visible: false, filter: '', startIndex: -1 }); + + // Focus textarea + requestAnimationFrame(() => { + const el = textareaRef.current; + if (el) { + el.focus(); + } + }); }; const handleSend = () => { @@ -243,6 +306,23 @@ export const Inputbox = ({ }; const handleKeyDown = (e: React.KeyboardEvent) => { + // When mention dropdown is open, let it handle navigation keys + if (mentionState.visible) { + if (['ArrowUp', 'ArrowDown', 'Tab', 'Escape', 'Enter'].includes(e.key)) { + e.preventDefault(); // Stop textarea from scrolling / inserting newline + return; // Let MentionDropdown's global handler handle these + } + } + // Backspace at cursor position 0 removes the mention tag + if ( + e.key === 'Backspace' && + mentionTarget && + textareaRef.current?.selectionStart === 0 && + textareaRef.current?.selectionEnd === 0 + ) { + e.preventDefault(); + onMentionTargetChange?.(null); + } if (e.key === 'Enter' && !e.shiftKey && !disabled && !isComposing) { e.preventDefault(); handleSend(); @@ -350,15 +430,43 @@ export const Inputbox = ({ )} + {/* @Mention Dropdown */} + + setMentionState({ visible: false, filter: '', startIndex: -1 }) + } + /> + {/* Text Input Area */}
-
+
+ {/* @Mention Tag */} + {mentionTarget && ( + onMentionTargetChange?.(null)} + title="Click to remove" + > + @ + {BUILTIN_AGENTS.find((a) => a.id === mentionTarget)?.label ?? + mentionTarget} + + + )}