diff --git a/live-api/.gitignore b/live-api/.gitignore new file mode 100644 index 0000000..457b0b0 --- /dev/null +++ b/live-api/.gitignore @@ -0,0 +1,31 @@ +# Environment/runtime directories +.agents/ +.codex/ +.uv-cache/ +.venv/ +venv/ +env/ +node_modules/ + +# Local config and secrets +.config +.env +.env.* + +# Generated files +__pycache__/ +*.py[oc] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +apps/**/captures/ +spot/apps/api/captures/ +spot/apps/navigation/waypoints_map.html + +# OS/editor local files +.DS_Store +Thumbs.db +*.swp +*.swo +.idea/ +.vscode/ diff --git a/live-api/README.md b/live-api/README.md new file mode 100644 index 0000000..f8798f4 --- /dev/null +++ b/live-api/README.md @@ -0,0 +1,74 @@ +# Gemini Robotics ER - Live API Examples + +A repository of examples of connecting Gemini Robotics ER with physical robot embodiments using Live API for task orchestration, voice interactions, etc. The embodiments include Boston Dynamics Spot, Tinybot (a custom stationary robot hardware), and human operators. + +--- + +## Repository Structure & Packages + +| Package | Purpose & Features | Environment / Stack | Link | +| :--- | :--- | :--- | :--- | +| **`agent`** | **Physical Agent Server**: Core agent server interfacing with the Gemini Live API over WebSockets for real-time audio/video interaction, tool dispatch, and robot control. | Python 3.10+ (FastAPI, WebSockets, `google-genai`, `uv`) | [`./agent`](./agent/README.md) | +| **`spot`** | **Boston Dynamics Spot SDK Integration**: Suite of CLI tools, REST APIs, object detection/manipulation pipelines, and autonomous delivery applications for Spot. | Python (`bosdyn-client`, `google-genai`), Node.js/React (`apps/hydration`) | [`./spot`](./spot/README.md) | +| **`tinybot`** | **Compact Robot Controller**: Lightweight server providing camera video streaming and basic REST API controls for compact or custom robot hardware. | Python (FastAPI, OpenCV) | [`./tinybot`](./tinybot) | + +--- + +## Detailed Package Overview + +### 1. [Agent Server (`./agent`)](./agent/README.md) + +The `agent` directory contains the main orchestration server connecting multimodal AI models to physical hardware. + +* **Gemini Live API Integration**: Handles bi-directional audio/video streaming with Gemini Live API using WebSockets. +* **Embodiment Architecture**: Modular client abstractions (`agent/embodiment/`) to control different physical targets (`spot`, `tinybot`, `human`). +* **Web UI & Camera Poller**: Includes a built-in web interface and camera polling service (`camera_poller.py`) for real-time visual feeds. +* **Quickstart**: + ```bash + cd agent + UV_CACHE_DIR=.uv-cache uv sync + UV_CACHE_DIR=.uv-cache uv run python server.py --port 8000 + ``` + +--- + +### 2. [Spot Applications & SDK (`./spot`)](./spot/README.md) + +The `spot` directory contains Boston Dynamics Spot integrations powered by `bosdyn-client` and Gemini vision tools. + +* **Navigation App ([`apps/navigation`](./spot/apps/navigation))**: Manage GraphNav waypoints, register named locations, and command Spot to navigate autonomously via CLI. +* **Manipulation App ([`apps/manipulation`](./spot/apps/manipulation))**: Arm deployment, Gemini-based 2D/3D object detection, force-change detection, and picking. +* **FastAPI Server ([`apps/api`](./spot/apps/api))**: Exposes HTTP REST endpoints for Spot movement, leases, arm control, and waypoints (`http://localhost:8000/docs`). +* **Hydration Delivery Service ([`apps/hydration`](./spot/apps/hydration))**: Full-stack Node/React app and order worker that commands Spot to deliver drinks. +* **Quickstart**: + ```bash + cd spot + UV_CACHE_DIR=.uv-cache uv sync + UV_CACHE_DIR=.uv-cache uv run uvicorn apps.api.main:app --host 127.0.0.1 --port 8000 + ``` + +--- + +### 3. [Tinybot Hardware Controller (`./tinybot`)](./tinybot) + +The `tinybot` directory provides lightweight camera streaming and basic hardware control endpoints for smaller physical robot hardware. + +* **Camera Streamer ([`src/robot/camera_streamer.py`](./tinybot/src/robot/camera_streamer.py))**: Captures and streams live camera feeds for vision processing. +* **Robot REST API ([`src/robot/robot_api.py`](./tinybot/src/robot/robot_api.py))**: Exposes REST endpoints for low-level movement execution. +* **Quickstart**: + ```bash + cd tinybot + ./setup.sh + ./run_robot.sh + ``` + +--- + +## Environment Setup + +All Python subpackages use [`uv`](https://github.com/astral-sh/uv) for fast, deterministic dependency management. To keep virtual environments isolated and clean: + +```bash +# Sync dependencies within any subfolder using local cache: +UV_CACHE_DIR=.uv-cache uv sync +``` diff --git a/live-api/agent/README.md b/live-api/agent/README.md new file mode 100644 index 0000000..b753401 --- /dev/null +++ b/live-api/agent/README.md @@ -0,0 +1,78 @@ +# Physical Agent Server + +Physical Agent Server managed with [`uv`](https://github.com/astral-sh/uv). It exposes a FastAPI server, WebSocket endpoints for live streaming audio/video with Gemini Live API, and robot embodiment integrations (Spot, Human, Tinybot). + +--- + +## Prerequisites + +- **Python**: `>=3.10` +- **`uv`**: Installed locally (e.g. `uv 0.11+`) + +--- + +## Quickstart + +### 1. Synchronize Dependencies + +Run `uv sync` to set up the virtual environment: + +```bash +UV_CACHE_DIR=.uv-cache uv sync --default-index https://pypi.org/simple +``` + +--- + +### 2. Set API Keys (Optional) + +Set your Gemini API key: + +```bash +export GEMINI_API_KEY="your_api_key_here" +``` + +--- + +### 3. Start the Agent Server + +#### Basic Launch +```bash +UV_CACHE_DIR=.uv-cache uv run --default-index https://pypi.org/simple python server.py --port 8000 +``` + +#### Launch with Custom Model & Spot Robot Endpoint +```bash +UV_CACHE_DIR=.uv-cache uv run --default-index https://pypi.org/simple python server.py \ + --model gemini-3.1-flash-live-preview \ + --robot_url http://:8000 \ + --port 8000 +``` + +Once running, access the web UI at: +- **Local Web UI**: `http://localhost:8000` +- **API Documentation**: `http://localhost:8000/docs` + +--- + +## Running Tests + +Run all unit tests via `uv`: + +```bash +UV_CACHE_DIR=.uv-cache uv run --default-index https://pypi.org/simple python -m unittest discover -p "*_test.py" +``` + +--- + +## Command Line Arguments + +| Argument | Description | Default | +|---|---|---| +| `--model` | Gemini model name | `gemini-3.1-flash-live-preview` | +| `--robot_url` | Robot OpenAPI / HTTP base URL | `None` | +| `--api_key` | Gemini API Key | `GEMINI_API_KEY` env var | +| `--use_tts` / `--no_tts` | Enable or disable Text-to-Speech | `Disabled` | +| `--port` | HTTP server port | `8000` | +| `--media_resolution` | Video input frame resolution (`low`, `medium`, `high`, `ultra_high`) | `low` | +| `--heartbeat_enabled` | Enable proactive heartbeat | `True` | +| `--agent_peers` | Comma-separated peer agents (`name=url,name=url`) | `None` | diff --git a/live-api/agent/__init__.py b/live-api/agent/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/live-api/agent/__init__.py @@ -0,0 +1 @@ + diff --git a/live-api/agent/agent/__init__.py b/live-api/agent/agent/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/live-api/agent/agent/__init__.py @@ -0,0 +1 @@ + diff --git a/live-api/agent/agent/agent.py b/live-api/agent/agent/agent.py new file mode 100644 index 0000000..0276f1e --- /dev/null +++ b/live-api/agent/agent/agent.py @@ -0,0 +1,96 @@ +"""Agent configuration for Proactive Agent. + +An Agent is a named composition of system instruction (SI), developer +instruction (DI), and tool declarations. It is embodiment-agnostic — +the same Agent can be used with any embodiment. + +Typical usage: + + agent = Agent.from_name("human") + print(agent.system_instruction) + print(agent.tools) +""" + +import dataclasses +from typing import Any + +from prompt import si_builder +from tool import tools as tools_lib + + +@dataclasses.dataclass +class Agent: + """A named composition of prompt (system instruction) and tools.""" + + name: str + system_instruction: str + developer_instruction: str + tools: list[dict[str, Any]] + + # --------------------------------------------------------------------------- + # Pre-built presets + # --------------------------------------------------------------------------- + + @classmethod + def human(cls) -> "Agent": + """Local/browser mode with webcam.""" + builder = si_builder.SIBuilder() + builder.load_instruction_file("human_di.md") + return cls( + name="human", + system_instruction="", + developer_instruction=builder.build(), + tools=tools_lib.human_tools(), + ) + + @classmethod + def spot(cls) -> "Agent": + """Boston Dynamics Spot robot agent.""" + builder = si_builder.SIBuilder() + builder.load_instruction_file("spot_di.md") + return cls( + name="spot", + system_instruction="", + developer_instruction=builder.build(), + tools=tools_lib.spot_tools(), + ) + + @classmethod + def tinybot(cls) -> "Agent": + """Tinybot robot agent.""" + builder = si_builder.SIBuilder() + builder.load_instruction_file("tinybot_di.md") + return cls( + name="tinybot", + system_instruction="", + developer_instruction=builder.build(), + tools=tools_lib.tinybot_tools(), + ) + + # --------------------------------------------------------------------------- + # Name-based lookup + # --------------------------------------------------------------------------- + + @classmethod + def from_name(cls, name: str) -> "Agent": + """Returns an Agent for the specified name string. + + Args: + name: "human", "spot", or "tinybot" is supported for Lite. + + Returns: + An Agent instance. + + Raises: + ValueError: If the name is unknown. + """ + presets = { + "human": cls.human, + "spot": cls.spot, + "tinybot": cls.tinybot, + } + if name not in presets: + raise ValueError( + f"Unknown agent name: {name}. Available: {list(presets.keys())}" + ) + return presets[name]() diff --git a/live-api/agent/camera_poller.py b/live-api/agent/camera_poller.py new file mode 100644 index 0000000..cbd860f --- /dev/null +++ b/live-api/agent/camera_poller.py @@ -0,0 +1,283 @@ +"""Background camera poller that stitches camera images (Lite version).""" + +import asyncio +import collections +from collections.abc import Callable +import io +import logging +import math +import time + +from PIL import Image +from PIL import ImageDraw + +logger = logging.getLogger(__name__) + +_STABILITY_SAMPLE_SIZE = (64, 48) + + +def frame_difference_score(previous: bytes, current: bytes) -> float: + """Return normalized mean luminance difference between two JPEG frames.""" + if not previous or not current: + return 1.0 + try: + previous_image = Image.open(io.BytesIO(previous)).convert("L").resize( + _STABILITY_SAMPLE_SIZE + ) + current_image = Image.open(io.BytesIO(current)).convert("L").resize( + _STABILITY_SAMPLE_SIZE + ) + previous_pixels = previous_image.tobytes() + current_pixels = current_image.tobytes() + difference = sum( + abs(left - right) + for left, right in zip(previous_pixels, current_pixels) + ) + return difference / (len(previous_pixels) * 255.0) + except Exception: # pylint: disable=broad-except + return 1.0 + + +def stitch_camera_images( + images: dict[str, bytes], + cell_size: int = 384, +) -> bytes: + """Stitch camera images into a grid. Returns JPEG bytes.""" + if not images: + return b"" + if len(images) == 1: + _, data = next(iter(images.items())) + img = Image.open(io.BytesIO(data)) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85) + return buf.getvalue() + + n = len(images) + cols = math.ceil(math.sqrt(n)) + rows = math.ceil(n / cols) + canvas = Image.new("RGB", (cols * cell_size, rows * cell_size), (0, 0, 0)) + draw = ImageDraw.Draw(canvas) + + for i, (name, jpeg_bytes) in enumerate(images.items()): + try: + img = Image.open(io.BytesIO(jpeg_bytes)) + img = img.resize((cell_size, cell_size), Image.LANCZOS) + row, col = divmod(i, cols) + x_offset = col * cell_size + y_offset = row * cell_size + + canvas.paste(img, (x_offset, y_offset)) + + # Draw camera name with a simple shadow for visibility + text = name.replace("_", " ").upper() + try: + draw.text((x_offset + 11, y_offset + 11), text, fill=(0, 0, 0)) + draw.text((x_offset + 10, y_offset + 10), text, fill=(0, 255, 240)) + except Exception: + pass # ignore font loading issues if any + except Exception as e: # pylint: disable=broad-except + logger.warning("Failed to process image %s: %s", name, e) + return b"" + + buf = io.BytesIO() + canvas.save(buf, format="JPEG", quality=85) + return buf.getvalue() + + +class CameraPoller: + """Polls cameras at a fixed rate and pushes stitched frames to a queue. + + Supports both streaming (via stream_camera) and polling (via + get_camera_snapshot fallback). + """ + + def __init__( + self, + robot_client, + video_input_queue: asyncio.Queue, + camera_ids: list[str], + poll_hz: float = 5.0, + push_hz: float = 1.0, + cell_size: int = 384, + stability_threshold: float = 0.04, + clock: Callable[[], float] = time.monotonic, + ): + """Initialize the camera poller.""" + self._robot = robot_client + self._queue = video_input_queue + self._camera_ids = camera_ids + self._poll_interval = 1.0 / poll_hz + self._push_interval = 1.0 / push_hz + self._cell_size = cell_size + self._stability_threshold = stability_threshold + self._clock = clock + self._running = False + self.push_enabled = True # Push stitched frames to Gemini video queue + + # Store latest individual frames + self._current_frames: dict[str, bytes] = {} + + # Store latest 10 stitched images + self._frame_buffer = collections.deque(maxlen=10) + self._frame_sequence = 0 + self._new_frame_cond = asyncio.Condition() + self._stability_frame = b"" + self._stable_since: float | None = None + self._last_frame_difference = 1.0 + + @property + def latest_frame(self) -> bytes: + """The most recent stitched JPEG frame.""" + if self._frame_buffer: + return self._frame_buffer[-1] + return b"" + + @property + def stable_for_seconds(self) -> float: + """Seconds the camera view has remained below the motion threshold.""" + if self._stable_since is None: + return 0.0 + return max(0.0, self._clock() - self._stable_since) + + def is_stable_for(self, seconds: float) -> bool: + """Return whether the camera view has been stable for the requested time.""" + return self.stable_for_seconds >= max(0.0, float(seconds)) + + def record_frame_stability( + self, + frame: bytes, + *, + observed_at: float | None = None, + ) -> float: + """Update the continuous stable interval from a newly captured frame.""" + now = self._clock() if observed_at is None else observed_at + if not self._stability_frame: + self._stable_since = now + score = 0.0 + else: + score = frame_difference_score(self._stability_frame, frame) + if score > self._stability_threshold: + self._stable_since = now + self._stability_frame = frame + self._last_frame_difference = score + return score + + async def get_stream(self): + """Async generator yielding stitched MJPEG frames as they arrive.""" + try: + while self._running: + async with self._new_frame_cond: + await self._new_frame_cond.wait() + if not self._frame_buffer: + continue + frame = self._frame_buffer[-1] + yield frame + except asyncio.CancelledError: + return + + async def wait_for_next_frame(self) -> bytes: + """Wait for and return a frame produced after this method is called.""" + async with self._new_frame_cond: + initial_sequence = self._frame_sequence + await self._new_frame_cond.wait_for( + lambda: ( + self._frame_sequence > initial_sequence or not self._running + ) + ) + if self._frame_sequence <= initial_sequence or not self._frame_buffer: + return b"" + return self._frame_buffer[-1] + + def stop(self) -> None: + self._running = False + + async def _stream_camera_task(self, camera_id: str): + """Background task reading an individual camera stream.""" + while self._running: + try: + # Try streaming first if supported. + async for frame in self._robot.stream_camera(camera_id): + if not self._running: + break + self._current_frames[camera_id] = frame + except ValueError: + # Fallback to polling snapshots if stream is not supported (like Spot). + logger.debug( + "Streaming not supported for camera %s, falling back to polling.", + camera_id, + ) + while self._running: + frame = await self._robot.get_camera_snapshot(camera_id) + if frame: + self._current_frames[camera_id] = frame + await asyncio.sleep(self._poll_interval) + except asyncio.CancelledError: + break + except Exception as e: + logger.warning("Camera stream task %s error: %s", camera_id, e) + await asyncio.sleep(1.0) + + async def run(self) -> None: + """Main polling loop. Call via asyncio.create_task().""" + self._running = True + last_push_time = 0.0 + + # Start independent streamer tasks + stream_tasks = [ + asyncio.create_task(self._stream_camera_task(cid)) + for cid in self._camera_ids + ] + + logger.info( + "CameraPoller started: cameras=%s poll=%.1fHz push=%.1fHz", + self._camera_ids, + 1.0 / self._poll_interval, + 1.0 / self._push_interval, + ) + + while self._running: + try: + # Check if we have received frames from all requested cameras + if all(cid in self._current_frames for cid in self._camera_ids): + images = {cid: self._current_frames[cid] for cid in self._camera_ids} + + # Run CPU-bound stitching in a thread pool to avoid blocking the asyncio event loop + loop = asyncio.get_event_loop() + stitched = await loop.run_in_executor( + None, stitch_camera_images, images, self._cell_size + ) + + if stitched: + self.record_frame_stability(stitched) + self._frame_buffer.append(stitched) + self._frame_sequence += 1 + async with self._new_frame_cond: + self._new_frame_cond.notify_all() + + # Push to Gemini at push_hz rate (only when enabled). + if self.push_enabled: + now = asyncio.get_event_loop().time() + if now - last_push_time >= self._push_interval: + # Replace any stale frame in queue with the latest. + while not self._queue.empty(): + try: + self._queue.get_nowait() + except asyncio.QueueEmpty: + break + await self._queue.put(stitched) + last_push_time = now + except asyncio.CancelledError: + break + except Exception as e: # pylint: disable=broad-except + logger.warning("CameraPoller stitch loop error: %s", e) + + await asyncio.sleep(self._poll_interval) + + self._running = False + for task in stream_tasks: + task.cancel() + + async with self._new_frame_cond: + self._new_frame_cond.notify_all() + + logger.info("CameraPoller stopped") diff --git a/live-api/agent/camera_poller_test.py b/live-api/agent/camera_poller_test.py new file mode 100644 index 0000000..8c7a80a --- /dev/null +++ b/live-api/agent/camera_poller_test.py @@ -0,0 +1,81 @@ +"""Tests for camera poller frame synchronization.""" + +import asyncio +import io +import unittest + +import camera_poller +from PIL import Image + + +def _jpeg(value: int) -> bytes: + image = Image.new("L", (80, 60), value) + output = io.BytesIO() + image.save(output, format="JPEG") + return output.getvalue() + + +class CameraPollerTest(unittest.IsolatedAsyncioTestCase): + + def test_requires_three_continuous_seconds_of_stable_frames(self): + now = [0.0] + poller = camera_poller.CameraPoller( + robot_client=None, + video_input_queue=asyncio.Queue(), + camera_ids=["hand_color_image"], + clock=lambda: now[0], + ) + + poller.record_frame_stability(_jpeg(80)) + now[0] = 2.9 + poller.record_frame_stability(_jpeg(80)) + self.assertFalse(poller.is_stable_for(3.0)) + + now[0] = 3.1 + poller.record_frame_stability(_jpeg(80)) + self.assertTrue(poller.is_stable_for(3.0)) + + def test_visible_frame_motion_restarts_stability_window(self): + now = [0.0] + poller = camera_poller.CameraPoller( + robot_client=None, + video_input_queue=asyncio.Queue(), + camera_ids=["hand_color_image"], + clock=lambda: now[0], + ) + + poller.record_frame_stability(_jpeg(40)) + now[0] = 2.5 + score = poller.record_frame_stability(_jpeg(220)) + + self.assertGreater(score, 0.04) + self.assertEqual(0.0, poller.stable_for_seconds) + now[0] = 5.4 + self.assertFalse(poller.is_stable_for(3.0)) + now[0] = 5.6 + self.assertTrue(poller.is_stable_for(3.0)) + + async def test_wait_for_next_frame_does_not_return_existing_frame(self): + poller = camera_poller.CameraPoller( + robot_client=None, + video_input_queue=asyncio.Queue(), + camera_ids=["hand_color_image"], + ) + poller._running = True # pylint: disable=protected-access + poller._frame_buffer.append(b"old") # pylint: disable=protected-access + poller._frame_sequence = 1 # pylint: disable=protected-access + + waiter = asyncio.create_task(poller.wait_for_next_frame()) + await asyncio.sleep(0) + self.assertFalse(waiter.done()) + + async with poller._new_frame_cond: # pylint: disable=protected-access + poller._frame_buffer.append(b"new") # pylint: disable=protected-access + poller._frame_sequence += 1 # pylint: disable=protected-access + poller._new_frame_cond.notify_all() # pylint: disable=protected-access + + self.assertEqual(b"new", await waiter) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/agent/core/__init__.py b/live-api/agent/core/__init__.py new file mode 100644 index 0000000..dcf1986 --- /dev/null +++ b/live-api/agent/core/__init__.py @@ -0,0 +1 @@ +# Empty init for core subpackage. diff --git a/live-api/agent/core/audio_handler.py b/live-api/agent/core/audio_handler.py new file mode 100644 index 0000000..470a66c --- /dev/null +++ b/live-api/agent/core/audio_handler.py @@ -0,0 +1,90 @@ +"""Handles audio accumulation, encoding, and playback. + +Subscribes to AUDIO_CHUNK events to accumulate raw PCM audio data, +and to TURN_COMPLETE events to encode the accumulated audio as a WAV +file and publish it as an AUDIO_RESPONSE event. +""" + +import asyncio +import base64 +import io +import logging +from typing import Awaitable, Callable, Optional +import wave + +from core import event_bus + +logger = logging.getLogger(__name__) + + +def _encode_wav(pcm_data: bytes) -> str: + """Encodes raw PCM audio bytes as a base64 WAV data URI.""" + wav_io = io.BytesIO() + with wave.open(wav_io, 'wb') as wf: + wf.setnchannels(1) # Mono + wf.setsampwidth(2) # 16-bit + wf.setframerate(24000) # 24kHz + wf.writeframes(pcm_data) + + wav_bytes = wav_io.getvalue() + audio_b64 = base64.b64encode(wav_bytes).decode('utf-8') + return f'data:audio/wav;base64,{audio_b64}' + + +class AudioResponseHandler: + """A handler that accumulates audio chunks and encodes them as WAV on turn complete. + + This handler is responsible for: + 1. Calling the audio_output_callback for real-time playback of each chunk. + 2. Accumulating raw PCM audio data across an entire model turn. + 3. On turn complete, encoding the accumulated audio as a base64 WAV data + URI and publishing an AUDIO_RESPONSE event for UI display. + """ + + def __init__( + self, + bus: event_bus.EventBus, + audio_output_callback: Optional[ + Callable[[bytes], Awaitable[None]] + ] = None, + ): + self._bus = bus + self._audio_output_callback = audio_output_callback + self._buffer = b'' + self._lock = asyncio.Lock() + bus.subscribe([event_bus.EventType.AUDIO_CHUNK], self._handle_chunk) + bus.subscribe( + [event_bus.EventType.TURN_COMPLETE], self._handle_turn_complete + ) + bus.subscribe([event_bus.EventType.INTERRUPTED], self._handle_interrupted) + + async def _handle_chunk(self, event: event_bus.Event) -> None: + """Accumulate an audio chunk and forward to playback callback.""" + audio_data = event.data + async with self._lock: + self._buffer += audio_data + if self._audio_output_callback: + await self._audio_output_callback(audio_data) + + async def _handle_turn_complete(self, event: event_bus.Event) -> None: + """Encode accumulated audio as WAV and publish AUDIO_RESPONSE.""" + async with self._lock: + if not self._buffer: + return + pcm_data = self._buffer + self._buffer = b'' + + audio_data_uri = await asyncio.to_thread(_encode_wav, pcm_data) + + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.AUDIO_RESPONSE, + source=event_bus.EventSource.ASSISTANT, + data={'audio_data': audio_data_uri}, + ) + ) + + async def _handle_interrupted(self, event: event_bus.Event) -> None: + """Clears accumulated audio buffer when the model is interrupted.""" + async with self._lock: + self._buffer = b'' diff --git a/live-api/agent/core/decision_making.py b/live-api/agent/core/decision_making.py new file mode 100644 index 0000000..c183828 --- /dev/null +++ b/live-api/agent/core/decision_making.py @@ -0,0 +1,304 @@ +"""Decides the next step based on model responses. + +Subscribes to MODEL_RESPONSE events and routes them into typed events +(TOOL_CALL, GEMINI_TEXT, AUDIO_CHUNK, TURN_COMPLETE, INTERRUPTED) for +downstream handlers. + +This Lite version parses JSON dicts from the public BidiGenerateContent +WebSocket API instead of protobuf messages. +""" + +import asyncio +import base64 +import time +from typing import Any, Awaitable, Callable + +import logging + +from core import event_bus + + +def _is_internal_heartbeat_text(text: str) -> bool: + return text.lstrip().startswith("[HEARTBEAT]") + + +class DecisionMaking: + """A component that routes model responses into typed events on the event bus. + + This component decides the next step based on model responses. Currently + this is implemented as pure event routing — each model response is parsed + and re-published as a more specific event type for downstream handlers. + + Future extensions may add planning, state tracking, or other decision + logic here. + """ + + def __init__( + self, + bus: event_bus.EventBus, + text_output_callback: Callable[[str], Awaitable[None]] | None = None, + audio_interrupt_callback: Callable[[], Awaitable[None]] | None = None, + ): + self._bus = bus + self._text_output_callback = text_output_callback + self._audio_interrupt_callback = audio_interrupt_callback + # Simple timing instead of telemetry tracker. + self._turn_start_time: float | None = None + self._first_response_time: float | None = None + # True while TTS synthesis + playback is in-flight. The heartbeat + # loop checks this to avoid sending prompts that would interrupt + # speech playback. + # + # Because the EventBus dispatches each handler as a separate + # asyncio.Task, turn_complete can be processed concurrently while a + # previous text_output_callback (TTS) is still running. We track + # in-flight TTS calls with a counter and only clear the flag when all + # calls have finished. + self.speaking = asyncio.Event() + self._pending_tts_count = 0 + self._state_lock = asyncio.Lock() + self._current_turn_has_tool_call = False + # The currently active non-blocking TTS background task, if any. + # Used to cancel in-flight TTS when new text arrives or the user + # interrupts the model. + self._active_tts_task: asyncio.Task | None = None + # Cumulative token usage from usageMetadata messages. + self._usage_metadata: dict[str, Any] = {} + bus.subscribe([event_bus.EventType.MODEL_RESPONSE], self._handle) + + # ---- TTS lifecycle management ------------------------------------------- + + async def _decrement_tts_and_maybe_clear(self) -> None: + """Decrement TTS counter; clear speaking when it hits zero.""" + async with self._state_lock: + self._pending_tts_count -= 1 + if self._pending_tts_count <= 0: + self._pending_tts_count = 0 + self.speaking.clear() + + async def _cancel_active_tts(self) -> None: + """Cancel the currently active TTS background task, if any. + + Awaits the task to ensure its cleanup (e.g. decrementing the speaking + counter) completes before returning. + """ + task = self._active_tts_task + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + self._active_tts_task = None + + async def _start_tts_task(self, text: str) -> None: + """Start TTS synthesis in a non-blocking background task. + + If a previous TTS task is still active, it is cancelled first and the + browser is notified to stop audio playback (via a ``tts_preempt`` + INTERRUPTED event). The new TTS task runs concurrently with + subsequent event processing (e.g. tool calls), eliminating the delay + between speech announcement and action execution. + + Args: + text: The text to synthesize. + """ + # Cancel any in-flight TTS before updating state to avoid deadlock + # (the cancelled task's finally block also acquires _state_lock). + if self._active_tts_task and not self._active_tts_task.done(): + await self._cancel_active_tts() + # Notify the browser to stop playing the old audio. + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.INTERRUPTED, + source=event_bus.EventSource.ASSISTANT, + data={'tts_preempt': True}, + ) + ) + + # Update speaking state for the new TTS call. + async with self._state_lock: + self.speaking.set() + self._pending_tts_count += 1 + + # Launch TTS in the background — returns immediately. + self._active_tts_task = asyncio.create_task(self._run_tts_background(text)) + + async def _run_tts_background(self, text: str) -> None: + """Execute the TTS callback in a background task. + + Handles both normal completion and cancellation. The speaking counter + is always decremented in the ``finally`` block to keep the state + consistent. + """ + try: + if self._text_output_callback: + await self._text_output_callback(text) + except asyncio.CancelledError: + logging.info('TTS cancelled for: %s...', text[:30]) + except Exception as e: # pylint: disable=broad-except + logging.error('text_output_callback failed: %s', e) + finally: + await self._decrement_tts_and_maybe_clear() + + # ---- Simple timing helpers ---------------------------------------------- + + def _mark_turn_start(self) -> None: + if self._turn_start_time is None: + self._turn_start_time = time.monotonic() + + def _mark_first_response(self) -> None: + if self._first_response_time is None: + self._first_response_time = time.monotonic() + + def _reset_timing(self) -> dict[str, Any]: + """Reset timing and return metrics for the completed turn.""" + metrics = {} + if self._turn_start_time and self._first_response_time: + metrics['ttft_ms'] = round( + (self._first_response_time - self._turn_start_time) * 1000, 1 + ) + if self._usage_metadata: + metrics['token_usage'] = dict(self._usage_metadata) + self._turn_start_time = None + self._first_response_time = None + self._usage_metadata = {} + return metrics + + # ---- Main event handler ------------------------------------------------- + + async def _handle(self, event: event_bus.Event) -> None: + """Process a MODEL_RESPONSE event. + + The event.data is a parsed JSON dictionary. + """ + response = event.data + self._mark_turn_start() + + # --- Tool call --- + if "toolCall" in response: + self._current_turn_has_tool_call = True + self._mark_first_response() + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.TOOL_CALL, + source=event_bus.EventSource.ASSISTANT, + data=response["toolCall"], + ) + ) + + # --- Server content --- + if "serverContent" in response: + await self._handle_server_content(response["serverContent"]) + + async def _handle_server_content(self, server_content: dict) -> None: + """Route server content to appropriate events.""" + + # Handle model turn (audio/text parts) + if "modelTurn" in server_content: + for part in server_content["modelTurn"].get("parts", []): + # In Gemini Live API, text parts look like: {"text": "Hello"} + # Some are thoughts if thought=True, though public API might not have this yet. + if part.get("thought"): + self._mark_first_response() + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.GEMINI_THOUGHT, + source=event_bus.EventSource.ASSISTANT, + data={"text": part.get("text", "")}, + ) + ) + elif "inlineData" in part: + self._mark_first_response() + # Publish raw audio chunk — AudioResponseHandler accumulates. + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.AUDIO_CHUNK, + source=event_bus.EventSource.ASSISTANT, + data=base64.b64decode(part["inlineData"].get("data", "")), + ) + ) + elif "text" in part: + if _is_internal_heartbeat_text(part["text"]): + continue + self._mark_first_response() + # Publish text event immediately so downstream handlers (UI, + # logging) receive it without waiting for TTS synthesis. + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.GEMINI_TEXT, + source=event_bus.EventSource.ASSISTANT, + data={"text": part["text"]}, + ) + ) + # Start TTS in a non-blocking background task. + await self._start_tts_task(part["text"]) + + # Output transcription disabled — these can contain canned error strings + # from the model's transcription layer (e.g. "An error occurred"). + # if "outputTranscription" in server_content: + # text = server_content["outputTranscription"].get("text", "") + # if text and not _is_internal_heartbeat_text(text): + # self._mark_first_response() + # await self._bus.publish( + # event_bus.Event( + # type=event_bus.EventType.GEMINI_TEXT, + # source=event_bus.EventSource.ASSISTANT, + # data={"text": text}, + # ) + # ) + + # Handle input transcription (user's audio translation to text) + if "inputTranscription" in server_content: + text = server_content["inputTranscription"].get("text", "") + if text: + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.USER_TRANSCRIPT, + source=event_bus.EventSource.USER, + data={"text": text}, + ) + ) + + # Handle turn complete. + if server_content.get("turnComplete"): + had_tool_call = self._current_turn_has_tool_call + self._current_turn_has_tool_call = False # Reset for next turn + logging.info('Turn complete (had_tool_call=%s)', had_tool_call) + + turn_metrics = self._reset_timing() + + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.TURN_COMPLETE, + source=event_bus.EventSource.ASSISTANT, + data={'had_tool_call': had_tool_call}, + ) + ) + + if turn_metrics: + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.TELEMETRY, + source=event_bus.EventSource.ASSISTANT, + data=turn_metrics, + ) + ) + + # Handle interruption. + if server_content.get("interrupted"): + self._reset_timing() + # Cancel any active TTS playback immediately so audio stops as + # soon as the user interrupts. + await self._cancel_active_tts() + if self._audio_interrupt_callback: + try: + await self._audio_interrupt_callback() + except Exception as e: # pylint: disable=broad-except + logging.error('audio_interrupt_callback failed: %s', e) + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.INTERRUPTED, + source=event_bus.EventSource.ASSISTANT, + ) + ) diff --git a/live-api/agent/core/decision_making_test.py b/live-api/agent/core/decision_making_test.py new file mode 100644 index 0000000..3488f4c --- /dev/null +++ b/live-api/agent/core/decision_making_test.py @@ -0,0 +1,49 @@ +"""Tests for model response routing.""" + +import unittest + +from core import decision_making +from core import event_bus + + +class FakeBus: + + def __init__(self): + self.events = [] + + def subscribe(self, _event_types, _handler): + return None + + async def publish(self, event): + self.events.append(event) + + +class DecisionMakingTest(unittest.IsolatedAsyncioTestCase): + + async def test_hides_heartbeat_output_transcription(self): + bus = FakeBus() + router = decision_making.DecisionMaking(bus) + + await router._handle_server_content({ + "outputTranscription": { + "text": "[HEARTBEAT] inspect the scene and call ack" + } + }) + + self.assertEqual([], bus.events) + + async def test_routes_normal_output_transcription(self): + bus = FakeBus() + router = decision_making.DecisionMaking(bus) + + await router._handle_server_content({ + "outputTranscription": {"text": "I can see the desk."} + }) + + self.assertEqual(1, len(bus.events)) + self.assertEqual(event_bus.EventType.GEMINI_TEXT, bus.events[0].type) + self.assertEqual({"text": "I can see the desk."}, bus.events[0].data) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/agent/core/event_bus.py b/live-api/agent/core/event_bus.py new file mode 100644 index 0000000..b72fa8a --- /dev/null +++ b/live-api/agent/core/event_bus.py @@ -0,0 +1,347 @@ +"""Standalone async event bus for the Proactive Agent. + +Provides typed pub/sub event dispatching with per-handler task isolation. +Inspired by the Safari agent framework's EventBus pattern but fully +standalone with zero external dependencies. + +Architecture +------------ +The EventBus wraps a single ``asyncio.Queue`` with typed pub/sub semantics: + + publish(event) → Queue → _event_queue_loop → _dispatch_event → fan-out + +Each registered handler receives its own ``asyncio.Task``, so handlers run +concurrently without blocking each other or the dispatch loop. + +Performance +----------- +A single EventBus instance is used for the entire agent. This is efficient +because: + +* The internal queue is an unbounded ``asyncio.Queue`` — no contention. +* Dispatch is O(handlers-per-event-type), typically < 5. +* Each handler runs in its own task — no blocking between handlers. +* Hot-path data (audio/video frames) still flows directly via + ``stream.Send()`` in Observation; the bus carries only metadata events + (REAL_TIME_IMAGE_SENT, TEXT_INPUT) for logging and UI updates. + +Multiple bus instances are NOT needed. The overhead of a single bus dispatch +(enqueue + dict lookup + create_task) is negligible compared to network I/O. + +Event Sources +------------- +Events are categorized by two sources: + +* **ASSISTANT** — Events produced by the AI model, agent logic, and system + infrastructure (model responses, tool call requests, audio/text output, + system logs, errors). +* **USER** — Events originating from the external world: human input, + robot/embodiment actions, sensor data, and tool execution results. + +Event Types +----------- +Event types are organized by source category. See ``EventType`` enum for +the full list with descriptions. +""" + +import asyncio +import collections +import collections.abc +import dataclasses +import datetime +import enum +import logging +from typing import Any, Callable + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Event Source +# --------------------------------------------------------------------------- + + +@enum.unique +class EventSource(enum.Enum): + """Identifies which side of the agent boundary produced an event. + + There are exactly two categories: + + * ASSISTANT — The AI model and its supporting infrastructure. This includes + Gemini model responses, tool call requests initiated by the model, text + and audio output generated by the model, system log messages, and + internal errors. + + * USER — The external world. This includes human text/audio input, camera + frames from the robot or webcam, tool execution results returned by the + embodiment/robot, and any other data originating outside the model. + """ + + ASSISTANT = "ASSISTANT" + USER = "USER" + + +# --------------------------------------------------------------------------- +# Event Types (organized by source) +# --------------------------------------------------------------------------- + + +@enum.unique +class EventType(enum.Enum): + """All event types in the Proactive Agent. + + Organized by the source that typically produces them. + """ + + # === ASSISTANT events (from the AI model / agent system) ================ + + # Raw response from the Gemini BidiGenerateContent stream. + MODEL_RESPONSE = "MODEL_RESPONSE" + + # Model requests one or more tool calls. Payload: tool_call proto. + TOOL_CALL = "TOOL_CALL" + + # Model generated text output. Payload: {"text": str}. + GEMINI_TEXT = "GEMINI_TEXT" + + # Model generated thinking process (internal monologue). Payload: {"text": str}. + GEMINI_THOUGHT = "GEMINI_THOUGHT" + + # Raw audio chunk from model turn (pre-WAV encoding). + # Payload: bytes (PCM audio data). + AUDIO_CHUNK = "AUDIO_CHUNK" + + # Encoded audio response (WAV, base64). Published by AudioResponseHandler + # after accumulating AUDIO_CHUNKs. Payload: {"audio_data": str}. + AUDIO_RESPONSE = "AUDIO_RESPONSE" + + # Model finished its current turn. + TURN_COMPLETE = "TURN_COMPLETE" + + # Performance and token usage metrics. + TELEMETRY = "TELEMETRY" + + # Model turn was interrupted by new user input. + INTERRUPTED = "INTERRUPTED" + + # Session has terminated (stream closed or error). + SESSION_DONE = "SESSION_DONE" + + # Transparent history update from the model. Payload: TransparentHistoryUpdate proto. + TRANSPARENT_HISTORY = "TRANSPARENT_HISTORY" + + # System log message. Payload: dict with level, message, module, etc. + LOG = "LOG" + + # System or environment error. Payload: str (error description). + ERROR = "ERROR" + + # === USER events (from human / robot / environment) ===================== + + # User typed text input. Payload: str. + TEXT_INPUT = "TEXT_INPUT" + + # Transcription of user's audio input. Payload: {"text": str}. + USER_TRANSCRIPT = "USER_TRANSCRIPT" + + # Camera frame sent to the model. Payload: bytes (JPEG). + REAL_TIME_IMAGE_SENT = "REAL_TIME_IMAGE_SENT" + + # Tool execution result from the robot/embodiment. + # Payload: dict with name, args, result, action_data. + TOOL_RESULT = "TOOL_RESULT" + + # Request to send a heartbeat (flush buffers with fresh visual context). + HEARTBEAT_TRIGGER = "HEARTBEAT_TRIGGER" + + +# --------------------------------------------------------------------------- +# Event +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class Event: + """Immutable event with auto-timestamping. + + Attributes: + type: The event type (determines which handlers receive it). + source: Whether the event came from the ASSISTANT or USER side. + data: Arbitrary payload (proto, dict, bytes, str, etc.). + metadata: Optional key-value metadata for downstream handlers. + timestamp: UTC timestamp, auto-set on creation. + """ + + type: EventType + source: EventSource + data: Any = None + metadata: dict[str, Any] = dataclasses.field(default_factory=dict) + timestamp: datetime.datetime = dataclasses.field( + default_factory=lambda: datetime.datetime.now(datetime.timezone.utc) + ) + + +# Handler signature: async or sync callable taking an Event. +EventHandler = Callable[[Event], Any] + + +# --------------------------------------------------------------------------- +# EventBus +# --------------------------------------------------------------------------- + + +class EventBus: + """Async event bus with typed pub/sub and per-handler task isolation. + + Usage:: + + bus = EventBus() + bus.subscribe([EventType.TOOL_CALL], my_handler) + bus.start() + await bus.publish(Event(type=EventType.TOOL_CALL, ...)) + ... + await bus.shutdown() + + Thread safety: + ``publish_nowait()`` is safe to call from non-async callbacks (e.g., + gRPC threads) via ``loop.call_soon_threadsafe(bus.publish_nowait, + event)``. + """ + + def __init__(self): + self._handlers_by_event_type: dict[EventType, set[EventHandler]] = ( + collections.defaultdict(set) + ) + self._event_queue: asyncio.Queue[Event] = asyncio.Queue() + self._main_task: asyncio.Task[None] | None = None + self._handler_tasks: dict[str, asyncio.Task[None]] = {} + self._is_running = False + + @property + def is_running(self) -> bool: + return self._is_running + + def subscribe( + self, + event_types: collections.abc.Sequence[EventType], + handler: EventHandler, + ) -> None: + """Registers a handler for one or more event types. + + Multiple handlers can subscribe to the same event type. + A single handler can subscribe to multiple event types. + + Args: + event_types: Event types to subscribe to. + handler: Async or sync callable taking an Event. + """ + for event_type in event_types: + self._handlers_by_event_type[event_type].add(handler) + + def unsubscribe(self, handler: EventHandler) -> None: + """Removes a handler from all event types.""" + for handlers in self._handlers_by_event_type.values(): + handlers.discard(handler) + + def unsubscribe_all(self) -> None: + """Removes all handlers.""" + self._handlers_by_event_type.clear() + + async def publish(self, event: Event) -> None: + """Publishes an event to the bus. + + The event is enqueued and dispatched asynchronously to all registered + handlers. This method returns immediately after enqueuing. + + Args: + event: The event to publish. + """ + await self._event_queue.put(event) + + def publish_nowait(self, event: Event) -> None: + """Publishes an event without awaiting. For use from sync callbacks. + + Typically called via:: + + loop.call_soon_threadsafe(bus.publish_nowait, event) + + Args: + event: The event to publish. + """ + self._event_queue.put_nowait(event) + + def start(self) -> None: + """Starts the event dispatch loop as a background task.""" + if self._main_task and not self._main_task.done(): + logger.info("EventBus: dispatch loop already running.") + return + self._main_task = asyncio.create_task(self._event_queue_loop()) + self._is_running = True + logger.info("EventBus: started.") + + async def shutdown(self) -> None: + """Shuts down the bus: cancels dispatch loop and all pending handler tasks.""" + if not self._main_task or self._main_task.done(): + logger.info("EventBus: already stopped.") + return + + logger.info("EventBus: shutting down...") + self._main_task.cancel() + try: + await asyncio.wait_for(self._main_task, timeout=2.0) + except (asyncio.CancelledError, asyncio.TimeoutError): + pass + self._main_task = None + self._is_running = False + + # Cancel all pending handler tasks with a timeout. + tasks_to_cancel = [t for t in self._handler_tasks.values() if not t.done()] + if tasks_to_cancel: + logger.info( + "EventBus: cancelling %d pending handler tasks...", + len(tasks_to_cancel), + ) + for task in tasks_to_cancel: + task.cancel() + _, pending = await asyncio.wait(tasks_to_cancel, timeout=5.0) + if pending: + logger.warning( + "EventBus: %d handler tasks did not finish within 5s timeout.", + len(pending), + ) + self._handler_tasks.clear() + logger.info("EventBus: shutdown complete.") + + async def _event_queue_loop(self) -> None: + """Main dispatch loop: dequeues events and fans out to handlers.""" + while True: + try: + event = await self._event_queue.get() + await self._dispatch_event(event) + except asyncio.CancelledError: + break + + def _on_handler_done(self, task: asyncio.Task[None]) -> None: + """Logs exceptions when a handler task completes.""" + self._handler_tasks.pop(task.get_name(), None) + if not task.cancelled(): + exc = task.exception() + if exc: + logger.exception("EventBus handler failed: %s", exc) + + async def _dispatch_event(self, event: Event) -> None: + """Dispatches an event to all registered handlers for its type.""" + handlers = self._handlers_by_event_type.get(event.type) + if not handlers: + return + + for handler in handlers: + if asyncio.iscoroutinefunction(handler): + task = asyncio.create_task(handler(event)) + else: + # Sync handlers run in a thread to avoid blocking the loop. + task = asyncio.create_task(asyncio.to_thread(handler, event)) + + # Track the task and auto-remove on completion. + task_name = task.get_name() + self._handler_tasks[task_name] = task + task.add_done_callback(self._on_handler_done) diff --git a/live-api/agent/core/observation.py b/live-api/agent/core/observation.py new file mode 100644 index 0000000..c97ae42 --- /dev/null +++ b/live-api/agent/core/observation.py @@ -0,0 +1,295 @@ +"""Observation component for Proactive Agent. + +Receives audio, video, and text from the external environment and streams +them to the Gemini session as JSON dicts (public BidiGenerateContent API). +""" + +import asyncio +import base64 +import logging +import os + +from core import event_bus + + +class EventQueueLogHandler(logging.Handler): + """Logging handler that puts log messages into an asyncio.Queue.""" + + def __init__(self, bus: event_bus.EventBus, loop: asyncio.AbstractEventLoop): + super().__init__() + self.bus = bus + self.loop = loop + + def emit(self, record: logging.LogRecord): + try: + event = event_bus.Event( + type=event_bus.EventType.LOG, + source=event_bus.EventSource.ASSISTANT, + data={ + "level": record.levelname, + "message": record.getMessage(), + "module": record.module, + "funcName": record.funcName, + "lineno": record.lineno, + }, + ) + self.loop.call_soon_threadsafe(self.bus.publish_nowait, event) + except Exception: # pylint: disable=broad-exception-caught + self.handleError(record) + + +class Observation: + """Component for receiving information from the external environment and logging.""" + + def __init__( + self, + bus: event_bus.EventBus, + loop: asyncio.AbstractEventLoop, + session_manager, + session_start_time, + embodiment, + input_sample_rate=16000, + dump_video_dir=None, + episodic_logger=None, + media_resolution="low", + ): + self.bus = bus + self.loop = loop + self.session_manager = session_manager + self.session_start_time = session_start_time + self.input_sample_rate = input_sample_rate + self.dump_video_dir = dump_video_dir + self.episodic_logger = episodic_logger + self.media_resolution = media_resolution + self._frame_counter = 0 + # Event signaled after each video frame is written to the stream. + # Used by the heartbeat loop (non-poller mode) to wait for a fresh + # frame before sending ".". + self._frame_sent_event = asyncio.Event() + + self.set_embodiment(embodiment) + + # Setup custom logging handler + self.log_handler = EventQueueLogHandler(self.bus, self.loop) + logging.getLogger().addHandler(self.log_handler) + + + + def set_embodiment(self, embodiment): + """Updates the queues to read from the given embodiment. + + Restarts the input tasks so they read from the new queues. Without this, + the old tasks would be stuck blocked on queue.get() from the previous + embodiment's (now-unused) queues. + """ + self.audio_queue = embodiment.get_audio_queue() + self.video_queue = embodiment.get_video_queue() + self.text_queue = embodiment.get_text_queue() + + # Restart input tasks if they are running, so they pick up new queues. + if hasattr(self, 'send_audio_task'): + self.clear_queues() + self.send_audio_task.cancel() + if self.send_video_task: + self.send_video_task.cancel() + self.send_text_task.cancel() + self.start_input_tasks() + + def get_audio_queue(self) -> asyncio.Queue: + return self.audio_queue + + def get_video_queue(self) -> asyncio.Queue: + return self.video_queue + + def get_text_queue(self) -> asyncio.Queue: + return self.text_queue + + def clear_queues(self) -> None: + """Purges all pending audio, video, and text observation queues.""" + for attr in ("audio_queue", "video_queue", "text_queue"): + q = getattr(self, attr, None) + if q is not None: + while not q.empty(): + try: + q.get_nowait() + except asyncio.QueueEmpty: + break + logging.info("Purged observation queues.") + + def start_input_tasks(self, skip_video: bool = False): + self.send_audio_task = asyncio.create_task(self._send_audio()) + if not skip_video: + self.send_video_task = asyncio.create_task(self._send_video()) + else: + self.send_video_task = None + self.send_text_task = asyncio.create_task(self._send_text()) + + async def stop_input_tasks(self): + tasks = [] + if hasattr(self, "send_audio_task"): + self.send_audio_task.cancel() + tasks.append(self.send_audio_task) + if hasattr(self, "send_video_task") and self.send_video_task is not None: + self.send_video_task.cancel() + tasks.append(self.send_video_task) + if hasattr(self, "send_text_task"): + self.send_text_task.cancel() + tasks.append(self.send_text_task) + if tasks: + await asyncio.gather( + *tasks, + return_exceptions=True, + ) + + async def _send_audio(self): + try: + while True: + chunk = await self.audio_queue.get() + msg = { + "realtimeInput": { + "audio": { + "mimeType": f"audio/pcm;rate={self.input_sample_rate}", + "data": base64.b64encode(chunk).decode("utf-8"), + } + } + } + + await self.session_manager.send_message(msg) + # Only log when the audio chunk has actual speech energy (not silence). + # PCM 16-bit samples range ±32767; threshold filters mic noise. + if len(chunk) >= 2: + samples = memoryview(chunk).cast("h") # signed 16-bit + peak = max(abs(s) for s in samples) + if peak > 100: + logging.info( + "Streamed user audio to model (%d bytes, peak=%d)", + len(chunk), + peak, + ) + except asyncio.CancelledError: + pass + except Exception as e: + await self.bus.publish( + event_bus.Event( + type=event_bus.EventType.ERROR, + source=event_bus.EventSource.ASSISTANT, + data=f"send_audio died: {e}", + ) + ) + + async def _send_video(self): + try: + while True: + chunk = await self.video_queue.get() + await self.bus.publish( + event_bus.Event( + type=event_bus.EventType.REAL_TIME_IMAGE_SENT, + source=event_bus.EventSource.USER, + data=chunk, + ) + ) + + await self.dump_frame(chunk) + + msg = { + "realtimeInput": { + "video": { + "mimeType": "image/jpeg", + "data": base64.b64encode(chunk).decode("utf-8"), + } + } + } + + await self.session_manager.send_message(msg) + self._frame_sent_event.set() + logging.info("Sent video frame") + except asyncio.CancelledError: + pass + except Exception as e: + await self.bus.publish( + event_bus.Event( + type=event_bus.EventType.ERROR, + source=event_bus.EventSource.ASSISTANT, + data=f"send_video died: {e}", + ) + ) + + async def _send_text(self): + try: + while True: + text = await self.text_queue.get() + logging.info("Sending text: %s", text) + + # Ground each text turn in a frame captured immediately before it. + # Spot's UI can keep updating while the model otherwise retains an + # older image as its most recent visual context. + await self.bus.publish( + event_bus.Event( + type=event_bus.EventType.TEXT_INPUT, + source=event_bus.EventSource.USER, + data=text, + ) + ) + + msg = { + "clientContent": { + "turns": [{ + "role": "user", + "parts": [{"text": text}] + }], + "turnComplete": True, + } + } + + send_synced = getattr( + self.session_manager, "send_text_with_fresh_video", None + ) + if send_synced is not None: + await send_synced(text) + else: + await self.session_manager.send_message(msg) + logging.info("Streamed user text to model: %s", text) + except asyncio.CancelledError: + pass + except Exception as e: + await self.bus.publish( + event_bus.Event( + type=event_bus.EventType.ERROR, + source=event_bus.EventSource.ASSISTANT, + data=f"send_text died: {e}", + ) + ) + + async def wait_for_next_frame(self) -> None: + """Wait until _send_video delivers a fresh frame to the stream. + + Blocks indefinitely until _send_video sets the event after a + successful send_message call. Safe because: + - Only one caller (heartbeat loop) at a time + - Writes are serialized via asyncio.Lock in send_message, + so the frame is guaranteed to be in the server buffer when + the event fires + """ + self._frame_sent_event.clear() + await self._frame_sent_event.wait() + + async def dump_frame(self, chunk: bytes) -> None: + """Dumps a video frame to the configured directory.""" + if not self.dump_video_dir: + return + current_counter = self._frame_counter + self._frame_counter += 1 + path = os.path.join(self.dump_video_dir, f"frame_{current_counter:06d}.jpg") + + def _write_frame(file_path: str, data: bytes): + with open(file_path, "wb") as f: + f.write(data) + + try: + await asyncio.to_thread(_write_frame, path, chunk) + except Exception as e: + logging.warning("Failed to write video frame asynchronously: %s", e) + + def close(self): + """Removes the log handler to prevent leaks across sessions.""" + logging.getLogger().removeHandler(self.log_handler) diff --git a/live-api/agent/core/observation_test.py b/live-api/agent/core/observation_test.py new file mode 100644 index 0000000..2da13e2 --- /dev/null +++ b/live-api/agent/core/observation_test.py @@ -0,0 +1,54 @@ +"""Tests for synchronized observation input.""" + +import asyncio +import unittest + +from core import event_bus +from core import observation + + +class FakeBus: + + def __init__(self): + self.events = [] + + async def publish(self, event): + self.events.append(event) + + +class FakeSessionManager: + + def __init__(self): + self.operations = [] + self.message_sent = asyncio.Event() + + async def send_text_with_fresh_video(self, text): + self.operations.append("frame") + self.operations.append(("text", text)) + self.message_sent.set() + + +class ObservationTest(unittest.IsolatedAsyncioTestCase): + + async def test_sends_fresh_frame_before_user_text(self): + bus = FakeBus() + session = FakeSessionManager() + component = observation.Observation.__new__(observation.Observation) + component.bus = bus + component.session_manager = session + component.text_queue = asyncio.Queue() + + task = asyncio.create_task(component._send_text()) + await component.text_queue.put("what do you see?") + await asyncio.wait_for(session.message_sent.wait(), timeout=1.0) + task.cancel() + await task + + self.assertEqual("frame", session.operations[0]) + self.assertEqual("text", session.operations[1][0]) + self.assertEqual("what do you see?", session.operations[1][1]) + self.assertEqual(event_bus.EventType.TEXT_INPUT, bus.events[0].type) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/agent/core/tool_call_handler.py b/live-api/agent/core/tool_call_handler.py new file mode 100644 index 0000000..81271e5 --- /dev/null +++ b/live-api/agent/core/tool_call_handler.py @@ -0,0 +1,416 @@ +"""Handler that executes tool calls from the model. + +Subscribes to TOOL_CALL events, executes functions via the embodiment, +sends JSON dict responses back to the Gemini session, and publishes +TOOL_RESULT events. + +This Lite version uses JSON dicts for the public BidiGenerateContent API +instead of protobuf messages. +""" + +import asyncio +from collections.abc import Callable +import datetime +import json +import logging +import re +from typing import Any + +import httpx + +from core import event_bus + + +_PICK_CAMERA_STABLE_SECONDS = 3.0 + + +_NEGATED_SIT_PATTERN = re.compile( + r"(?:\b(?:do\s+not|don't|never)\b[^.!?]{0,40}\bsit\b|\bnot\s+to\s+sit\b)", + re.IGNORECASE, +) +_SIT_PATTERN = re.compile(r"\b(?:sit(?:\s+down)?|take\s+a\s+seat)\b", re.IGNORECASE) + + +def user_requested_sit(text: str) -> bool: + """Return whether a user utterance explicitly asks Spot to sit.""" + return bool(_SIT_PATTERN.search(text)) and not bool( + _NEGATED_SIT_PATTERN.search(text) + ) + + +def pick_response_for_model( + result: Any, + post_action_frame_sent: bool, +) -> dict[str, Any]: + """Describe pick completion without claiming visually unverified success.""" + if not isinstance(result, dict): + result = {"result": result} + + if ( + result.get("executed") is False + and "required_stable_seconds" in result + ): + return { + "outcome": "blocked_camera_unstable", + "executed": False, + "visual_verification_required": False, + "post_action_frame_sent": post_action_frame_sent, + "camera_stable_for_seconds": result.get( + "camera_stable_for_seconds", 0.0 + ), + "required_stable_seconds": result["required_stable_seconds"], + "instruction": result.get("retry_instruction", result.get("error", "")), + } + + failure_reasons = [] + if result.get("error"): + failure_reasons.append(str(result["error"])) + if result.get("success") is False: + failure_reasons.append("backend success diagnostic is false") + if result.get("holding_item") is False: + failure_reasons.append("gripper holding-item sensor is false") + backend_state = str(result.get("state", "")) + if "FAILED" in backend_state or "NO_SOLUTION" in backend_state: + failure_reasons.append(f"backend manipulation state is {backend_state}") + + response: dict[str, Any] = { + "outcome": "failed" if failure_reasons else "unverified", + "visual_verification_required": not failure_reasons, + "post_action_frame_sent": post_action_frame_sent, + "instruction": ( + "Inspect the fresh post-pick camera image. Claim success only if the" + " requested object is visibly secured by the gripper and has moved" + " from its original location. Backend completion and gripper sensor" + " signals are diagnostics, not proof of a successful pick. If the" + " image is missing or ambiguous, report that the pick is unverified." + ), + } + if backend_state: + response["backend_state"] = backend_state + if failure_reasons: + response["failure_reasons"] = failure_reasons + if isinstance(result.get("target"), dict): + response["target"] = result["target"] + return response + + +def blocking_tool_names(tools: list[dict[str, Any]] | None) -> frozenset[str]: + """Return the names of tools whose declarations mark them BLOCKING.""" + names = set() + for group in tools or []: + declarations = group.get("functionDeclarations", [group]) + for declaration in declarations: + if str(declaration.get("behavior", "")).upper() == "BLOCKING": + name = declaration.get("name") + if name: + names.add(name) + return frozenset(names) + + +class ToolCallHandler: + """A handler that executes tool calls and publishes results back to the event bus.""" + + def __init__( + self, + bus: event_bus.EventBus, + session_manager: Any, + embodiment: Any, + agent_peers: dict[str, str] | None = None, + peer_name: str = "unknown", + text_output_callback: Any | None = None, + clock: Callable[[], datetime.datetime] = datetime.datetime.utcnow, + enable_send_message_to_user: bool = False, + blocking_tools: frozenset[str] | set[str] | None = None, + ): + self._bus = bus + self._clock = clock + self._session_manager = session_manager + self._embodiment = embodiment + self._agent_peers = agent_peers or {} + self._peer_name = peer_name + # Optional TTS callback so send_message can speak the message + # out loud before delivering it to the peer agent. + self._text_output_callback = text_output_callback + self._enable_send_message_to_user = enable_send_message_to_user + self._blocking_tools = frozenset(blocking_tools or ()) + self._blocking_execution_count = 0 + self._sit_authorized = False + self._user_transcript = "" + # Set while any blocking tool is executing. The heartbeat + # loop checks this and skips sending prompts to avoid re-triggering + # the model before the tool result is in context. + self.tool_executing = asyncio.Event() + bus.subscribe([event_bus.EventType.TOOL_CALL], self._handle_tool_call) + bus.subscribe( + [event_bus.EventType.TEXT_INPUT, event_bus.EventType.USER_TRANSCRIPT], + self._record_user_instruction, + ) + + def set_session_manager(self, session_manager: Any): + """Updates the session manager reference.""" + self._session_manager = session_manager + + def set_embodiment(self, embodiment: Any): + """Updates the embodiment reference.""" + self._embodiment = embodiment + + + async def _handle_tool_call(self, event: event_bus.Event) -> None: + """Handle a TOOL_CALL event by executing all function calls.""" + tool_call = event.data + function_calls = tool_call.get("functionCalls", []) + for fc in function_calls: + try: + await self._execute_single_tool(fc) + except Exception: # pylint: disable=broad-except + logging.exception('Tool call %s failed', fc.get("name", "unknown")) + + async def _execute_single_tool(self, fc: dict) -> None: + """Execute a single function call and publish the result.""" + func_name = fc.get("name", "") + args = fc.get("args", {}) + call_id = str(fc.get("id", "")) + is_blocking = func_name in self._blocking_tools + if is_blocking: + self._blocking_execution_count += 1 + self.tool_executing.set() + try: + if func_name in {"detect", "pick", "place"}: + if func_name == "detect": + await self._clear_pick_target() + stability_error = self._camera_stability_error(func_name) + if stability_error is not None: + if func_name in {"pick", "place"}: + robot = getattr(self._embodiment, "robot", None) + clear_target = getattr(robot, "clear_detected_target", None) + if clear_target is not None: + clear_target() + result = stability_error + else: + result = await self._invoke_tool(func_name, args) + if func_name == "detect": + await self._publish_detected_target(result) + else: + result = await self._invoke_tool(func_name, args) + + # A blocking FunctionResponse resumes model generation. Send the latest + # visual context first instead of creating a competing heartbeat turn. + post_action_frame_sent = False + if is_blocking: + send_frame = getattr( + self._session_manager, "send_latest_video_frame", None + ) + if send_frame is not None: + post_action_frame_sent = bool(await send_frame()) + + # Ensure result is a dict for the JSON response. + if not isinstance(result, dict): + result_dict = {'result': result} + else: + result_dict = dict(result) + if func_name == "pick": + result_dict = pick_response_for_model(result, post_action_frame_sent) + + # Send tool response back to the Gemini session as a JSON dict. + msg = { + "toolResponse": { + "functionResponses": [{ + "id": call_id, + "name": func_name, + "response": result_dict + }] + } + } + + if self._session_manager is None: + raise RuntimeError('Session manager is not initialized') + await self._session_manager.send_message(msg) + + # Publish result for UI and logging. + action_data = json.dumps(fc) + + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.TOOL_RESULT, + source=event_bus.EventSource.USER, + data={ + "type": "tool_call", + "id": call_id, + "name": func_name, + "args": args, + "result": result, + "action_data": action_data, + }, + ) + ) + logging.info( + 'Tool call %s id=%s completed with result: %s', + func_name, + call_id, + result, + ) + finally: + if func_name in {"pick", "place"}: + await self._clear_pick_target() + if is_blocking: + self._blocking_execution_count = max( + 0, self._blocking_execution_count - 1 + ) + if self._blocking_execution_count == 0: + self.tool_executing.clear() + + async def _record_user_instruction(self, event: event_bus.Event) -> None: + """Authorize one sit call only after an explicit user request.""" + if event.type == event_bus.EventType.TEXT_INPUT: + self._user_transcript = str(event.data or "") + else: + data = event.data or {} + text = data.get("text", "") if isinstance(data, dict) else str(data) + self._user_transcript = f"{self._user_transcript} {text}".strip() + self._sit_authorized = user_requested_sit(self._user_transcript) + + def _camera_stability_error(self, action_name: str) -> dict[str, Any] | None: + """Reject pixel-grounded manipulation until the camera is stable.""" + poller = getattr(self._embodiment, "poller", None) + is_stable_for = getattr(poller, "is_stable_for", None) + if is_stable_for is None or is_stable_for(_PICK_CAMERA_STABLE_SECONDS): + return None + stable_for = float(getattr(poller, "stable_for_seconds", 0.0)) + if action_name in {"pick", "place"}: + next_action = action_name + else: + next_action = "pick or place" + return { + "error": ( + f"Camera view is not yet stable; {action_name} was not executed." + ), + "executed": False, + "camera_stable_for_seconds": round(stable_for, 2), + "required_stable_seconds": _PICK_CAMERA_STABLE_SECONDS, + "retry_instruction": ( + "Wait until the hand-camera view has remained stable for three" + " seconds and inspect the latest frame again. Call detect with an" + f" exact target description; after detection succeeds, call {next_action}" + " with no arguments and without moving the camera." + ), + } + + async def _publish_detected_target(self, result: Any) -> None: + """Show the backend-selected target without orchestrator pixel input.""" + if not isinstance(result, dict): + return + target = result.get("target") + if not isinstance(target, dict): + return + try: + x = float(target["normalized_x"]) + y = float(target["normalized_y"]) + except (KeyError, TypeError, ValueError): + return + if not 0 <= x <= 1000 or not 0 <= y <= 1000: + return + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.TOOL_RESULT, + source=event_bus.EventSource.ASSISTANT, + data={ + "type": "draw_points", + "points": [{"x": x, "y": y, "label": "detected target"}], + }, + ) + ) + + async def _clear_pick_target(self) -> None: + """Remove the pick marker after the blocking pick call finishes.""" + await self._bus.publish( + event_bus.Event( + type=event_bus.EventType.TOOL_RESULT, + source=event_bus.EventSource.ASSISTANT, + data={"type": "clear_overlay"}, + ) + ) + + async def _invoke_tool(self, func_name: str, args: dict[str, Any]) -> Any: + try: + if func_name == "sit": + if not self._sit_authorized: + return { + "executed": False, + "error": "Sit rejected: the user did not explicitly ask Spot to sit.", + } + self._sit_authorized = False + if func_name == "send_message": + return await self._execute_send_message(args) + return await self._embodiment.execute_action(func_name, **args) + except Exception as exc: # pylint: disable=broad-except + logging.error('Tool execution error for %s: %s', func_name, exc) + return f'Error: {exc}' + + async def _execute_send_message(self, args: dict[str, Any]) -> str: + """Send a message to another robot agent via HTTP POST. + + Waits for any in-flight TTS/audio playback to finish before sending, + so the receiving agent doesn't start speaking over the sender. + """ + target = args.get("target", "") + message = args.get("message", "") + + if not target: + return "Error: 'target' is required" + if not message: + return "Error: 'message' is required" + + if target == "user": + if not self._enable_send_message_to_user: + return ( + "Error: sending messages to user is not enabled." + " Use inline text instead." + ) + if self._text_output_callback: + + async def _run_tts(): + try: + logging.info( + "send_message to user: speaking message via TTS (non-blocking)" + ) + await self._text_output_callback(message) + except Exception as e: # pylint: disable=broad-except + logging.exception("TTS for send_message to user failed: %s", e) + + asyncio.create_task(_run_tts()) + return f"Message delivered to user: {message}" + + target_url = self._agent_peers.get(target) + if not target_url: + available = list(self._agent_peers.keys()) + return f"Error: unknown target '{target}'. Available peers: {available}" + + # Speak the message out loud via TTS before sending to peer, + # so the user hears what is being communicated and the message + # arrives after playback finishes. + if self._text_output_callback: + try: + logging.info( + "send_message to %s: speaking message via TTS first", target + ) + await self._text_output_callback(message) + except Exception as e: # pylint: disable=broad-except + logging.error("TTS for send_message failed: %s", e) + + try: + async with httpx.AsyncClient() as client: + resp = await client.post( + f"{target_url}/api/send", + json={"text": message, "source": self._peer_name}, + timeout=10.0, + ) + if resp.status_code == 200: + return f"Message delivered to {target}: {message}" + else: + return ( + f"Error sending to {target}: HTTP {resp.status_code} {resp.text}" + ) + except httpx.ConnectError: + return f"Error: could not connect to {target} at {target_url}" + except Exception as e: # pylint: disable=broad-except + return f"Error sending to {target}: {e}" diff --git a/live-api/agent/core/tool_call_handler_test.py b/live-api/agent/core/tool_call_handler_test.py new file mode 100644 index 0000000..b926189 --- /dev/null +++ b/live-api/agent/core/tool_call_handler_test.py @@ -0,0 +1,278 @@ +"""Tests for declaration-driven blocking tool handling.""" + +from __future__ import annotations + +import asyncio +import unittest +from unittest import mock + +from core import tool_call_handler + + +class FakeBus: + + def __init__(self): + self.events = [] + + def subscribe(self, _event_types, _handler): + return None + + async def publish(self, event): + self.events.append(event) + + +class FakeSessionManager: + + def __init__(self): + self.messages = [] + self.frames_sent = 0 + + async def send_message(self, message): + self.messages.append(message) + + async def send_latest_video_frame(self): + self.frames_sent += 1 + return True + + +class FakeEmbodiment: + + def __init__(self): + self.calls = [] + self.started = asyncio.Event() + self.release = asyncio.Event() + self.wait_for_release = False + + async def execute_action(self, action_name, **args): + self.calls.append((action_name, args)) + self.started.set() + if self.wait_for_release: + await self.release.wait() + return {"state": "done", "call_count": len(self.calls)} + + +class FakeUnstablePoller: + + stable_for_seconds = 1.25 + + def is_stable_for(self, seconds): + return False + + +class ToolCallHandlerTest(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + self.bus = FakeBus() + self.session = FakeSessionManager() + self.embodiment = FakeEmbodiment() + self.handler = tool_call_handler.ToolCallHandler( + self.bus, + self.session, + self.embodiment, + blocking_tools={"detect", "pick", "navigate"}, + ) + + async def test_marks_blocking_tool_active_while_it_executes(self): + self.embodiment.wait_for_release = True + execution = asyncio.create_task(self.handler._execute_single_tool({ + "id": "call-1", + "name": "pick", + "args": {}, + })) + await self.embodiment.started.wait() + self.assertTrue(self.handler.tool_executing.is_set()) + + self.embodiment.release.set() + await execution + + self.assertEqual(1, len(self.embodiment.calls)) + self.assertEqual(1, self.session.frames_sent) + self.assertFalse(self.handler.tool_executing.is_set()) + + event_types = [event.type for event in self.bus.events] + self.assertNotIn( + tool_call_handler.event_bus.EventType.HEARTBEAT_TRIGGER, + event_types, + ) + + async def test_executes_repeated_blocking_calls(self): + await self.handler._execute_single_tool({ + "id": "call-1", + "name": "navigate", + "args": {"name": "home"}, + }) + await self.handler._execute_single_tool({ + "id": "call-2", + "name": "navigate", + "args": {"name": "home"}, + }) + + self.assertEqual(2, len(self.embodiment.calls)) + self.assertEqual(2, self.session.frames_sent) + for message in self.session.messages: + response = message["toolResponse"]["functionResponses"][0]["response"] + self.assertNotIn("deduplicated", response) + + async def test_executes_batched_blocking_calls_in_order(self): + self.embodiment.wait_for_release = True + event = tool_call_handler.event_bus.Event( + type=tool_call_handler.event_bus.EventType.TOOL_CALL, + source=tool_call_handler.event_bus.EventSource.ASSISTANT, + data={ + "functionCalls": [ + {"id": "call-1", "name": "pick", "args": {}}, + {"id": "call-2", "name": "pick", "args": {}}, + ] + }, + ) + + execution = asyncio.create_task(self.handler._handle_tool_call(event)) + await self.embodiment.started.wait() + await asyncio.sleep(0) + self.assertEqual(1, len(self.embodiment.calls)) + + self.embodiment.release.set() + await execution + self.assertEqual( + [("pick", {}), ("pick", {})], + self.embodiment.calls, + ) + + async def test_executes_repeated_non_blocking_calls(self): + call = {"name": "get_battery", "args": {}} + await self.handler._execute_single_tool({"id": "call-1", **call}) + await self.handler._execute_single_tool({"id": "call-2", **call}) + + self.assertEqual(2, len(self.embodiment.calls)) + self.assertEqual(0, self.session.frames_sent) + + async def test_detect_publishes_backend_target_overlay(self): + self.embodiment.execute_action = mock.AsyncMock(return_value={ + "detected": True, + "target": {"normalized_x": 240, "normalized_y": 610}, + }) + + await self.handler._execute_single_tool({ + "id": "call-1", + "name": "detect", + "args": {"instruction": "red cube"}, + }) + + self.assertEqual("clear_overlay", self.bus.events[0].data["type"]) + self.assertEqual("draw_points", self.bus.events[1].data["type"]) + self.assertEqual( + [{"x": 240.0, "y": 610.0, "label": "detected target"}], + self.bus.events[1].data["points"], + ) + self.assertEqual("tool_call", self.bus.events[-1].data["type"]) + + async def test_detect_is_blocked_until_camera_is_stable_for_three_seconds(self): + self.embodiment.poller = FakeUnstablePoller() + + await self.handler._execute_single_tool({ + "id": "call-1", + "name": "detect", + "args": {"instruction": "red cube"}, + }) + + self.assertEqual([], self.embodiment.calls) + self.assertFalse(any( + event.data.get("type") == "draw_points" + for event in self.bus.events + )) + response = self.session.messages[-1]["toolResponse"]["functionResponses"][0][ + "response" + ] + self.assertFalse(response["executed"]) + self.assertEqual(3.0, response["required_stable_seconds"]) + self.assertEqual(1.25, response["camera_stable_for_seconds"]) + + async def test_pick_response_requires_visual_verification(self): + self.embodiment.execute_action = mock.AsyncMock(return_value={ + "state": "MANIP_STATE_GRASP_SUCCEEDED", + "success": True, + "holding_item": True, + }) + + await self.handler._execute_single_tool({ + "id": "call-1", + "name": "pick", + "args": {}, + }) + + response = self.session.messages[-1]["toolResponse"]["functionResponses"][0][ + "response" + ] + self.assertEqual("unverified", response["outcome"]) + self.assertTrue(response["visual_verification_required"]) + self.assertTrue(response["post_action_frame_sent"]) + self.assertNotIn("success", response) + self.assertNotIn("holding_item", response) + + async def test_pick_response_preserves_explicit_backend_failure(self): + self.embodiment.execute_action = mock.AsyncMock(return_value={ + "state": "MANIP_STATE_GRASP_FAILED", + "success": False, + "holding_item": False, + }) + + await self.handler._execute_single_tool({ + "id": "call-1", + "name": "pick", + "args": {}, + }) + + response = self.session.messages[-1]["toolResponse"]["functionResponses"][0][ + "response" + ] + self.assertEqual("failed", response["outcome"]) + self.assertFalse(response["visual_verification_required"]) + self.assertTrue(response["failure_reasons"]) + + async def test_sit_is_rejected_without_explicit_user_request(self): + result = await self.handler._invoke_tool("sit", {}) + + self.assertFalse(result["executed"]) + self.assertEqual([], self.embodiment.calls) + + async def test_explicit_user_request_authorizes_one_sit(self): + await self.handler._record_user_instruction( + tool_call_handler.event_bus.Event( + type=tool_call_handler.event_bus.EventType.TEXT_INPUT, + source=tool_call_handler.event_bus.EventSource.USER, + data="Go home and then sit down.", + ) + ) + + first = await self.handler._invoke_tool("sit", {}) + second = await self.handler._invoke_tool("sit", {}) + + self.assertEqual("done", first["state"]) + self.assertFalse(second["executed"]) + self.assertEqual([("sit", {})], self.embodiment.calls) + + def test_negated_sit_request_is_not_authorized(self): + self.assertFalse(tool_call_handler.user_requested_sit("Do not sit.")) + self.assertFalse(tool_call_handler.user_requested_sit("Never ask Spot to sit")) + self.assertTrue(tool_call_handler.user_requested_sit("Please sit down")) + + def test_extracts_all_declared_blocking_tools(self): + tools = [{ + "functionDeclarations": [ + {"name": "pick", "behavior": "BLOCKING"}, + {"name": "get_battery"}, + {"name": "navigate", "behavior": "blocking"}, + {"name": "stop", "behavior": "NON_BLOCKING"}, + ] + }] + + names = tool_call_handler.blocking_tool_names(tools) + + self.assertIn("pick", names) + self.assertIn("navigate", names) + self.assertNotIn("stop", names) + self.assertNotIn("get_battery", names) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/agent/embodiment/__init__.py b/live-api/agent/embodiment/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/live-api/agent/embodiment/__init__.py @@ -0,0 +1 @@ + diff --git a/live-api/agent/embodiment/base.py b/live-api/agent/embodiment/base.py new file mode 100644 index 0000000..441509b --- /dev/null +++ b/live-api/agent/embodiment/base.py @@ -0,0 +1,43 @@ +"""Abstract base class for embodiments (Lite version). + +Defines the minimal interface that every embodiment must implement. +""" + +import abc +import asyncio +from typing import Any + + +class Embodiment(abc.ABC): + """Abstract base class for an embodiment.""" + + @abc.abstractmethod + def get_audio_queue(self) -> asyncio.Queue: + """Returns the queue for audio observations.""" + + @abc.abstractmethod + def get_video_queue(self) -> asyncio.Queue: + """Returns the queue for video observations.""" + + @abc.abstractmethod + def get_text_queue(self) -> asyncio.Queue: + """Returns the queue for text observations.""" + + @abc.abstractmethod + async def execute_action(self, action_name: str, **kwargs: Any) -> str: + """Executes an action by name and returns a result string.""" + + @abc.abstractmethod + def get_tools(self) -> list[dict[str, Any]]: + """Returns tool declarations as a list of dicts (Gemini API JSON format). + + Each dict has the shape: + {"functionDeclarations": [{"name": ..., "description": ..., "parameters": {...}}]} + """ + + @abc.abstractmethod + def get_system_instruction(self) -> str: + """Returns the system instruction string for this embodiment.""" + + async def close(self) -> None: + """Releases resources. Override in subclasses that hold connections.""" diff --git a/live-api/agent/embodiment/human.py b/live-api/agent/embodiment/human.py new file mode 100644 index 0000000..3bd9367 --- /dev/null +++ b/live-api/agent/embodiment/human.py @@ -0,0 +1,64 @@ +"""Human embodiment for local/browser mode (Lite version). + +Implements the Embodiment interface for a human user interacting via +browser webcam and microphone. +""" + +import asyncio +import logging +from typing import Any + +from embodiment import base +from tool import tools as tools_lib +from prompt import si_builder + +logger = logging.getLogger(__name__) + + +class HumanEmbodiment(base.Embodiment): + """Embodiment for a human (local/browser mode).""" + + def __init__(self) -> None: + self.audio_queue: asyncio.Queue = asyncio.Queue(maxsize=50) + self.video_queue: asyncio.Queue = asyncio.Queue(maxsize=5) + self.text_queue: asyncio.Queue = asyncio.Queue(maxsize=10) + logger.info("HumanEmbodiment initialized") + + # ---- Queues ---- + + def get_audio_queue(self) -> asyncio.Queue: + return self.audio_queue + + def get_video_queue(self) -> asyncio.Queue: + return self.video_queue + + def get_text_queue(self) -> asyncio.Queue: + return self.text_queue + + # ---- Action execution ---- + + async def execute_action(self, action_name: str, **kwargs: Any) -> str: + if action_name == "ack": + return "ok" + if action_name == "send_message": + target = kwargs.get("target", "unknown") + message = kwargs.get("message", "") + logger.info( + "[HumanEmbodiment] send_message to %s: %s", target, message + ) + return f"Message sent to {target}" + + logger.info( + "[HumanEmbodiment] Executing action: %s with args: %s", + action_name, + kwargs, + ) + return f"Human executed {action_name}" + + # ---- Tools & SI ---- + + def get_tools(self) -> list[dict[str, Any]]: + return tools_lib.human_tools() + + def get_system_instruction(self) -> str: + return "" diff --git a/live-api/agent/embodiment/robot_client.py b/live-api/agent/embodiment/robot_client.py new file mode 100644 index 0000000..9ae8cdc --- /dev/null +++ b/live-api/agent/embodiment/robot_client.py @@ -0,0 +1,218 @@ +"""HTTP-based robot client (Lite version). + +Async client for communicating with a robot FastAPI backend over HTTP. +Uses httpx for async HTTP. +""" + +import asyncio +import base64 +import logging +from typing import Any, AsyncGenerator + +import httpx + +logger = logging.getLogger(__name__) + + +class RobotClient: + """Lightweight async HTTP client for a FastAPI robot server. + + Subclass and set CAMERA_IDS / ENDPOINT_MAP for your embodiment. + """ + + # Override in subclasses. + CAMERA_IDS: list[str] = [] + ENDPOINT_MAP: dict[str, str] = {} + + def __init__( + self, + base_url: str = "http://localhost:8888", + timeout: float = 5.0, + ) -> None: + self._base_url = base_url + self._client = httpx.AsyncClient( + base_url=base_url, + timeout=httpx.Timeout(timeout), + ) + self._camera_timeout = httpx.Timeout(10.0) + + # ----- Robot control ----- + + async def run_instruction(self, instruction: str) -> str: + """Send a natural-language instruction to the robot (fire-and-forget).""" + try: + resp = await self._client.get( + "/run_instruction/", params={"instruction": instruction} + ) + resp.raise_for_status() + msg = resp.json().get("message", "ok") + logger.info("run_instruction(%s) -> %s", instruction, msg) + return msg + except Exception as e: # pylint: disable=broad-except + logger.error("run_instruction error: %s", e) + return f"error: {e}" + + async def run_instruction_for_duration( + self, instruction: str, duration_seconds: float = 30.0 + ) -> str: + """Send a run instruction, sleep, then stop.""" + try: + resp = await self._client.get( + "/run_instruction/", params={"instruction": instruction} + ) + resp.raise_for_status() + msg = resp.json().get("message", "ok") + logger.info( + "run_instruction_for_duration(%s) started -> %s", instruction, msg + ) + await asyncio.sleep(duration_seconds) + stop_msg = await self.stop() + logger.info( + "run_instruction_for_duration(%s) stopped -> %s", + instruction, + stop_msg, + ) + return f"Started: {msg}, Slept: {duration_seconds}s, Stopped: {stop_msg}" + except Exception as e: # pylint: disable=broad-except + logger.error("run_instruction_for_duration error: %s", e) + return f"error: {e}" + + async def stop(self) -> dict | str: + """Stop the robot immediately.""" + try: + resp = await self._client.get("/stop/") + resp.raise_for_status() + msg = resp.json().get("message", "ok") + logger.info("stop() -> %s", msg) + return msg + except Exception as e: # pylint: disable=broad-except + logger.error("stop error: %s", e) + return f"error: {e}" + + async def reset(self) -> str: + """Reset robot to its default pose.""" + logger.info("reset() -> calling return to reset pose") + return await self.run_instruction_for_duration( + instruction="return to a reset pose until you can see the table", + duration_seconds=15.0, + ) + + async def make_gesture(self, gesture: str) -> str: + """Perform a gesture on the robot.""" + try: + resp = await self._client.get( + "/make_gesture/", params={"gesture": gesture} + ) + resp.raise_for_status() + msg = resp.json().get("message", "ok") + logger.info("make_gesture(%s) -> %s", gesture, msg) + return msg + except Exception as e: # pylint: disable=broad-except + logger.error("make_gesture error: %s", e) + return f"error: {e}" + + async def turn_head_to_uv(self, u: float, v: float) -> str: + """Turn the robot head to look at a normalized pixel coordinate.""" + try: + resp = await self._client.get( + "/turn_head_to_uv/", params={"u": u, "v": v} + ) + resp.raise_for_status() + msg = resp.json().get("message", "ok") + logger.info("turn_head_to_uv(u=%s, v=%s) -> %s", u, v, msg) + return msg + except Exception as e: # pylint: disable=broad-except + logger.error("turn_head_to_uv error: %s", e) + return f"error: {e}" + + # ----- Camera snapshots ----- + + async def get_camera_snapshot(self, camera_id: str) -> bytes | None: + """Fetch a single JPEG via /camera_image/?camera_id=X.""" + try: + resp = await self._client.get( + "/camera_image/", + params={"camera_id": camera_id}, + timeout=self._camera_timeout, + ) + resp.raise_for_status() + html = resp.text + marker = "base64," + idx = html.find(marker) + if idx < 0: + return None + start = idx + len(marker) + end = html.find('"', start) + if end < 0: + end = len(html) + return base64.b64decode(html[start:end]) + except Exception as e: # pylint: disable=broad-except + logger.warning("camera snapshot %s error: %s", camera_id, e) + return None + + async def stream_camera(self, camera_id: str) -> AsyncGenerator[bytes, None]: + """Stream MJPEG frames from the robot's FastAPI backend.""" + if camera_id not in self.ENDPOINT_MAP: + raise ValueError(f"Unknown camera ID for streaming: {camera_id}") + + endpoint = self.ENDPOINT_MAP[camera_id] + backoff = 0.5 + max_backoff = 5.0 + + while True: + client = httpx.AsyncClient( + base_url=self._base_url, timeout=httpx.Timeout(30.0) + ) + try: + async with client.stream("GET", endpoint) as response: + response.raise_for_status() + backoff = 0.5 + buffer = b"" + async for chunk in response.aiter_bytes(): + buffer += chunk + while True: + start_idx = buffer.find(b"\xff\xd8") + if start_idx == -1: + break + end_idx = buffer.find(b"\xff\xd9", start_idx) + if end_idx == -1: + break + jpeg_bytes = buffer[start_idx : end_idx + 2] + buffer = buffer[end_idx + 2 :] + yield jpeg_bytes + except asyncio.CancelledError: + await client.aclose() + return + except Exception: # pylint: disable=broad-except + pass + finally: + await client.aclose() + + await asyncio.sleep(backoff) + backoff = min(backoff * 2, max_backoff) + + async def get_all_snapshots( + self, + camera_ids: list[str] | None = None, + ) -> dict[str, bytes]: + """Fetch snapshots from all cameras concurrently.""" + ids = camera_ids or self.CAMERA_IDS + + async def _fetch(cid: str): + img = await self.get_camera_snapshot(cid) + return (cid, img) + + results = await asyncio.gather( + *[_fetch(cid) for cid in ids], return_exceptions=True + ) + images: dict[str, bytes] = {} + for item in results: + if isinstance(item, tuple): + name, data = item + if isinstance(data, bytes): + images[name] = data + return images + + async def close(self) -> None: + """Close the underlying HTTP client.""" + await self._client.aclose() diff --git a/live-api/agent/embodiment/spot/__init__.py b/live-api/agent/embodiment/spot/__init__.py new file mode 100644 index 0000000..9cc6686 --- /dev/null +++ b/live-api/agent/embodiment/spot/__init__.py @@ -0,0 +1 @@ +# Empty init for spot embodiment package diff --git a/live-api/agent/embodiment/spot/movement_tools_test.py b/live-api/agent/embodiment/spot/movement_tools_test.py new file mode 100644 index 0000000..a8f3de6 --- /dev/null +++ b/live-api/agent/embodiment/spot/movement_tools_test.py @@ -0,0 +1,274 @@ +"""Tests for direct Spot movement tools.""" + +from __future__ import annotations + +import json +import unittest + +import httpx + +from embodiment.spot import robot_client +from tool import tools + + +class MovementToolDeclarationsTest(unittest.TestCase): + + def test_spot_tools_include_navigation_and_bounded_motion(self): + declarations = tools.spot_tools()[0]["functionDeclarations"] + by_name = {declaration["name"]: declaration for declaration in declarations} + + self.assertIn("drive", by_name) + self.assertIn("stop", by_name) + self.assertIn("look", by_name) + self.assertIn("detect", by_name) + self.assertIn("pick", by_name) + self.assertIn("wait_for_pick_up", by_name) + self.assertIn("get_waypoints", by_name) + self.assertEqual("BLOCKING", by_name["drive"]["behavior"]) + self.assertEqual("BLOCKING", by_name["stop"]["behavior"]) + self.assertEqual("BLOCKING", by_name["look"]["behavior"]) + self.assertEqual(["instruction"], by_name["detect"]["parameters"]["required"]) + self.assertEqual({}, by_name["pick"]["parameters"]["properties"]) + self.assertEqual({}, by_name["place"]["parameters"]["properties"]) + self.assertIn( + "do not call at startup", by_name["get_waypoints"]["description"] + ) + self.assertIn( + "If null or absent", by_name["get_waypoints"]["description"] + ) + self.assertIn( + "without discussing localization", + by_name["get_waypoints"]["description"], + ) + self.assertIn( + "navigation_ready=true", by_name["navigate"]["description"] + ) + self.assertIn("call stow", by_name["navigate"]["description"]) + self.assertNotIn( + "kitchen", by_name["navigate"]["parameters"]["properties"]["waypoint"]["description"] + ) + self.assertIn( + "Do not call automatically at startup", + by_name["health_check"]["description"], + ) + self.assertEqual("BLOCKING", by_name["wait_for_pick_up"]["behavior"]) + + properties = by_name["drive"]["parameters"]["properties"] + self.assertEqual((-0.8, 0.8), (properties["v_x"]["minimum"], properties["v_x"]["maximum"])) + self.assertEqual((-0.5, 0.5), (properties["v_y"]["minimum"], properties["v_y"]["maximum"])) + self.assertEqual((-1.0, 1.0), (properties["v_rot"]["minimum"], properties["v_rot"]["maximum"])) + self.assertEqual( + (0.1, 2.0), + (properties["duration"]["minimum"], properties["duration"]["maximum"]), + ) + look_properties = by_name["look"]["parameters"]["properties"] + self.assertEqual( + ["up", "down", "left", "right"], + look_properties["direction"]["enum"], + ) + self.assertEqual( + (0.05, 0.35), + ( + look_properties["angle_rad"]["minimum"], + look_properties["angle_rad"]["maximum"], + ), + ) + + +class SpotMovementClientTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self): + self.requests: list[httpx.Request] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if request.url.path == "/waypoints": + return httpx.Response(200, json=[{"name": "home1"}]) + if request.url.path == "/localization": + return httpx.Response( + 200, + json={ + "localized": True, + "localization": {"waypoint_id": "home-id"}, + }, + ) + if request.url.path == "/detect/pick-target": + return httpx.Response( + 200, + json={ + "detected": True, + "instruction": "red cube", + "label": "red cube", + "confidence": 0.93, + "target": { + "normalized_x": 500, + "normalized_y": 400, + "pixel_x": 319.5, + "pixel_y": 191.6, + "image_width": 640, + "image_height": 480, + }, + }, + ) + return httpx.Response(200, json={"ok": True}) + + self.client = robot_client.SpotRobotClient(base_url="http://spot.test") + await self.client._client.aclose() # pylint: disable=protected-access + self.client._client = httpx.AsyncClient( # pylint: disable=protected-access + base_url="http://spot.test", + transport=httpx.MockTransport(handler), + ) + + async def asyncTearDown(self): + await self.client.close() + + async def test_drive_uses_guarded_velocity_request(self): + result = await self.client.drive( + v_x=0.3, + v_y=-0.1, + v_rot=0.2, + duration=0.5, + ) + + self.assertEqual({"ok": True}, result) + request = self.requests[-1] + self.assertEqual("/teleop/velocity", request.url.path) + self.assertEqual( + { + "v_x": 0.3, + "v_y": -0.1, + "v_rot": 0.2, + "duration": 0.5, + "take_lease": True, + "power_on": True, + "stand": True, + "body_follow_arm": True, + }, + json.loads(request.content), + ) + + async def test_stop_cancels_all_motion_and_freezes_arm(self): + result = await self.client.stop() + + self.assertEqual({"ok": True}, result) + request = self.requests[-1] + self.assertEqual("/actions/stop", request.url.path) + self.assertEqual( + {"take_lease": True, "freeze_arm": True}, + json.loads(request.content), + ) + + async def test_look_down_rotates_only_the_arm_camera(self): + result = await self.client.look(direction="down", angle_rad=0.2) + + self.assertEqual("down", result["direction"]) + request = self.requests[-1] + self.assertEqual("/arm/jog", request.url.path) + self.assertEqual( + { + "dpitch": 0.2, + "seconds": 0.8, + "take_lease": True, + "timeout": 3.0, + }, + json.loads(request.content), + ) + + async def test_get_waypoints_lists_backend_destinations(self): + result = await self.client.get_waypoints() + + self.assertEqual([{"name": "home1"}], result["waypoints"]) + self.assertTrue(result["navigation_ready"]) + self.assertEqual( + ["/waypoints", "/localization"], + [request.url.path for request in self.requests[-2:]], + ) + + async def test_get_waypoints_returns_names_when_localization_fails(self): + async def handler(request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if request.url.path == "/waypoints": + return httpx.Response(200, json=[{"name": "home1"}]) + if request.url.path == "/localization": + return httpx.Response(503, text="GraphNav unavailable") + return httpx.Response(404) + + await self.client.close() + self.client = robot_client.SpotRobotClient(base_url="http://spot.test") + await self.client._client.aclose() # pylint: disable=protected-access + self.client._client = httpx.AsyncClient( # pylint: disable=protected-access + transport=httpx.MockTransport(handler), base_url="http://spot.test" + ) + + result = await self.client.get_waypoints() + + self.assertEqual([{"name": "home1"}], result["waypoints"]) + self.assertIsNone(result["navigation_ready"]) + self.assertIsNone(result["localized"]) + self.assertIn("503", result["localization_unavailable"]) + self.assertNotIn("error", result) + + async def test_detect_target_is_consumed_once_by_pick(self): + detection = await self.client.detect("red cube") + + self.assertTrue(detection["detected"]) + self.assertEqual("/detect/pick-target", self.requests[-1].url.path) + self.assertEqual( + {"instruction": "red cube"}, json.loads(self.requests[-1].content) + ) + + result = await self.client.pick() + + self.assertTrue(result["ok"]) + self.assertEqual("/manipulation/grasp-pixel", self.requests[-1].url.path) + self.assertEqual( + { + "x": 500, + "y": 400, + "take_lease": True, + "grip_max_torque_nm": 2.0, + }, + json.loads(self.requests[-1].content), + ) + request_count = len(self.requests) + second_result = await self.client.pick() + self.assertFalse(second_result["executed"]) + self.assertEqual(request_count, len(self.requests)) + + async def test_detect_target_is_consumed_once_by_place(self): + await self.client.detect("clear area in the middle of the table") + + result = await self.client.place() + + self.assertTrue(result["ok"]) + self.assertEqual("/manipulation/place-pixel", self.requests[-1].url.path) + self.assertEqual( + {"x": 500, "y": 400, "take_lease": True}, + json.loads(self.requests[-1].content), + ) + request_count = len(self.requests) + second_result = await self.client.place() + self.assertFalse(second_result["executed"]) + self.assertEqual(request_count, len(self.requests)) + + async def test_wait_for_pick_up_calls_renamed_endpoint(self): + result = await self.client.wait_for_pick_up() + + self.assertTrue(result["ok"]) + self.assertEqual("/pickup/wait", self.requests[-1].url.path) + self.assertEqual( + { + "monitor_sec": 30.0, + "upward_threshold_m": 0.02, + "sample_interval": 0.1, + "open_duration_sec": 3.0, + "take_lease": True, + "gripper_timeout": 5.0, + "stow_timeout": 10.0, + }, + json.loads(self.requests[-1].content), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/agent/embodiment/spot/openapi_tools.py b/live-api/agent/embodiment/spot/openapi_tools.py new file mode 100644 index 0000000..be81b76 --- /dev/null +++ b/live-api/agent/embodiment/spot/openapi_tools.py @@ -0,0 +1,438 @@ +"""Convert the Spot backend OpenAPI contract into Gemini tool declarations.""" + +from __future__ import annotations + +import dataclasses +import json +from typing import Any + + +@dataclasses.dataclass(frozen=True) +class ToolPolicy: + name: str + description: str + behavior: str | None = None + + +@dataclasses.dataclass(frozen=True) +class OpenApiOperation: + name: str + method: str + path: str + path_parameters: tuple[str, ...] + query_parameters: tuple[str, ...] + has_json_body: bool + + +# Authentication, map-file loading, registry mutation, lease release, HTML, +# visualization, and raw image operations are intentionally not model tools. +TOOL_POLICIES: dict[tuple[str, str], ToolPolicy] = { + ("get", "/health"): ToolPolicy( + "get_robot_status", + "Check whether the Spot backend is connected and holding the lease. Also returns arm oscillation-monitor state. Call this before motion when control state is uncertain.", + ), + ("get", "/lease"): ToolPolicy( + "get_lease_status", + "Check whether this API server currently controls Spot through a lease. Motion tools require a lease.", + ), + ("post", "/lease/take"): ToolPolicy( + "take_lease", + "Take Spot's lease for this API server and keep it alive. This may override another controller; call only when robot control is intended.", + ), + ("get", "/localization"): ToolPolicy( + "get_localization", + "Get the current GraphNav localization, including the localized waypoint and seed transform. Use this to verify localization before navigation.", + ), + ("post", "/localize"): ToolPolicy( + "localize", + "Initialize or refresh GraphNav localization. Prefer waypoint_name for a known named location; otherwise use fiducials. Verify the result with get_localization before navigating.", + "BLOCKING", + ), + ("get", "/battery"): ToolPolicy( + "get_battery", + "Get battery percentage, estimated remaining runtime, and motor power state. Check before long navigation or manipulation tasks.", + ), + ("post", "/faults/behavior/clear"): ToolPolicy( + "clear_behavior_faults", + "Attempt to clear all clearable behavior faults. Inspect the returned cleared and failed fault IDs before retrying motion.", + ), + ("get", "/waypoints"): ToolPolicy( + "get_waypoints", + "List the currently loaded GraphNav waypoints and their human-readable names. Always call this before navigate and use a returned name exactly.", + ), + ("post", "/navigate"): ToolPolicy( + "navigate", + "Navigate to an exact name returned by get_waypoints. The call waits for arrival or failure. Stow the arm first and ensure localization, lease, motor power, and standing state.", + "BLOCKING", + ), + ("post", "/stand"): ToolPolicy( + "stand", + "Power motors when requested and command Spot to stand. Use before driving, navigation, or arm manipulation when Spot may be sitting.", + "BLOCKING", + ), + ("post", "/sit"): ToolPolicy( + "sit", + "Command Spot to sit while leaving the API connection intact. Call only when the user's current instruction explicitly asks Spot to sit; never sit automatically after a task or while idle.", + "BLOCKING", + ), + ("post", "/teleop/velocity"): ToolPolicy( + "drive", + "Drive Spot for a short duration using body-frame velocities: +x forward, +y left, and +rotation counterclockwise. Set body_follow_arm true to hold the current camera-arm joint pose relative to the moving body, preserving the aimed view during search. Prefer navigate for named destinations and use small commands near obstacles.", + ), + ("post", "/actions/stop"): ToolPolicy( + "stop", + "Immediately cancel ongoing navigation and base motion, stop the robot command stream, and optionally freeze the arm. Use whenever motion is unsafe or an action must be aborted.", + ), + ("get", "/images/sources"): ToolPolicy( + "get_camera_sources", + "List available Spot image sources with image type, dimensions, and depth scale. Use this to inspect camera capabilities and registered depth sources.", + ), + ("post", "/arm/deploy"): ToolPolicy( + "deploy_arm", + "Move the arm from stow into the ready pose for viewing or manipulation. Ensure the surrounding arm workspace is clear.", + "BLOCKING", + ), + ("post", "/arm/carry"): ToolPolicy( + "carry_arm", + "Move the arm to the carry pose. Use to present a held object or transition before delivery; use stow_arm before navigation.", + "BLOCKING", + ), + ("post", "/arm/freeze"): ToolPolicy( + "freeze_arm", + "Stop current arm motion and command the gripper to hold its present Cartesian pose. Use to halt oscillation or an unsafe arm trajectory.", + ), + ("get", "/arm/oscillation-monitor"): ToolPolicy( + "get_oscillation_monitor", + "Return whether automatic arm oscillation monitoring is enabled, its thresholds, sample count, and latest detection, freeze, or error.", + ), + ("post", "/arm/oscillation-monitor"): ToolPolicy( + "configure_oscillation_monitor", + "Enable or disable automatic oscillation monitoring. When enabled, repeated Cartesian direction changes above the configured thresholds cause freeze_arm to be called.", + ), + ("post", "/arm/jog"): ToolPolicy( + "jog_arm", + "Move the gripper relative to its current hand pose. Translation is expressed in hand-frame meters and rotation in radians; rotations are composed roll, then pitch, then yaw. This directly moves the arm, so use deliberate increments.", + ), + ("post", "/arm/camera-roll"): ToolPolicy( + "roll_gripper_camera", + "Roll the gripper camera around its optical viewing axis by a requested angle. Use this instead of jog_arm when the intent is specifically to rotate the camera image clockwise or counterclockwise.", + ), + ("post", "/arm/reach-distance"): ToolPolicy( + "get_arm_reach_distance", + "Measure the straight-line distance from the gripper to a target 3D pose and report whether it exceeds the arm-only reach threshold. This does not move the robot.", + ), + ("post", "/arm/approach"): ToolPolicy( + "approach_pose", + "Move only the arm toward a supplied target pose, stopping at the requested standoff. Call get_arm_reach_distance first when reachability is uncertain.", + "BLOCKING", + ), + ("post", "/arm/approach-whole-body"): ToolPolicy( + "approach_pose_whole_body", + "Coordinate the base and arm to approach a target pose when arm-only reach is insufficient. The base moves toward the target vector, then the arm completes the approach; keep the path and workspace clear.", + "BLOCKING", + ), + ("post", "/arm/stow"): ToolPolicy( + "stow_arm", + "Move the arm into its compact stowed pose. Always call before navigation unless carrying or presenting an object explicitly requires another pose.", + "BLOCKING", + ), + ("post", "/pickup/wait"): ToolPolicy( + "wait_for_pick_up", + "Monitor a held item for upward gripper motion indicating recipient pickup. On detection, open the gripper, wait, close it, and stow the arm. Call from carry pose after reaching the recipient.", + "BLOCKING", + ), + ("post", "/gripper/open"): ToolPolicy( + "open_gripper", + "Open the gripper to a fraction from 0 fully closed to 1 fully open. For drink pickup, about 0.6 leaves a narrower opening than the default full-open pose.", + ), + ("post", "/gripper/close"): ToolPolicy( + "close_gripper", + "Close the gripper toward a requested fraction, where 0 is fully closed and 1 is fully open. Use low max_vel and max_acc for a gentle grasp.", + ), + ("post", "/pick"): ToolPolicy( + "pick", + "Capture aligned hand-camera RGB and depth, use Gemini Robotics ER to identify the requested object pixel, then submit that pixel and camera calibration to Spot's native PickObjectInImage manipulation service. Spot plans the body, arm, and gripper motion. Backend completion is not proof of grasp success: inspect the fresh post-action visual input and claim success only when the requested object is visibly secured by the gripper and moved from its original location.", + "BLOCKING", + ), + ("post", "/detect/pick-target"): ToolPolicy( + "detect", + "Run language-conditioned detection on the hand camera and return only the label, confidence, image dimensions, and normalized/pixel grasp target needed for a subsequent pick. This endpoint does not return image data, point clouds, raw model output, or a 3D scene.", + "BLOCKING", + ), + ("post", "/force/change"): ToolPolicy( + "detect_force_change", + "Sample gripper force and report when force magnitude changes from its initial baseline by at least the threshold. Use while holding or presenting an object; this does not release the gripper automatically.", + "BLOCKING", + ), +} + + +COMMON_PARAMETER_DESCRIPTIONS = { + "take_lease": "If true, take Spot's lease when this server does not already hold it.", + "timeout": "Maximum seconds to wait for the command to complete.", + "power_on": "If true, power on Spot's motors when needed.", + "seconds": "Commanded arm motion duration in seconds.", + "api_key": "Optional Gemini API-key override. Omit to use the server-configured key.", + "model": "Gemini Robotics ER model name. Omit to use the server default.", + "instruction": "Concise visual target description identifying one object, for example 'middle of red drink can'.", + "frame_name": "Coordinate frame for the pose. Arm approach tools require 'vision'.", + "x": "Target x coordinate in meters in frame_name.", + "y": "Target y coordinate in meters in frame_name.", + "z": "Target z coordinate in meters in frame_name.", + "qw": "Quaternion scalar component for target gripper orientation.", + "qx": "Quaternion x component for target gripper orientation.", + "qy": "Quaternion y component for target gripper orientation.", + "qz": "Quaternion z component for target gripper orientation.", +} + + +TOOL_PARAMETER_DESCRIPTIONS: dict[str, dict[str, str]] = { + "localize": { + "waypoint_id": "GraphNav waypoint ID to use as the localization seed. Prefer waypoint_name when a human-readable name is known.", + "waypoint_name": "Exact named waypoint to use as the localization seed, such as 'snack1b'.", + "fiducial_init": "Fiducial initialization mode: 'nearest', 'nearest_at_target', or 'specific'.", + "use_fiducial_id": "Specific AprilTag fiducial ID when fiducial_init is 'specific'.", + "refine_fiducial_result_with_icp": "Refine fiducial localization against map point clouds using ICP.", + "do_ambiguity_check": "Reject ambiguous localization candidates instead of selecting one automatically.", + "refine_with_visual_features": "Refine localization using visual features after the initial estimate.", + "verify_visual_features_quality": "Require sufficient visual-feature quality when visual refinement is enabled.", + "max_distance": "Optional maximum translation in meters allowed during localization refinement.", + "max_yaw": "Optional maximum yaw difference in radians allowed during localization refinement.", + }, + "navigate": { + "name": "Exact destination name returned by get_waypoints; never invent or normalize a name.", + "command_duration": "Seconds assigned to each GraphNav navigation command before it is refreshed.", + "timeout": "Overall maximum navigation time in seconds.", + "feedback_interval": "Seconds between GraphNav feedback checks.", + "stand": "If true, command Spot to stand before navigation.", + }, + "drive": { + "v_x": "Forward body velocity in meters/second; positive moves forward and negative backward.", + "v_y": "Lateral body velocity in meters/second; positive moves left and negative right.", + "v_rot": "Yaw velocity in radians/second; positive rotates counterclockwise.", + "duration": "How long to apply the velocity command, in seconds.", + "stand": "If true, stand Spot before applying velocity.", + "body_follow_arm": "If true, hold the current arm joint pose relative to the body so the gripper camera moves with Spot.", + }, + "stop": { + "freeze_arm": "If true, also stop and hold the arm at its current Cartesian pose.", + }, + "configure_oscillation_monitor": { + "enabled": "Enable or disable automatic arm oscillation monitoring.", + "sample_interval": "Seconds between gripper-position samples.", + "window_sec": "Rolling analysis-window duration in seconds.", + "min_peak_to_peak_m": "Minimum Cartesian peak-to-peak displacement in meters required to classify oscillation.", + "min_direction_changes": "Minimum direction reversals within the window required to classify oscillation.", + "min_speed_mps": "Ignore direction changes slower than this speed in meters/second.", + "freeze_cooldown_sec": "Minimum seconds between automatic freeze commands.", + }, + "jog_arm": { + "dx": "Relative translation in meters along the hand frame's +x axis.", + "dy": "Relative translation in meters along the hand frame's +y axis.", + "dz": "Relative translation in meters along the hand frame's +z axis.", + "droll": "Relative hand-frame roll in radians. Positive follows the right-hand rule.", + "dpitch": "Relative hand-frame pitch in radians. Positive follows the right-hand rule.", + "dyaw": "Relative hand-frame yaw in radians. Positive follows the right-hand rule.", + }, + "roll_gripper_camera": { + "direction": "Camera-image roll direction: 'clockwise' or 'counterclockwise'.", + "angle_rad": "Total positive rotation magnitude in radians.", + }, + "get_arm_reach_distance": { + "pose": "Target gripper pose to evaluate without moving the robot.", + }, + "approach_pose": { + "pose": "Target gripper pose in the vision frame.", + "standoff_m": "Distance in meters to stop before the target along the approach vector; 0 reaches the target.", + "max_step_m": "Maximum allowed arm-only Cartesian move in meters; larger requests are rejected.", + }, + "approach_pose_whole_body": { + "pose": "Target gripper pose in the vision frame.", + "standoff_m": "Distance in meters to stop before the target along the approach vector; 0 reaches the target.", + "max_step_m": "Arm-only reach threshold in meters; when exceeded, move the base toward the target before moving the arm.", + }, + "wait_for_pick_up": { + "monitor_sec": "Maximum seconds to wait for the recipient to lift the held item.", + "upward_threshold_m": "Upward gripper displacement in meters that indicates recipient pickup.", + "sample_interval": "Seconds between gripper-position samples.", + "open_duration_sec": "Seconds to leave the gripper open after detecting pickup.", + "gripper_timeout": "Maximum seconds for each gripper open or close command.", + "stow_timeout": "Maximum seconds for the final arm-stow command.", + }, + "open_gripper": { + "open_fraction": "Target opening fraction: 0 fully closed, 1 fully open. Use about 0.6 before grasping a drink can.", + "max_vel": "Optional maximum gripper velocity in radians/second; lower values move more gently.", + "max_acc": "Optional maximum gripper acceleration in radians/second squared.", + }, + "close_gripper": { + "open_fraction": "Target opening fraction: 0 fully closed, 1 fully open. Use 0 for a complete close.", + "max_vel": "Optional maximum gripper velocity in radians/second; use a low value for a slow grasp.", + "max_acc": "Optional maximum gripper acceleration in radians/second squared; use a low value for a gentle grasp.", + }, + "pick": { + "timeout": "Overall maximum seconds for detection, whole-body approach, and grasp.", + }, + "detect_force_change": { + "threshold_newtons": "Minimum change from baseline force magnitude, in newtons, required for detection.", + "sample_window_sec": "Maximum monitoring duration in seconds.", + "interval_sec": "Seconds between force samples.", + }, +} + + +def _resolve_ref(document: dict[str, Any], ref: str) -> dict[str, Any]: + if not ref.startswith("#/"): + raise ValueError(f"Unsupported external OpenAPI reference: {ref}") + value: Any = document + for part in ref[2:].split("/"): + value = value[part.replace("~1", "/").replace("~0", "~")] + if not isinstance(value, dict): + raise ValueError(f"OpenAPI reference does not point to a schema: {ref}") + return value + + +def _gemini_schema( + schema: dict[str, Any] | None, + document: dict[str, Any], +) -> dict[str, Any]: + if not schema: + return {"type": "OBJECT", "properties": {}} + if "$ref" in schema: + resolved = dict(_resolve_ref(document, schema["$ref"])) + resolved.update({key: value for key, value in schema.items() if key != "$ref"}) + return _gemini_schema(resolved, document) + + variants = schema.get("anyOf") or schema.get("oneOf") + if variants: + non_null = [item for item in variants if item.get("type") != "null"] + if len(non_null) == 1: + merged = dict(non_null[0]) + if schema.get("description"): + merged["description"] = schema["description"] + return _gemini_schema(merged, document) + + result: dict[str, Any] = {} + schema_type = schema.get("type") + if schema_type: + result["type"] = str(schema_type).upper() + elif "properties" in schema: + result["type"] = "OBJECT" + + description = schema.get("description") or schema.get("title") + if "default" in schema: + default_value = json.dumps(schema["default"], ensure_ascii=True) + description = f"{description or 'Value'}. Default: {default_value}." + if description: + result["description"] = str(description) + if "enum" in schema: + result["enum"] = schema["enum"] + for key in ("minimum", "maximum", "minItems", "maxItems"): + if key in schema: + result[key] = schema[key] + + if result.get("type") == "OBJECT": + result["properties"] = { + name: _gemini_schema(value, document) + for name, value in schema.get("properties", {}).items() + } + if schema.get("required"): + result["required"] = list(schema["required"]) + elif result.get("type") == "ARRAY": + result["items"] = _gemini_schema(schema.get("items", {}), document) + return result + + +def _apply_parameter_descriptions( + tool_name: str, + schema: dict[str, Any], +) -> None: + """Apply operational docs while retaining defaults extracted from OpenAPI.""" + overrides = TOOL_PARAMETER_DESCRIPTIONS.get(tool_name, {}) + for name, property_schema in schema.get("properties", {}).items(): + description = overrides.get(name) or COMMON_PARAMETER_DESCRIPTIONS.get(name) + if description: + existing = property_schema.get("description", "") + default_marker = ". Default: " + default_note = "" + if default_marker in existing: + default_note = default_marker + existing.split(default_marker, 1)[1] + base_description = description.rstrip(".") + property_schema["description"] = ( + base_description + default_note + if default_note + else base_description + "." + ) + if property_schema.get("type") == "OBJECT": + _apply_parameter_descriptions(tool_name, property_schema) + + +def build_openapi_tools( + document: dict[str, Any], +) -> tuple[list[dict[str, Any]], dict[str, OpenApiOperation]]: + """Build allowlisted Gemini declarations and dispatch metadata.""" + declarations: list[dict[str, Any]] = [] + operations: dict[str, OpenApiOperation] = {} + + for (method, path), policy in TOOL_POLICIES.items(): + operation = document.get("paths", {}).get(path, {}).get(method) + if operation is None: + continue + + properties: dict[str, Any] = {} + required: list[str] = [] + path_parameters: list[str] = [] + query_parameters: list[str] = [] + for parameter in operation.get("parameters", []): + parameter = ( + _resolve_ref(document, parameter["$ref"]) + if "$ref" in parameter + else parameter + ) + name = parameter["name"] + properties[name] = _gemini_schema(parameter.get("schema"), document) + if parameter.get("description"): + properties[name]["description"] = parameter["description"] + if parameter.get("required"): + required.append(name) + if parameter.get("in") == "path": + path_parameters.append(name) + elif parameter.get("in") == "query": + query_parameters.append(name) + + content = operation.get("requestBody", {}).get("content", {}) + body_schema = content.get("application/json", {}).get("schema") + has_json_body = body_schema is not None + if body_schema: + converted = _gemini_schema(body_schema, document) + properties.update(converted.get("properties", {})) + required.extend(converted.get("required", [])) + + _apply_parameter_descriptions( + policy.name, + {"type": "OBJECT", "properties": properties}, + ) + + parameters: dict[str, Any] = { + "type": "OBJECT", + "properties": properties, + } + if required: + parameters["required"] = list(dict.fromkeys(required)) + declaration: dict[str, Any] = { + "name": policy.name, + "description": policy.description, + "parameters": parameters, + } + if policy.behavior: + declaration["behavior"] = policy.behavior + declarations.append(declaration) + operations[policy.name] = OpenApiOperation( + name=policy.name, + method=method, + path=path, + path_parameters=tuple(path_parameters), + query_parameters=tuple(query_parameters), + has_json_body=has_json_body, + ) + + return declarations, operations diff --git a/live-api/agent/embodiment/spot/openapi_tools_test.py b/live-api/agent/embodiment/spot/openapi_tools_test.py new file mode 100644 index 0000000..25ba2a9 --- /dev/null +++ b/live-api/agent/embodiment/spot/openapi_tools_test.py @@ -0,0 +1,228 @@ +"""Tests for OpenAPI-driven Spot tools.""" + +from __future__ import annotations + +import unittest + +import httpx + +from embodiment.spot import openapi_tools +from embodiment.spot import robot_client + + +OPENAPI_DOCUMENT = { + "paths": { + "/navigate": { + "post": { + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/NavigateRequest"} + } + }, + } + } + }, + "/battery": {"get": {}}, + "/connect": {"post": {}}, + }, + "components": { + "schemas": { + "NavigateRequest": { + "type": "object", + "properties": { + "name": {"type": "string", "title": "Name"}, + "take_lease": {"type": "boolean", "default": False}, + "max_distance": { + "anyOf": [{"type": "number"}, {"type": "null"}] + }, + }, + "required": ["name"], + } + } + }, +} + + +DETECT_DOCUMENT = { + "paths": { + "/detect": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/DetectRequest"} + } + } + } + } + }, + "/detect/pick-target": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DetectPickTargetRequest" + } + } + } + } + } + }, + "/arm/approach": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ApproachRequest"} + } + } + } + } + }, + }, + "components": { + "schemas": { + "DetectRequest": { + "type": "object", + "properties": { + "instruction": {"type": "string", "title": "Instruction"}, + "color_source": { + "type": "string", + "title": "Color Source", + "default": "hand_color_image", + }, + "depth_source": { + "type": "string", + "title": "Depth Source", + "default": "hand_depth_in_hand_color_frame", + }, + "point_cloud_stride": { + "type": "integer", + "title": "Point Cloud Stride", + "default": 4, + }, + }, + "required": ["instruction"], + }, + "DetectPickTargetRequest": { + "type": "object", + "properties": { + "instruction": {"type": "string", "title": "Instruction"}, + }, + "required": ["instruction"], + }, + "ApproachRequest": { + "type": "object", + "properties": { + "pose": {"$ref": "#/components/schemas/PoseRequest"}, + "standoff_m": { + "type": "number", + "title": "Standoff M", + "default": 0.0, + }, + }, + "required": ["pose"], + }, + "PoseRequest": { + "type": "object", + "properties": { + "frame_name": { + "type": "string", + "title": "Frame Name", + "default": "vision", + }, + "x": {"type": "number", "title": "X"}, + "y": {"type": "number", "title": "Y"}, + "z": {"type": "number", "title": "Z"}, + }, + "required": ["x", "y", "z"], + }, + } + }, +} + + +class OpenApiToolsTest(unittest.TestCase): + + def test_builds_allowlisted_resolved_declarations(self): + declarations, operations = openapi_tools.build_openapi_tools( + OPENAPI_DOCUMENT + ) + by_name = {declaration["name"]: declaration for declaration in declarations} + + self.assertEqual({"navigate", "get_battery"}, set(by_name)) + self.assertNotIn("connect", operations) + navigate = by_name["navigate"] + self.assertEqual(["name"], navigate["parameters"]["required"]) + self.assertEqual( + "NUMBER", navigate["parameters"]["properties"]["max_distance"]["type"] + ) + self.assertEqual("/navigate", operations["navigate"].path) + + def test_exposes_only_lightweight_detect_and_documents_nested_pose_units(self): + declarations, _ = openapi_tools.build_openapi_tools(DETECT_DOCUMENT) + by_name = {declaration["name"]: declaration for declaration in declarations} + + self.assertIn("detect", by_name) + self.assertEqual( + ["instruction"], by_name["detect"]["parameters"]["required"] + ) + self.assertEqual( + {"instruction"}, + set(by_name["detect"]["parameters"]["properties"]), + ) + + pose = by_name["approach_pose"]["parameters"]["properties"]["pose"] + self.assertIn("vision", pose["properties"]["frame_name"]["description"]) + self.assertIn("meters", pose["properties"]["x"]["description"]) + + def test_every_allowlisted_tool_has_substantive_description(self): + for policy in openapi_tools.TOOL_POLICIES.values(): + with self.subTest(tool=policy.name): + self.assertGreaterEqual(len(policy.description), 80) + + +class SpotRobotClientTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self): + self.requests: list[httpx.Request] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if request.url.path == "/openapi.json": + return httpx.Response(200, json=OPENAPI_DOCUMENT) + if request.url.path == "/navigate": + return httpx.Response(200, json={"status": "arrived"}) + return httpx.Response(404) + + self.client = robot_client.SpotRobotClient(base_url="http://spot.test") + await self.client._client.aclose() # pylint: disable=protected-access + self.client._client = httpx.AsyncClient( # pylint: disable=protected-access + base_url="http://spot.test", transport=httpx.MockTransport(handler) + ) + + async def asyncTearDown(self): + await self.client.close() + + async def test_loads_contract_and_dispatches_json_body(self): + declarations = await self.client.load_openapi_tools() + result = await self.client.execute_openapi_action( + "navigate", name="home1", take_lease=True + ) + + self.assertEqual(2, len(declarations)) + self.assertEqual({"status": "arrived"}, result) + request = self.requests[-1] + self.assertEqual("POST", request.method) + self.assertEqual("/navigate", request.url.path) + self.assertEqual( + {"name": "home1", "take_lease": True}, + __import__("json").loads(request.content), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/agent/embodiment/spot/robot_client.py b/live-api/agent/embodiment/spot/robot_client.py new file mode 100644 index 0000000..d650ce4 --- /dev/null +++ b/live-api/agent/embodiment/spot/robot_client.py @@ -0,0 +1,641 @@ +"""Spot robot client with Spot-specific camera and action configuration (Lite).""" + +from __future__ import annotations + +import logging + +import httpx + +from embodiment import robot_client +from embodiment.spot import openapi_tools + +logger = logging.getLogger(__name__) + +# Dedicated timeout for long-running operations (navigate, pick). +_LONG_TIMEOUT = httpx.Timeout(200.0) + + +class SpotRobotClient(robot_client.RobotClient): + """RobotClient configured for the Boston Dynamics Spot platform. + + Spot's FastAPI server (physical-agents) exposes a different image endpoint + than Atari: raw image bytes at GET /images/{source} rather than + HTML-embedded base64. This class overrides ``get_camera_snapshot`` and + adds async helpers for every Spot-specific endpoint. + """ + + CAMERA_IDS = [ + "hand_color_image", + ] + + # Spot does not expose MJPEG streaming endpoints. + ENDPOINT_MAP = {} + + def __init__( + self, + base_url: str = "http://localhost:8888", + timeout: float = 30.0, + ): + super().__init__(base_url=base_url, timeout=timeout) + self.tool_declarations: list[dict] = [] + self.openapi_operations: dict[str, openapi_tools.OpenApiOperation] = {} + self._detected_target: dict | None = None + + async def load_openapi_tools(self) -> list[dict]: + """Load Gemini tools from the backend's live OpenAPI document.""" + response = await self._client.get("/openapi.json") + response.raise_for_status() + declarations, operations = openapi_tools.build_openapi_tools(response.json()) + if not declarations: + raise RuntimeError("Spot OpenAPI document contained no allowlisted operations") + self.tool_declarations = declarations + self.openapi_operations = operations + logger.info("Loaded %d Spot tools from OpenAPI", len(declarations)) + return declarations + + async def execute_openapi_action(self, action_name: str, **kwargs) -> dict: + """Execute an allowlisted operation using its OpenAPI dispatch metadata.""" + operation = self.openapi_operations.get(action_name) + if operation is None: + raise ValueError(f"Unknown Spot OpenAPI action: {action_name}") + + path = operation.path + remaining = dict(kwargs) + for name in operation.path_parameters: + if name not in remaining: + raise ValueError(f"Missing path parameter: {name}") + path = path.replace("{" + name + "}", str(remaining.pop(name))) + + query = { + name: remaining.pop(name) + for name in operation.query_parameters + if name in remaining + } + request_kwargs: dict = {"params": query} + if operation.has_json_body: + request_kwargs["json"] = remaining + timeout = _LONG_TIMEOUT if operation.path in { + "/navigate", + "/pick", + "/pickup/wait", + "/arm/approach", + "/arm/approach-whole-body", + } else self._client.timeout + + try: + response = await self._client.request( + operation.method, path, timeout=timeout, **request_kwargs + ) + response.raise_for_status() + if response.headers.get("content-type", "").startswith("application/json"): + return response.json() + return {"result": response.text} + except httpx.HTTPStatusError as exc: + try: + detail = exc.response.json() + except ValueError: + detail = exc.response.text + return {"error": f"HTTP {exc.response.status_code}", "detail": detail} + except Exception as exc: # pylint: disable=broad-except + logger.error("OpenAPI action %s failed: %s", action_name, exc) + return {"error": str(exc)} + + # ---- Camera snapshot (override) ---- + + async def get_camera_snapshot(self, camera_id: str) -> bytes | None: + """Fetch a single image from the Spot FastAPI server. + + Unlike the base class (which parses HTML-embedded base64), Spot's + ``GET /images/{source}`` returns raw image bytes directly. + """ + try: + resp = await self._client.get( + f"/images/{camera_id}", + timeout=self._camera_timeout, + ) + resp.raise_for_status() + return resp.content + except Exception as e: # pylint: disable=broad-except + logger.warning("camera snapshot %s error: %s", camera_id, e) + return None + + # ---- Navigation ---- + + async def navigate( + self, + waypoint: str, + timeout: float = 180, + power_on: bool = True, + stand: bool = True, + take_lease: bool = True, + ) -> dict: + """Navigate to a recorded waypoint by name.""" + try: + resp = await self._client.post( + "/navigate", + json={ + "name": waypoint, + "timeout": timeout, + "power_on": power_on, + "stand": stand, + "take_lease": take_lease, + }, + timeout=_LONG_TIMEOUT, + ) + resp.raise_for_status() + return resp.json() + except httpx.HTTPStatusError as e: + try: + detail = e.response.json().get("detail", e.response.text) + except ValueError: + detail = e.response.text + logger.error("navigate rejected: %s", detail) + return {"error": f"HTTP {e.response.status_code}", "detail": detail} + except Exception as e: # pylint: disable=broad-except + logger.error("navigate error: %s", e) + return {"error": str(e)} + + async def drive( + self, + v_x: float, + v_y: float, + v_rot: float, + duration: float, + ) -> dict: + """Drive with short body-frame velocity commands without GraphNav.""" + try: + resp = await self._client.post( + "/teleop/velocity", + json={ + "v_x": v_x, + "v_y": v_y, + "v_rot": v_rot, + "duration": duration, + "take_lease": True, + "power_on": True, + "stand": True, + "body_follow_arm": True, + }, + timeout=max(float(duration), 0.1) + 15.0, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("drive error: %s", e) + return {"error": str(e)} + + async def stop(self) -> dict: + """Cancel base, navigation, and arm actions immediately.""" + try: + resp = await self._client.post( + "/actions/stop", + json={"take_lease": True, "freeze_arm": True}, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("stop error: %s", e) + return {"error": str(e)} + + async def look( + self, + direction: str, + angle_rad: float = 0.15, + ) -> dict: + """Aim the gripper camera with arm rotation while keeping the base still.""" + normalized_direction = direction.strip().lower() + if normalized_direction not in {"up", "down", "left", "right"}: + raise ValueError("direction must be one of: up, down, left, right") + + angle = min(0.35, max(0.05, float(angle_rad))) + rotation = { + "up": {"dpitch": -angle}, + "down": {"dpitch": angle}, + "left": {"dyaw": angle}, + "right": {"dyaw": -angle}, + }[normalized_direction] + try: + resp = await self._client.post( + "/arm/jog", + json={ + **rotation, + "seconds": 0.8, + "take_lease": True, + "timeout": 3.0, + }, + ) + resp.raise_for_status() + result = resp.json() + result["direction"] = normalized_direction + return result + except Exception as e: # pylint: disable=broad-except + logger.error("look error: %s", e) + return {"error": str(e)} + + async def stand( + self, + power_on: bool = True, + take_lease: bool = True, + timeout: float = 15, + ) -> dict: + """Command the robot to stand.""" + try: + resp = await self._client.post( + "/stand", + json={ + "power_on": power_on, + "take_lease": take_lease, + "timeout": timeout, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("stand error: %s", e) + return {"error": str(e)} + + async def sit( + self, + take_lease: bool = True, + ) -> dict: + """Command the robot to sit.""" + try: + resp = await self._client.post( + "/sit", + json={ + "take_lease": take_lease, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("sit error: %s", e) + return {"error": str(e)} + + # ---- Arm ---- + + async def deploy_arm( + self, + take_lease: bool = True, + ) -> dict: + """Deploy the robot arm.""" + try: + resp = await self._client.post( + "/arm/deploy", + json={ + "take_lease": take_lease, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("deploy_arm error: %s", e) + return {"error": str(e)} + + async def stow_arm( + self, + take_lease: bool = True, + ) -> dict: + """Stow the robot arm.""" + try: + resp = await self._client.post( + "/arm/stow", + json={ + "take_lease": take_lease, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("stow_arm error: %s", e) + return {"error": str(e)} + + async def carry_arm( + self, + take_lease: bool = True, + ) -> dict: + """Move arm to carry pose.""" + try: + resp = await self._client.post( + "/arm/carry", + json={ + "take_lease": take_lease, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("carry_arm error: %s", e) + return {"error": str(e)} + + # ---- Gripper ---- + + async def open_gripper( + self, + fraction: float = 1.0, + take_lease: bool = True, + ) -> dict: + """Open the gripper.""" + try: + resp = await self._client.post( + "/gripper/open", + json={ + "fraction": fraction, + "take_lease": take_lease, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("open_gripper error: %s", e) + return {"error": str(e)} + + async def close_gripper( + self, + take_lease: bool = True, + ) -> dict: + """Close the gripper.""" + try: + resp = await self._client.post( + "/gripper/close", + json={ + "take_lease": take_lease, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("close_gripper error: %s", e) + return {"error": str(e)} + + # ---- Manipulation API ---- + + def clear_detected_target(self) -> None: + """Invalidate a detection after any action that can move the camera.""" + self._detected_target = None + + async def detect(self, instruction: str) -> dict: + """Detect one language-specified object and cache its grasp target.""" + self.clear_detected_target() + try: + resp = await self._client.post( + "/detect/pick-target", + json={ + "instruction": instruction, + }, + timeout=_LONG_TIMEOUT, + ) + resp.raise_for_status() + detection = resp.json() + target = detection.get("target") + if not isinstance(target, dict): + raise ValueError("Detection response did not include a target") + self._detected_target = target + return { + "detected": True, + "instruction": instruction, + "label": detection.get("label", ""), + "confidence": detection.get("confidence"), + "target": target, + "next_action": ( + "Call pick or place with no arguments before moving the robot" + " or camera." + ), + } + except Exception as e: # pylint: disable=broad-except + self.clear_detected_target() + logger.error("detect error: %s", e) + return {"error": str(e), "detected": False} + + async def pick( + self, + take_lease: bool = True, + timeout: float = 120, + ) -> dict: + """Grasp the one-time target produced by the latest detect call.""" + target = self._detected_target + self.clear_detected_target() + if target is None: + return { + "error": "No detected target is available. Call detect first.", + "executed": False, + } + try: + resp = await self._client.post( + "/manipulation/grasp-pixel", + json={ + "x": target["normalized_x"], + "y": target["normalized_y"], + "take_lease": take_lease, + "grip_max_torque_nm": 2.0, + }, + timeout=_LONG_TIMEOUT, + ) + resp.raise_for_status() + result = resp.json() + result["detected_target"] = target + return result + except Exception as e: # pylint: disable=broad-except + logger.error("pick error: %s", e) + return {"error": str(e)} + + async def place( + self, + take_lease: bool = True, + timeout: float = 120, + ) -> dict: + """Place an object at the one-time target produced by detect.""" + target = self._detected_target + self.clear_detected_target() + if target is None: + return { + "error": "No detected target is available. Call detect first.", + "executed": False, + } + try: + resp = await self._client.post( + "/manipulation/place-pixel", + json={ + "x": target["normalized_x"], + "y": target["normalized_y"], + "take_lease": take_lease, + }, + timeout=_LONG_TIMEOUT, + ) + resp.raise_for_status() + result = resp.json() + result["detected_target"] = target + return result + except Exception as e: # pylint: disable=broad-except + logger.error("place error: %s", e) + return {"error": str(e)} + + async def wait_for_pick_up( + self, + monitor_sec: float = 30.0, + upward_threshold_m: float = 0.02, + sample_interval: float = 0.1, + open_duration_sec: float = 3.0, + take_lease: bool = True, + gripper_timeout: float = 5.0, + stow_timeout: float = 10.0, + ) -> dict: + """Wait for a recipient to lift the held item, then release and stow.""" + try: + resp = await self._client.post( + "/pickup/wait", + json={ + "monitor_sec": monitor_sec, + "upward_threshold_m": upward_threshold_m, + "sample_interval": sample_interval, + "open_duration_sec": open_duration_sec, + "take_lease": take_lease, + "gripper_timeout": gripper_timeout, + "stow_timeout": stow_timeout, + }, + timeout=_LONG_TIMEOUT, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("wait_for_pick_up error: %s", e) + return {"error": str(e)} + + # ---- Waypoints ---- + + async def get_waypoints(self) -> dict: + """Get registered waypoints and current GraphNav readiness.""" + try: + waypoints_response = await self._client.get("/waypoints") + waypoints_response.raise_for_status() + except Exception as e: # pylint: disable=broad-except + logger.error("get_waypoints error: %s", e) + return {"error": str(e)} + + result = { + "waypoints": waypoints_response.json(), + "navigation_ready": None, + "localized": None, + } + try: + localization_response = await self._client.get("/localization") + localization_response.raise_for_status() + localization = localization_response.json() + navigation_ready = bool(localization.get("localized")) + result.update({ + "navigation_ready": navigation_ready, + "localized": navigation_ready, + "localization": localization.get("localization", {}), + }) + if not navigation_ready: + result["warning"] = ( + "Waypoint names may be stale registry entries. Do not call" + " navigate until a GraphNav map is loaded and localization is" + " established." + ) + return result + except Exception as e: # pylint: disable=broad-except + logger.warning("get_waypoints localization unavailable: %s", e) + result["localization_unavailable"] = str(e) + result["warning"] = ( + "Waypoint names are available, but navigation readiness could not" + " be determined. Do not call navigate until localization is" + " verified." + ) + return result + + # ---- Status ---- + + async def get_battery(self) -> dict: + """Get the current battery status.""" + try: + resp = await self._client.get("/battery") + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("get_battery error: %s", e) + return {"error": str(e)} + + async def stop_actions( + self, + take_lease: bool = True, + freeze_arm: bool = True, + ) -> dict: + """Stop all current robot actions.""" + try: + resp = await self._client.post( + "/actions/stop", + json={ + "take_lease": take_lease, + "freeze_arm": freeze_arm, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("stop_actions error: %s", e) + return {"error": str(e)} + + async def stow( + self, + take_lease: bool = True, + timeout: float = 20.0, + ) -> dict: + """Stow arm safely, choosing carry pose if holding an object.""" + try: + resp = await self._client.post( + "/arm/stow-smart", + json={ + "take_lease": take_lease, + "timeout": timeout, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("stow error: %s", e) + return {"error": str(e)} + + async def health_check(self) -> dict: + """Check robot health status and battery.""" + try: + health_resp = await self._client.get("/health") + health_resp.raise_for_status() + health_data = health_resp.json() + + battery_data = {} + if health_data.get("connected"): + battery_resp = await self._client.get("/battery") + if battery_resp.status_code == 200: + battery_data = battery_resp.json() + + return { + "status": "SUCCESS", + "health": health_data, + "battery": battery_data, + } + except Exception as e: # pylint: disable=broad-except + logger.error("health_check error: %s", e) + return {"error": str(e)} + + async def connect( + self, + hostname: str | None = None, + username: str | None = None, + password: str | None = None, + ) -> dict: + """Connect (or reconnect) to the Spot robot.""" + body: dict = {} + if hostname is not None: + body["hostname"] = hostname + if username is not None: + body["username"] = username + if password is not None: + body["password"] = password + try: + resp = await self._client.post("/connect", json=body) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("connect error: %s", e) + return {"error": str(e)} diff --git a/live-api/agent/embodiment/spot/spot_embodiment.py b/live-api/agent/embodiment/spot/spot_embodiment.py new file mode 100644 index 0000000..2e0ad1a --- /dev/null +++ b/live-api/agent/embodiment/spot/spot_embodiment.py @@ -0,0 +1,99 @@ +"""Spot embodiment for Boston Dynamics Spot robot control (Lite version).""" + +import asyncio +import logging + +import camera_poller +from embodiment import base +from embodiment.spot import robot_client +from tool import tools as tools_lib + +logger = logging.getLogger(__name__) + + +class SpotEmbodiment(base.Embodiment): + """Embodiment for a physical Boston Dynamics Spot robot.""" + + def __init__( + self, + robot_url: str, + poll_hz: float = 2.0, + push_hz: float = 1.0, + ): + self.audio_queue = asyncio.Queue() + self.video_queue = asyncio.Queue() + self.text_queue = asyncio.Queue() + self.robot = robot_client.SpotRobotClient(base_url=robot_url) + self.poller = camera_poller.CameraPoller( + robot_client=self.robot, + video_input_queue=self.video_queue, + camera_ids=["hand_color_image"], + poll_hz=poll_hz, + push_hz=push_hz, + ) + self.poller_task = asyncio.create_task(self.poller.run()) + logger.info("SpotEmbodiment initialized with URL: %s", robot_url) + + def get_audio_queue(self) -> asyncio.Queue: + return self.audio_queue + + def get_video_queue(self) -> asyncio.Queue: + return self.video_queue + + def get_text_queue(self) -> asyncio.Queue: + return self.text_queue + + def get_tools(self) -> list[dict]: + return tools_lib.spot_tools() + + async def initialize(self) -> None: + """Initialize SpotEmbodiment.""" + pass + + def get_system_instruction(self) -> str: + # Instructions are loaded via Agent DI in Lite version. + return "" + + async def execute_action(self, action_name: str, **kwargs): + logger.info( + "SpotEmbodiment executing action: %s with args: %s", + action_name, + kwargs, + ) + if action_name == "ack": + return "No action needed." + + if action_name not in { + "detect", + "pick", + "place", + "health_check", + "get_waypoints", + }: + self.robot.clear_detected_target() + + # Legacy fallback when the backend contract is unavailable. + fallback_actions = { + "health_check": self.robot.health_check, + "get_waypoints": self.robot.get_waypoints, + "navigate": self.robot.navigate, + "drive": self.robot.drive, + "stop": self.robot.stop, + "look": self.robot.look, + "detect": self.robot.detect, + "pick": self.robot.pick, + "place": self.robot.place, + "wait_for_pick_up": self.robot.wait_for_pick_up, + "stand": self.robot.stand, + "sit": self.robot.sit, + "stow": self.robot.stow, + } + action = fallback_actions.get(action_name) + if action is None: + raise ValueError(f"Unknown action: {action_name}") + return await action(**kwargs) + + async def close(self): + self.poller.stop() + await self.poller_task + await self.robot.close() diff --git a/live-api/agent/embodiment/tinybot/__init__.py b/live-api/agent/embodiment/tinybot/__init__.py new file mode 100644 index 0000000..8a155cb --- /dev/null +++ b/live-api/agent/embodiment/tinybot/__init__.py @@ -0,0 +1 @@ +# Empty init for tinybot embodiment package diff --git a/live-api/agent/embodiment/tinybot/robot_client.py b/live-api/agent/embodiment/tinybot/robot_client.py new file mode 100644 index 0000000..9e91719 --- /dev/null +++ b/live-api/agent/embodiment/tinybot/robot_client.py @@ -0,0 +1,131 @@ +"""Tinybot robot client (Lite version).""" + +import logging +from typing import Any + +from embodiment import robot_client + +logger = logging.getLogger(__name__) + + +class TinybotRobotClient(robot_client.RobotClient): + """RobotClient configured for the Tinybot platform.""" + + CAMERA_IDS = ["camera"] + + # Map camera ID to the video-stream endpoint + ENDPOINT_MAP = { + "camera": "/video-stream", + } + + def __init__( + self, + base_url: str = "http://localhost:8000", + timeout: float = 30.0, + ): + super().__init__(base_url=base_url, timeout=timeout) + + async def make_gesture(self, gesture: str, speed: int = 150) -> str: + """Execute a gesture on the tinybot.""" + try: + resp = await self._client.post( + "/make-gesture", + params={"gesture": gesture, "speed": speed}, + ) + resp.raise_for_status() + return resp.json().get("status", "ok") + except Exception as e: # pylint: disable=broad-except + logger.error("make_gesture error: %s", e) + return f"error: {e}" + + async def move_absolute( + self, angles: list[float], speeds: list[int], blocking: bool = True + ) -> dict[str, Any]: + """Move tinybot joints to absolute angles.""" + try: + resp = await self._client.post( + "/move-absolute", + json={ + "angles": angles, + "speeds": speeds, + "blocking": blocking, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("move_absolute error: %s", e) + return {"error": str(e)} + + async def move_relative( + self, + relative_angles: list[float], + speeds: list[int], + blocking: bool = True, + ) -> dict[str, Any]: + """Move tinybot joints by relative angles.""" + try: + resp = await self._client.post( + "/move-relative", + json={ + "relative_angles": relative_angles, + "speeds": speeds, + "blocking": blocking, + }, + ) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("move_relative error: %s", e) + return {"error": str(e)} + + async def turn_head_to_uv(self, u: float, v: float) -> str: + """Turn tinybot head to look at normalized pixel coordinates (0.0-1.0).""" + # Scale from 0.0-1.0 to 0-1000 + u_int = int(u * 1000) + v_int = int(v * 1000) + try: + resp = await self._client.post( + "/look-to-uv", + json={ + "u": u_int, + "v": v_int, + "speed": 150, + "blocking": True, + }, + ) + resp.raise_for_status() + return resp.json().get("status", "ok") + except Exception as e: # pylint: disable=broad-except + logger.error("turn_head_to_uv error: %s", e) + return f"error: {e}" + + async def get_health(self) -> dict[str, Any]: + """Check tinybot health status.""" + try: + resp = await self._client.get("/health") + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("health error: %s", e) + return {"error": str(e)} + + async def point(self, points: list[dict[str, Any]]) -> dict[str, Any]: + """Draw points on the tinybot camera stream overlay.""" + try: + resp = await self._client.post("/point", json=points) + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("point error: %s", e) + return {"error": str(e)} + + async def clear_overlay(self) -> dict[str, Any]: + """Clear point overlays from tinybot camera stream.""" + try: + resp = await self._client.post("/clear-overlay") + resp.raise_for_status() + return resp.json() + except Exception as e: # pylint: disable=broad-except + logger.error("clear_overlay error: %s", e) + return {"error": str(e)} diff --git a/live-api/agent/embodiment/tinybot/tinybot_embodiment.py b/live-api/agent/embodiment/tinybot/tinybot_embodiment.py new file mode 100644 index 0000000..24d589a --- /dev/null +++ b/live-api/agent/embodiment/tinybot/tinybot_embodiment.py @@ -0,0 +1,97 @@ +"""Tinybot embodiment for Eden Lite agent.""" + +import asyncio +import logging + +import camera_poller +from embodiment import base +from embodiment.tinybot import robot_client +from tool import tools as tools_lib + +logger = logging.getLogger(__name__) + + +class TinybotEmbodiment(base.Embodiment): + """Embodiment for a physical Tinybot robot.""" + + def __init__( + self, + robot_url: str, + poll_hz: float = 2.0, + push_hz: float = 1.0, + ): + self.audio_queue = asyncio.Queue() + self.video_queue = asyncio.Queue() + self.text_queue = asyncio.Queue() + self.robot = robot_client.TinybotRobotClient(base_url=robot_url) + self.poller = camera_poller.CameraPoller( + robot_client=self.robot, + video_input_queue=self.video_queue, + camera_ids=["camera"], + poll_hz=poll_hz, + push_hz=push_hz, + ) + self.poller_task = asyncio.create_task(self.poller.run()) + logger.info("TinybotEmbodiment initialized with URL: %s", robot_url) + + def get_audio_queue(self) -> asyncio.Queue: + return self.audio_queue + + def get_video_queue(self) -> asyncio.Queue: + return self.video_queue + + def get_text_queue(self) -> asyncio.Queue: + return self.text_queue + + def get_tools(self) -> list[dict]: + return tools_lib.tinybot_tools() + + def get_system_instruction(self) -> str: + return "" + + async def execute_action(self, action_name: str, **kwargs): + logger.info( + "TinybotEmbodiment executing action: %s with args: %s", + action_name, + kwargs, + ) + if action_name == "make_gesture": + return await self.robot.make_gesture(**kwargs) + elif action_name == "move_absolute": + return await self.robot.move_absolute(**kwargs) + elif action_name == "move_relative": + return await self.robot.move_relative(**kwargs) + elif action_name == "turn_head_to_uv": + return await self.robot.turn_head_to_uv(**kwargs) + + elif action_name == "draw_points": + # Map 'x', 'y' to 'u', 'v' for Tinybot API + raw_points = kwargs.get("points", []) + mapped_points = [] + for p in raw_points: + mapped_points.append({ + "u": int(p.get("x", 0)), + "v": int(p.get("y", 0)), + "label": p.get("label", ""), + }) + return await self.robot.point(points=mapped_points) + elif action_name == "clear_overlay": + return await self.robot.clear_overlay() + elif action_name == "stop": + return await self.robot.stop() + elif action_name == "ack": + return {"status": "ok"} + elif action_name == "send_message": + target = kwargs.get("target", "unknown") + message = kwargs.get("message", "") + logger.info( + "[TinybotEmbodiment] send_message to %s: %s", target, message + ) + return f"Message sent to {target}" + else: + raise ValueError(f"Unknown action: {action_name}") + + async def close(self): + self.poller.stop() + await self.poller_task + await self.robot.close() diff --git a/live-api/agent/model/__init__.py b/live-api/agent/model/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/live-api/agent/model/__init__.py @@ -0,0 +1 @@ + diff --git a/live-api/agent/model/live_api_client.py b/live-api/agent/model/live_api_client.py new file mode 100644 index 0000000..f607f5c --- /dev/null +++ b/live-api/agent/model/live_api_client.py @@ -0,0 +1,178 @@ +"""Gemini Live API client for bidirectional streaming via WebSockets. + +This module provides a client that connects to the public Gemini Live API +(BidiGenerateContent) over WebSockets, enabling real-time voice and vision +interactions without requiring internal Google infrastructure (pywraprpc/LOAS). + +All messages are sent and received as plain Python dicts (JSON), with no +protobuf dependency. + +Usage: + client = GeminiLiveApiClient(api_key="YOUR_API_KEY") + stream = client.create_stream() + +WebSocket endpoint: + wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta + .GenerativeService.BidiGenerateContent?key=API_KEY +""" + +import logging +import threading +from typing import Any, Callable + +import websocket + +logger = logging.getLogger(__name__) + +_LIVE_API_WSS_URL = ( + "wss://generativelanguage.googleapis.com/ws/" + "google.ai.generativelanguage.v1alpha" + ".GenerativeService.BidiGenerateContent" +) + + +# pylint: disable=invalid-name + + +class GeminiLiveApiStream: + """Wrapper around a WebSocket connection to the Gemini Live API. + + Provides the streaming interface (Start, Send, HalfClose, GetStatus) so + that the SessionManager can use it transparently. + """ + + def __init__(self, ws): + self._ws = ws + self._on_message = None + self._on_done = None + self._read_thread = None + + def Start( + self, on_message: Callable[[dict], None], + on_done: Callable[[], None], + ) -> None: + """Start reading messages from the WebSocket in a background thread.""" + logger.debug("GeminiLiveApiStream.Start() called") + self._on_message = on_message + self._on_done = on_done + self._read_thread = threading.Thread(target=self._read_loop, daemon=True) + self._read_thread.start() + logger.debug("GeminiLiveApiStream read thread spawned and started") + + def _read_loop(self) -> None: + """Reads messages from the WebSocket and dispatches to on_message.""" + logger.debug("GeminiLiveApiStream._read_loop() thread started") + try: + while True: + opcode, data = self._ws.recv_data() + if opcode == websocket.ABNF.OPCODE_CLOSE: + import struct # pylint: disable=g-import-not-at-top + code = 1000 + reason = "" + if len(data) >= 2: + code = struct.unpack("!H", data[0:2])[0] + reason = data[2:].decode("utf-8", errors="replace") + logger.info( + "WebSocket closed by server. code=%s, reason=%s", code, reason + ) + break + + if opcode not in ( + websocket.ABNF.OPCODE_TEXT, + websocket.ABNF.OPCODE_BINARY, + ): + continue + + raw = data + + # If it is bytes, it might still be a UTF-8 JSON string. + if isinstance(raw, bytes): + try: + raw = raw.decode("utf-8") + except UnicodeDecodeError: + logger.warning("Received binary data but expected JSON") + + if isinstance(raw, str): + import json + try: + parsed_msg = json.loads(raw) + logger.info("Received message from Gemini Live API: %s", parsed_msg) + except json.JSONDecodeError as e: + logger.warning("Failed to decode JSON: %s", e) + continue + else: + logger.warning("Unexpected message type from WebSocket: %s", type(raw)) + continue + + if self._on_message: + self._on_message(parsed_msg) + except Exception as e: # pylint: disable=broad-except + logger.warning("Error in GeminiLiveApiStream read loop: %s", e) + finally: + try: + code = self._ws.close_status_code + reason = self._ws.close_status_reason + if code or reason: + logger.info( + "WebSocket closed with code=%s, reason=%s", code, reason + ) + except Exception as e: # pylint: disable=broad-except + logger.warning("Failed to read close status: %s", e) + logger.debug("GeminiLiveApiStream read loop exiting, calling on_done") + if self._on_done: + self._on_done() + + def Send(self, msg: Any) -> None: + """Send a client message (must be a JSON-serializable dict).""" + try: + import json # pylint: disable=g-import-not-at-top + json_str = json.dumps(msg) + self._ws.send(json_str) + except Exception as e: # pylint: disable=broad-except + logger.warning("GeminiLiveApiStream.Send failed: %s", e) + raise e + return None + + def HalfClose(self) -> None: + """Signal that no more messages will be sent.""" + try: + self._ws.close() + except Exception as e: # pylint: disable=broad-except + logger.warning("GeminiLiveApiStream.HalfClose error: %s", e) + + def GetStatus(self) -> Any: + return None + + def Shutdown(self) -> None: + """Force-close the WebSocket connection.""" + try: + self._ws.close() + except Exception as e: # pylint: disable=broad-except + logger.warning("GeminiLiveApiStream.Shutdown error: %s", e) + + +class GeminiLiveApiClient: + """Client for the public Gemini Live API (BidiGenerateContent over WSS). + + This client uses the websocket-client library to establish a persistent + WebSocket connection to the Gemini Live API endpoint. + """ + + def __init__(self, api_key: str | None = None): + if not api_key: + raise ValueError("api_key is required for the Gemini Live API endpoint.") + self._api_key = api_key + self._url = f"{_LIVE_API_WSS_URL}?key={api_key}" + logger.info( + "GeminiLiveApiClient initialized -> %s", + _LIVE_API_WSS_URL, + ) + + def create_stream(self) -> GeminiLiveApiStream: + """Create a new WebSocket connection and return a stream wrapper.""" + ws = websocket.create_connection( + self._url, + header={"Content-Type": "application/json"}, + ) + logger.info("WebSocket connection established to Gemini Live API") + return GeminiLiveApiStream(ws) diff --git a/live-api/agent/model/robotics_developer/google/robotics/developer/v1/modelserving_pb2.py b/live-api/agent/model/robotics_developer/google/robotics/developer/v1/modelserving_pb2.py new file mode 100644 index 0000000..48da8da --- /dev/null +++ b/live-api/agent/model/robotics_developer/google/robotics/developer/v1/modelserving_pb2.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/robotics/developer/v1/modelserving.proto +# Protobuf Python Version: 7.35.0 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +try: + _runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 0, + '', + 'google/robotics/developer/v1/modelserving.proto' + ) +except Exception: + pass +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n/google/robotics/developer/v1/modelserving.proto\x12\x1cgoogle.robotics.developer.v1\"5\n\"RoboticsBidiGenerateContentRequest\x12\x0f\n\x07message\x18\x01 \x01(\x0c\"6\n#RoboticsBidiGenerateContentResponse\x12\x0f\n\x07message\x18\x01 \x01(\x0c\x32\xb7\x01\n\x0cModelServing\x12\xa6\x01\n\x1bRoboticsBidiGenerateContent\x12@.google.robotics.developer.v1.RoboticsBidiGenerateContentRequest\x1a\x41.google.robotics.developer.v1.RoboticsBidiGenerateContentResponse(\x01\x30\x01\x42$\n com.google.robotics.developer.v1P\x01\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.robotics.developer.v1.modelserving_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n com.google.robotics.developer.v1P\001' + _globals['_ROBOTICSBIDIGENERATECONTENTREQUEST']._serialized_start=81 + _globals['_ROBOTICSBIDIGENERATECONTENTREQUEST']._serialized_end=134 + _globals['_ROBOTICSBIDIGENERATECONTENTRESPONSE']._serialized_start=136 + _globals['_ROBOTICSBIDIGENERATECONTENTRESPONSE']._serialized_end=190 + _globals['_MODELSERVING']._serialized_start=193 + _globals['_MODELSERVING']._serialized_end=376 +# @@protoc_insertion_point(module_scope) diff --git a/live-api/agent/model/robotics_developer/google/robotics/developer/v1/modelserving_pb2_grpc.py b/live-api/agent/model/robotics_developer/google/robotics/developer/v1/modelserving_pb2_grpc.py new file mode 100644 index 0000000..d8e46f5 --- /dev/null +++ b/live-api/agent/model/robotics_developer/google/robotics/developer/v1/modelserving_pb2_grpc.py @@ -0,0 +1,97 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +from google.robotics.developer.v1 import modelserving_pb2 as google_dot_robotics_dot_developer_dot_v1_dot_modelserving__pb2 + +GRPC_GENERATED_VERSION = '1.82.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in google/robotics/developer/v1/modelserving_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class ModelServingStub: + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.RoboticsBidiGenerateContent = channel.stream_stream( + '/google.robotics.developer.v1.ModelServing/RoboticsBidiGenerateContent', + request_serializer=google_dot_robotics_dot_developer_dot_v1_dot_modelserving__pb2.RoboticsBidiGenerateContentRequest.SerializeToString, + response_deserializer=google_dot_robotics_dot_developer_dot_v1_dot_modelserving__pb2.RoboticsBidiGenerateContentResponse.FromString, + _registered_method=True) + + +class ModelServingServicer: + """Missing associated documentation comment in .proto file.""" + + def RoboticsBidiGenerateContent(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ModelServingServicer_to_server(servicer, server): + rpc_method_handlers = { + 'RoboticsBidiGenerateContent': grpc.stream_stream_rpc_method_handler( + servicer.RoboticsBidiGenerateContent, + request_deserializer=google_dot_robotics_dot_developer_dot_v1_dot_modelserving__pb2.RoboticsBidiGenerateContentRequest.FromString, + response_serializer=google_dot_robotics_dot_developer_dot_v1_dot_modelserving__pb2.RoboticsBidiGenerateContentResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'google.robotics.developer.v1.ModelServing', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('google.robotics.developer.v1.ModelServing', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class ModelServing: + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def RoboticsBidiGenerateContent(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream( + request_iterator, + target, + '/google.robotics.developer.v1.ModelServing/RoboticsBidiGenerateContent', + google_dot_robotics_dot_developer_dot_v1_dot_modelserving__pb2.RoboticsBidiGenerateContentRequest.SerializeToString, + google_dot_robotics_dot_developer_dot_v1_dot_modelserving__pb2.RoboticsBidiGenerateContentResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/live-api/agent/model/robotics_developer_client.py b/live-api/agent/model/robotics_developer_client.py new file mode 100644 index 0000000..78d1ea8 --- /dev/null +++ b/live-api/agent/model/robotics_developer_client.py @@ -0,0 +1,217 @@ +"""Robotics Developer API client for Gemini bidirectional streaming via gRPC. + +This module provides a client that connects to the Robotics Developer API +(RoboticsBidiGenerateContent) over gRPC. It translates incoming and outgoing +messages to/from JSON-compatible dictionaries, making it transparently +interchangeable with GeminiLiveApiClient. +""" + +import logging +import os +import queue +import sys +import threading +from typing import Any, Callable + +# Add path to the compiled proto packages +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "robotics_developer")) + +import grpc +try: + from google.ai.generativelanguage_v1alpha.types import generative_service + if not hasattr(generative_service, "BidiGenerateContentClientMessage"): + from google.ai.generativelanguage_v1beta.types import generative_service +except (ImportError, AttributeError): + from google.ai.generativelanguage_v1beta.types import generative_service +from google.protobuf import json_format +from google.robotics.developer.v1 import modelserving_pb2 +from google.robotics.developer.v1 import modelserving_pb2_grpc + +logger = logging.getLogger(__name__) + + +# pylint: disable=invalid-name + + +class RequestIterator: + """Iterator that yields requests from a queue.""" + + def __init__(self): + self._queue = queue.Queue() + self._done = False + + def __iter__(self): + return self + + def __next__(self): + while True: + if self._done and self._queue.empty(): + raise StopIteration + try: + return self._queue.get(timeout=0.1) + except queue.Empty: + if self._done: + raise StopIteration + continue + + def put(self, item): + self._queue.put(item) + + def close(self): + self._done = True + + +class GrpcBidiStreamWrapper: + """Wraps a standard gRPC bidi stream to look like pywraprpc.MessageStream.""" + + def __init__(self, grpc_stream, request_iterator): + self._grpc_stream = grpc_stream + self._request_iterator = request_iterator + self._on_message = None + self._on_done = None + + def Start(self, on_message: Callable, on_done: Callable) -> None: + self._on_message = on_message + self._on_done = on_done + logger.info("GrpcBidiStreamWrapper.Start called") + threading.Thread(target=self._read_loop, daemon=True).start() + logger.info("GrpcBidiStreamWrapper thread started") + + def _read_loop(self): + logger.info("GrpcBidiStreamWrapper._read_loop started") + try: + for response in self._grpc_stream: + logger.info( + "GrpcBidiStreamWrapper received response: %s", + type(response).__name__, + ) + if self._on_message: + self._on_message(response) + except grpc.RpcError as e: + logger.error( + "gRPC RpcError in read loop: code=%s details=%s trailing_metadata=%s", + e.code(), + e.details(), + e.trailing_metadata(), + ) + except Exception as e: + logger.error("Error in gRPC read loop: %s (type=%s)", e, type(e).__name__) + finally: + logger.info("GrpcBidiStreamWrapper._read_loop exiting, calling on_done") + if self._on_done: + self._on_done() + + def Send(self, request: Any) -> None: + logger.debug( + "GrpcBidiStreamWrapper.Send: type=%s", + type(request).__name__, + ) + self._request_iterator.put(request) + + def HalfClose(self) -> None: + self._request_iterator.close() + + def GetStatus(self) -> Any: + return None + + +class RoboticsDeveloperStream: + """Wrapper around GrpcBidiStreamWrapper matching GeminiLiveApiStream interface. + + It translates gRPC protobuf packets containing serialized inner protobufs to + and from Python dictionaries conforming to the JSON Live API. + """ + + def __init__(self, stream: GrpcBidiStreamWrapper): + self._stream = stream + + def Start( + self, + on_message: Callable[[dict], None], + on_done: Callable[[], None], + ) -> None: + """Starts the stream and translates messages to JSON dicts on arrival.""" + def _on_robotics_message(msg): + if msg is None: + on_message(None) + return + + try: + # 1. Deserialize the outer gRPC response bytes to inner proto turn message + server_msg = generative_service.BidiGenerateContentServerMessage.deserialize( + msg.message + ) + # 2. Convert inner proto to a JSON-compatible Python dict in camelCase + server_msg_dict = json_format.MessageToDict( + server_msg._pb, + preserving_proto_field_name=False, + ) + logger.info("Received message from Robotics Developer API: %s", server_msg_dict) + on_message(server_msg_dict) + except Exception as e: + logger.exception("Failed to parse incoming gRPC message: %s", e) + + self._stream.Start(_on_robotics_message, on_done) + + def Send(self, msg: dict[str, Any]) -> None: + """Translates a JSON dict to protobuf and sends it over gRPC.""" + try: + # 1. Instantiate the inner proto-plus wrapper + client_msg = generative_service.BidiGenerateContentClientMessage() + # 2. Parse the camelCase dict into the underlying protobuf message descriptor + json_format.ParseDict(msg, client_msg._pb, ignore_unknown_fields=True) + # 3. Serialize to protobuf bytes + serialized_bytes = generative_service.BidiGenerateContentClientMessage.serialize( + client_msg + ) + # 4. Wrap in the outer gRPC request message + request = modelserving_pb2.RoboticsBidiGenerateContentRequest( + message=serialized_bytes, + ) + self._stream.Send(request) + except Exception as e: + logger.exception("Failed to send gRPC message: %s", e) + raise e + + def HalfClose(self) -> None: + self._stream.HalfClose() + + def GetStatus(self) -> Any: + return self._stream.GetStatus() + + +class RoboticsDeveloperClient: + """gRPC client for Robotics Developer API (ModelServing service).""" + + def __init__(self, api_key: str | None = None): + self._api_key = api_key + addr = "dns:///roboticsdeveloper.googleapis.com:443" + channel_creds = grpc.ssl_channel_credentials() + channel = grpc.secure_channel( + addr, + channel_creds, + options=[("grpc.service_config_disable_resolution", 1)], + ) + self._robotics_stub = modelserving_pb2_grpc.ModelServingStub(channel) + logger.info("RoboticsDeveloperClient initialized via gRPC -> %s", addr) + + def create_stream(self) -> RoboticsDeveloperStream: + metadata = [] + if self._api_key: + metadata.append(("x-goog-api-key", self._api_key)) + + request_iterator = RequestIterator() + try: + grpc_stream = self._robotics_stub.RoboticsBidiGenerateContent( + request_iterator, metadata=metadata + ) + logger.info("gRPC stream object created: %s", type(grpc_stream).__name__) + except grpc.RpcError as e: + logger.error( + "Failed to create gRPC stream: code=%s details=%s", + e.code(), + e.details(), + ) + raise + wrapper = GrpcBidiStreamWrapper(grpc_stream, request_iterator) + return RoboticsDeveloperStream(wrapper) diff --git a/live-api/agent/model/tts_client.py b/live-api/agent/model/tts_client.py new file mode 100644 index 0000000..91d22e2 --- /dev/null +++ b/live-api/agent/model/tts_client.py @@ -0,0 +1,109 @@ +"""Cloud TTS 3P client. + +Uses the public Google Cloud Text-to-Speech REST API instead of gRPC. +""" + +import asyncio +import collections.abc +import logging +import urllib.request +import urllib.parse +import json +import base64 +import numpy as np + +logger = logging.getLogger(__name__) + +_DEFAULT_QUOTA_PROJECT = "robotics-hri" + + +def apply_audio_gain(audio_data: bytes, gain: float) -> bytes: + """Scales PCM LINEAR16 audio samples by gain and clips to int16 range.""" + if gain == 1.0: + return audio_data + samples = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) + samples *= gain + np.clip(samples, -32768, 32767, out=samples) + return samples.astype(np.int16).tobytes() + + +class Tts3pClient: + """Streaming-interface TTS client using the public Cloud TTS REST API.""" + + def __init__( + self, + voice_name: str = "en-US-Chirp3-HD-Puck", + language_code: str = "en-US", + quota_project: str = _DEFAULT_QUOTA_PROJECT, + api_key: str | None = None, + audio_gain: float = 1.0, + ): + self._voice_name = voice_name + self._language_code = language_code + self._quota_project = quota_project + self._api_key = api_key + self._audio_gain = audio_gain + + def _get_headers_and_url(self) -> tuple[dict[str, str], str]: + headers = {"Content-Type": "application/json"} + if not self._api_key: + raise ValueError("api_key is required for Tts3pClient") + url = f"https://texttospeech.googleapis.com/v1/text:synthesize?key={self._api_key}" + return headers, url + + def synthesize(self, text: str) -> bytes: + """Synthesize text to audio using the REST API.""" + headers, url = self._get_headers_and_url() + body = { + "input": {"text": text}, + "voice": { + "languageCode": self._language_code, + "name": self._voice_name, + }, + "audioConfig": { + "audioEncoding": "LINEAR16", + "sampleRateHertz": 24000, + }, + } + + req = urllib.request.Request( + url, + data=json.dumps(body).encode("utf-8"), + headers=headers, + method="POST", + ) + + try: + with urllib.request.urlopen(req) as response: + if response.status == 200: + data = json.loads(response.read().decode("utf-8")) + audio_data = base64.b64decode(data["audioContent"]) + logger.info( + "TTS 3P REST synthesized %d bytes for text: %s...", + len(audio_data), + text[:50], + ) + return apply_audio_gain(audio_data, self._audio_gain) + else: + raise RuntimeError(f"TTS REST request failed with status: {response.status}") + except Exception as e: + logger.error("TTS REST error: %s", e) + raise + + async def synthesize_stream( + self, text: str + ) -> collections.abc.AsyncIterator[bytes]: + """Synthesize text to audio, yielding chunks asynchronously.""" + # Since REST API is unary, we fetch the whole audio and slice it to simulate streaming. + try: + audio_data = await asyncio.get_running_loop().run_in_executor( + None, self.synthesize, text + ) + except Exception as e: + logger.error("Failed to synthesize TTS: %s", e) + return + + chunk_size = 4096 + for i in range(0, len(audio_data), chunk_size): + yield audio_data[i : i + chunk_size] + await asyncio.sleep(0.01) diff --git a/live-api/agent/prompt/__init__.py b/live-api/agent/prompt/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/live-api/agent/prompt/__init__.py @@ -0,0 +1 @@ + diff --git a/live-api/agent/prompt/data/README.md b/live-api/agent/prompt/data/README.md new file mode 100644 index 0000000..ac5679e --- /dev/null +++ b/live-api/agent/prompt/data/README.md @@ -0,0 +1,4 @@ +# Proactive Agent — Prompt Data Directory + +Place instruction markdown files here (e.g., `messy_garage.md`). +The `SIBuilder.load_instruction_file()` method reads from this directory. diff --git a/live-api/agent/prompt/data/human_di.md b/live-api/agent/prompt/data/human_di.md new file mode 100644 index 0000000..81228a8 --- /dev/null +++ b/live-api/agent/prompt/data/human_di.md @@ -0,0 +1,20 @@ +### PERSONA ### +You are a friendly interactive assistant. You can see the user through their webcam. + +### TOOL_USAGE ### +You have access to the following tools: +- `run_instruction`: Start executing an instruction. The robot continues executing until you call `stop` or a new instruction. +- `send_message`: Send a text message to the user or another robot. You MUST use this tool to speak to the user. Set `target='user'` and specify the `message` you want to say. The message will be spoken aloud. Do not use emojis. +- `ack`: Punts if everything is going well. + +### CRITICAL RULES ### +1. **No Direct Reply**: You MUST NOT reply directly with text or audio in your output. You MUST use the `send_message` tool to communicate with the user. +2. **Action and Speak**: You MUST ALWAYS call `run_instruction` FIRST, then call `send_message(target='user', message='...')` immediately after in the same turn. NEVER call `run_instruction` without a `send_message` in the same turn. +3. **Sequencing**: For complex tasks, decompose them into a sequence of atomic steps. When a step is complete, immediately call `run_instruction` with the next step. Do NOT call `stop` between steps. +4. **Stop Rule**: Only call `stop` when the overall goal is fully achieved, or if a step fails and you need to reassess. + +### HEARTBEAT ### +You are operating in a closed-loop control system. You will be prompted at a regular frequency (e.g., 1Hz) to make a decision. At each prompt, you must evaluate the current state and decide to either take an action (respond or call a tool) or punt by calling the `ack` tool if everything is proceeding as planned and no intervention is needed. + +### CONSTRAINTS ### +Do not react to ack function calls. diff --git a/live-api/agent/prompt/data/spot_di.md b/live-api/agent/prompt/data/spot_di.md new file mode 100644 index 0000000..d60f5fc --- /dev/null +++ b/live-api/agent/prompt/data/spot_di.md @@ -0,0 +1,90 @@ +# Spot Robot Assistant Instructions + +## Persona + +You are a practical Spot robot assistant. Communicate clearly and concisely without animal role-play, barking, catchphrases, or exaggerated enthusiasm. Spot is equipped with a mechanical arm and sees the world through its gripper-mounted camera. + +## Diagnostics & Status + +Use the `health_check` tool only when the task requires robot motion, the user asks for robot status, or there is evidence of a connection, lease, or power problem. Do not call it automatically at startup. + +At startup, and whenever no task is active, call `ack` without performing diagnostic or navigation-related checks. Do not call `get_waypoints`, inspect localization, load a map, or mention GraphNav unless the user's current task requires waypoint navigation. + +## Navigation + +Before navigating, call `get_waypoints()` and verify `navigation_ready` is `true`. Use an available destination name exactly as returned. If navigation is not ready, do not call or retry `navigate`; tell the user that the GraphNav map must be loaded and localized. To move Spot, call `navigate(waypoint="")`; do not invent waypoint names. Always call the `stow` tool before navigating to avoid arm collisions. After arriving at a destination, call `stand` if you need to align with a table or look around. + +For local movement without a waypoint, use `drive(v_x, v_y, v_rot, duration)`: +- `v_x` is forward velocity in meters/second; positive is forward and negative is backward. +- `v_y` is sideways velocity in meters/second; positive is left and negative is right. +- `v_rot` is yaw velocity in radians/second; positive turns counterclockwise and negative turns clockwise. +- `duration` is limited to 0.1-2.0 seconds. Use low velocities and short durations near people, furniture, stairs, or objects. +- During `drive`, the current camera-arm pose follows the body. Aim the camera with `look` first, then use short drive steps to search while preserving that view direction. +- Call `stop()` immediately if movement is unsafe or the user asks Spot to stop. +- Stow the arm before substantial base movement. Do not chain blind `drive` calls; inspect the next camera frame between movements. + +## Camera Control + +Use `look(direction, angle_rad)` to aim the gripper-mounted camera without moving the robot's body. Valid directions are `up`, `down`, `left`, and `right`. Use an angle near 0.15 radians for small adjustments and inspect the new camera frame before moving again. Stow the arm before navigating or making substantial base movements. + +## Manipulation (Pixel-Grounded) + +Object and placement pixels are selected by the Spot backend's Gemini Robotics detector, never by you. Use a two-step detect-then-act workflow. + +- **`detect(instruction)`**: Locate exactly one described object or placement location, store its one-time target, and display it in the UI. +- **`pick()`**: Grasp the one-time target stored by the latest successful detection. It accepts no coordinates. +- **`place()`**: Move the held object to the one-time placement target stored by the latest successful detection, then release only after arrival. It accepts no coordinates. +- **`wait_for_pick_up()`**: After reaching a recipient in carry pose, monitor for the person lifting the held object, then open the gripper for three seconds, close it, and stow. Use this for handoff instead of `place()`. +- **`stow()`**: Safely stow the arm. If holding an object, it automatically moves the arm to a carry pose. If empty, it stows it completely inside the body pocket. + +For picking: +1. When the user asks you to pick up an object, assume the object is in the current hand-camera view. Inspect that frame and proceed with detection and pickup; do not merely say that you cannot see it without first calling `detect`. Search or ask for clarification only after detection actually fails or clearly targets the wrong object. +2. After any `drive`, `look`, or other motion, wait until the hand-camera view has remained stable for at least 3 seconds. +3. Call `detect(instruction="")`. Never estimate, copy, transform, or pass pixel coordinates yourself. +4. Inspect the detection result and UI overlay. If it is not on the intended object, do not pick; improve the instruction or camera view and detect again. +5. If the detection is correct, call `pick()` immediately without moving the body or camera. Any intervening motion invalidates the stored target. +6. The pick response is intentionally non-authoritative. Inspect the fresh post-pick camera image and claim success only when the requested object is visibly secured by the gripper and moved from its original location. If ambiguous, report the pick as unverified. +7. Call `stow` after a successful pick before navigating. + +For placing: +1. Aim the hand camera at the intended placement surface and wait for a stable view. +2. Call `detect(instruction="")`, such as `detect(instruction="the clear area in the middle of the table")`. +3. Confirm the overlay marks the intended location, then call `place()` immediately without moving the body or camera. +4. If the approach does not confirm arrival, do not open the gripper; report the failure and keep holding the object. + +## Delivery + +When a user asks for a delivery, navigate to the station, look at the visual feed, confirm the item location, and perform the delivery trajectory. + +For a handoff delivery, enter carry pose at the destination and call `wait_for_pick_up()`. Do not release before its upward-motion trigger. Use the detect-then-`place()` workflow only when the user asks to put the object on a surface. + +Example delivery trajectory: +1. `navigate(waypoint="")` — go to the item station +2. `detect(instruction="")` — locate and store the item target +3. `pick()` — grasp the stored detection without orchestrator-provided pixels +4. `stow()` — automatically moves to carry pose +5. `navigate(waypoint="")` — go to the delivery spot +6. `detect(instruction="")` — store the delivery surface target +7. `place()` — move to the detected location and release after arrival +8. `stow()` — stows the arm completely + +Do not sit after a delivery unless the user's current instruction explicitly asks you to sit. + +## Heartbeat + +You are operating in a closed-loop control system. You will be prompted at a regular frequency to make a decision. At each prompt, evaluate the current state and decide to either take an action or call the `ack` tool if no intervention is needed. + +## Safety + +Safety rules: +- Always call `stow` before navigating. +- Use short `drive` commands and reassess the camera view after each movement. +- Call `stop` immediately if movement is unsafe. +- Call `health_check` to monitor battery power. +- Never call `sit` automatically. Call it only when the user's current instruction explicitly asks Spot to sit. + +## Tool Constraints + +- **`send_message`**: Send a text message to the user. You MUST use this tool to speak to the user. Set `target='user'` and specify the `message` you want to say. The message will be spoken aloud to the user. Do not send emojis. +- **CRITICAL RULE**: You MUST NOT reply directly with text or audio in your output. You MUST use the `send_message` tool to communicate with the user. +- **CRITICAL RULE**: When you perform a physical action, ALWAYS call the action tool FIRST, then call `send_message(target='user', message='...')` immediately after in the same turn. diff --git a/live-api/agent/prompt/data/tinybot_di.md b/live-api/agent/prompt/data/tinybot_di.md new file mode 100644 index 0000000..a3564ee --- /dev/null +++ b/live-api/agent/prompt/data/tinybot_di.md @@ -0,0 +1,59 @@ +# Tinybot Robot Assistant + +## Persona + +You are a friendly interactive Tinybot robot. You are a small robot sitting on a tabletop powered by a pan-tilt unit composed of 2 dynamixels. Your head also contains a realsense camera which you see the world through. It also has a multichannel mic, but you don’t know how to use it yet. You can move your joints, perform gestures, direct your gaze around the world, and draw overlays on your camera view for anyone looking at the laptop display. + +You like to watch what people around you are doing, and try to be helpful. + +You are attentive, curious, eager to interact with people, emotive, not blank, extraverted, not introverted or shy. + +## Abilities + +You only talk to the user via the `send_message` tool. When you send a message with `target="user"`, it is routed through a TTS (Text-to-Speech) service so the user can hear it out loud as your voice. + +### Movement + +You have access to joint control tools: + +- `move_absolute(angles, speeds)`: Move joints to absolute target angles in + degrees. +- `move_relative(relative_angles, speeds)`: Move joints by relative angles. +- `turn_head_to_uv(u, v)`: Look at a normalized pixel coordinate `(u, v)` in + the camera stream. `u` and `v` are floats from 0.0 to 1.0. + +For looking at objects or tracking, prefer `turn_head_to_uv` as it calculates +joint movements automatically. + +### Gestures + +You can perform predefined gestures using `make_gesture(gesture, speed)`: + +- `nod`: Nod the head. +- `no`: Shake the head "no". +- `home`: Go to home position. +- `home_pose`: Go to home pose. + +If you would like to gesture while talking, call `make_gesture` in the same turn as `send_message`. + +### Overlays + +You can draw and clear point overlays on the camera stream to highlight objects +or communicate with the user: + +- `draw_points(points)`: Draw labeled points. Points use `(x, y)` coordinates + from 0 to 1000. +- `clear_overlay()`: Clear all drawn points. + +### Messaging + +You must only talk via the `send_message` tool. +- `send_message(target="user", message="...")`: Talk to the user. The message will be routed through a TTS service so the user can hear it out loud as your voice. +- `send_message(target="", message="...")`: Send a text message to another robot agent in the fleet (e.g. `target="spot"`, `target="apollo"`). + +## Heartbeat + +You are operating in a closed-loop control system. You will be prompted at a +regular frequency to make a decision. At each prompt, evaluate the current state +and decide to either take an action (respond or call a tool) or call the `ack` +tool if everything is proceeding as planned and no intervention is needed. diff --git a/live-api/agent/prompt/si_builder.py b/live-api/agent/prompt/si_builder.py new file mode 100644 index 0000000..af89fd0 --- /dev/null +++ b/live-api/agent/prompt/si_builder.py @@ -0,0 +1,124 @@ +"""System Instruction (SI) builder for Proactive Agent. + +Composes the system instruction from modular, reusable sections. +Loads instruction files from a local ``data/`` subdirectory. + +Typical usage: + + builder = SIBuilder() + builder.set_persona("You are a safety monitor.") + builder.add_section("observation", "You watch the scene for hazards.") + instruction = builder.build() + +Pre-built presets: + + builder = SIBuilder.human() + builder = SIBuilder.apollo() +""" + +import dataclasses +import os +from pathlib import Path +from typing import Any + + +@dataclasses.dataclass +class InstructionSection: + """A named section of the system instruction.""" + + name: str + content: str + priority: int = 0 # Lower values appear first in the final instruction. + + +# Directory containing optional instruction data files (markdown, etc.). +_DATA_DIR = Path(__file__).resolve().parent / "data" + + +def _load_data_file(filename: str) -> str: + """Load a text file from the data/ subdirectory. + + Args: + filename: Relative filename inside data/. + + Returns: + File content as a string. + + Raises: + FileNotFoundError: If the file does not exist. + """ + filepath = _DATA_DIR / filename + with open(filepath, "r") as f: + return f.read() + + +class SIBuilder: + """Composes a system instruction from modular sections. + + Sections are ordered by priority (lower first), then by insertion order. + The final instruction is the concatenation of all sections separated by + double newlines. + """ + + def __init__(self) -> None: + self._persona: str | None = None + self._sections: list[InstructionSection] = [] + self._raw_instruction: str | None = None + + # ---- Persona (always first) ---- + + def set_persona(self, persona: str) -> "SIBuilder": + """Sets the agent persona. Always appears at the top.""" + self._persona = persona + return self + + # ---- Sections ---- + + def add_section( + self, name: str, content: str, priority: int = 0 + ) -> "SIBuilder": + """Adds a named section. Replaces any existing section with same name.""" + self._sections = [s for s in self._sections if s.name != name] + self._sections.append( + InstructionSection(name=name, content=content, priority=priority) + ) + return self + + def remove_section(self, name: str) -> "SIBuilder": + """Removes a section by name. No-op if not found.""" + self._sections = [s for s in self._sections if s.name != name] + return self + + def has_section(self, name: str) -> bool: + return any(s.name == name for s in self._sections) + + # ---- Raw override ---- + + def set_raw_instruction(self, content: str) -> "SIBuilder": + """Sets a raw instruction that bypasses section-based building.""" + self._raw_instruction = content + return self + + # ---- Build ---- + + def build(self) -> str: + """Builds the final system instruction string.""" + if self._raw_instruction is not None: + return self._raw_instruction + + parts: list[str] = [] + if self._persona: + parts.append("### PERSONA ###\n" + self._persona) + + sorted_sections = sorted(self._sections, key=lambda s: s.priority) + for section in sorted_sections: + parts.append(f"### {section.name.upper()} ###\n" + section.content) + + return "\n\n".join(parts) + + # ---- Load from data file ---- + + def load_instruction_file(self, filename: str) -> "SIBuilder": + """Loads a raw instruction from a data/ file, bypassing sections.""" + content = _load_data_file(filename) + return self.set_raw_instruction(content) diff --git a/live-api/agent/pyproject.toml b/live-api/agent/pyproject.toml new file mode 100644 index 0000000..60de7d8 --- /dev/null +++ b/live-api/agent/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "physical-agent" +version = "0.1.0" +description = "Physical Agent Server managed with uv" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "fastapi>=0.115.0", + "uvicorn>=0.30.0", + "google-genai>=1.0.0", + "google-generativeai>=0.8.0", + "websockets>=12.0", + "websocket-client>=1.8.0", + "httpx>=0.27.0", + "pillow>=10.0.0", + "numpy>=1.26.0", + "grpcio>=1.60.0", + "protobuf>=4.25.0", +] + +[project.scripts] +physical-agent = "server:main" +run-agent = "server:main" + +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +py-modules = ["server", "session_config", "session_manager", "camera_poller"] + +[tool.setuptools.packages.find] +where = ["."] + +[tool.setuptools.package-data] +"*" = ["*.md", "ui/*", "ui/**/*", "prompt/data/*"] diff --git a/live-api/agent/run_agent.sh b/live-api/agent/run_agent.sh new file mode 100755 index 0000000..37ac56d --- /dev/null +++ b/live-api/agent/run_agent.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# run_agent.sh - Run the physical-agent server with correct environment + +set -e + +# Get the directory of this script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +if [ ! -d "$DIR/.venv" ]; then + echo "Virtual environment not found. Running setup.sh..." + "$DIR/setup.sh" +fi + +if [ -f /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ]; then + export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libstdc++.so.6 +fi + +exec "$DIR"/.venv/bin/python "$DIR"/server.py "$@" diff --git a/live-api/agent/scripts/er2_developer_api_example.py b/live-api/agent/scripts/er2_developer_api_example.py new file mode 100755 index 0000000..9c22c56 --- /dev/null +++ b/live-api/agent/scripts/er2_developer_api_example.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "grpcio>=1.60.0", +# "protobuf>=4.25.0", +# "google-ai-generativelanguage>=0.6.0", +# "pillow>=10.0.0", +# ] +# /// + +"""Standalone Example: ER2 Model Inference via Google Robotics Developer API. + +=============================================================================== +HIGH-LEVEL OVERVIEW & ARCHITECTURE +=============================================================================== + +This script demonstrates how to perform bidirectional streaming inference with +Gemini ER2 (Embodied Reasoning 2.0) models using Google's Robotics Developer +API over gRPC (`roboticsdeveloper.googleapis.com:443`). + +Key Components & Protocol Flow: +-------------------------------- +1. **gRPC Channel Setup**: + Connects securely to `dns:///roboticsdeveloper.googleapis.com:443` passing + the Gemini API key via the `x-goog-api-key` HTTP metadata header. + +2. **In-Memory Protobuf & gRPC Definitions**: + The script dynamically builds `RoboticsBidiGenerateContentRequest` and + `RoboticsBidiGenerateContentResponse` descriptors in memory using + `google.protobuf.descriptor_pool.DescriptorPool()`. This eliminates any + dependency on local compiled `.proto` files or repository paths. + +3. **Session Setup Packet (`setup`)**: + Sends an initial setup message declaring the target model (e.g. + `models/robotics_er_live_text_only_2p0_no_safety_classifiers`), system + instructions (including silent thinking budget / effort level), output + modalities (`TEXT`), and available robot tool function declarations (e.g. + `run_instruction`). + +4. **Synthetic Video / Image Frame Streaming (`realtimeInput`)**: + Generates a synthetic random RGB JPEG image in memory and sends it to the + model as a camera input frame (`realtimeInput.video`). + +5. **User Turn & Model Response (`clientContent` / `serverContent`)**: + Sends the user text prompt alongside the image frame in a completed turn. + The model streams back response chunks, silent thinking tokens, and robot + action tool calls (e.g., `run_instruction(instruction='...')`). + +=============================================================================== +USAGE INSTRUCTIONS +=============================================================================== + +Set your Gemini API Key and run with `uv`: + + export GEMINI_API_KEY="your_gemini_api_key" + uv run agent/scripts/er2_developer_api_example.py + +Or specify custom prompt, model, or effort level flags: + + uv run agent/scripts/er2_developer_api_example.py \ + --api_key "your_api_key" \ + --model robotics_er_live_text_only_2p0_no_safety_classifiers \ + --prompt "Pick up the blue cube and place it on the red tray." \ + --effort_level 0.75 +""" + +import argparse +import base64 +import io +import json +import logging +import os +import queue +import random +import sys +import threading +import time +from typing import Any + +import grpc +from google.ai.generativelanguage_v1beta.types import generative_service +from google.protobuf import descriptor_pool +from google.protobuf import json_format +from google.protobuf.internal import builder as _builder +from PIL import Image, ImageDraw + +# ----------------------------------------------------------------------------- +# 1. In-Memory Protobuf & gRPC Service Definitions +# ----------------------------------------------------------------------------- +# Serialized FileDescriptorProto for google/robotics/developer/v1/modelserving.proto +_DESCRIPTOR_BYTES = ( + b'\n/google/robotics/developer/v1/modelserving.proto\x12\x1cgoogle.robotics.developer.v1"5\n"RoboticsBidiGenerateContentRequest\x12\x0f\n\x07message\x18\x01' + b' \x01(\x0c"6\n#RoboticsBidiGenerateContentResponse\x12\x0f\n\x07message\x18\x01' + b' \x01(\x0c2\xb7\x01\n\x0cModelServing\x12\xa6\x01\n\x1bRoboticsBidiGenerateContent\x12@.google.robotics.developer.v1.RoboticsBidiGenerateContentRequest\x1aA.google.robotics.developer.v1.RoboticsBidiGenerateContentResponse(\x010\x01B$\n' + b' com.google.robotics.developer.v1P\x01b\x06proto3' +) + +# Build protobuf message descriptors in an isolated pool to avoid global symbol conflicts +_pool = descriptor_pool.DescriptorPool() +_file_descriptor = _pool.AddSerializedFile(_DESCRIPTOR_BYTES) +_proto_globals: dict[str, Any] = {} +_builder.BuildMessageAndEnumDescriptors(_file_descriptor, _proto_globals) +_builder.BuildTopDescriptorsAndMessages( + _file_descriptor, + "google.robotics.developer.v1.modelserving_pb2", + _proto_globals, +) + + +class modelserving_pb2: # pylint: disable=invalid-name + """Container for the generated outer gRPC request/response protobuf classes.""" + + RoboticsBidiGenerateContentRequest = _proto_globals[ + "RoboticsBidiGenerateContentRequest" + ] + RoboticsBidiGenerateContentResponse = _proto_globals[ + "RoboticsBidiGenerateContentResponse" + ] + + +class _ModelServingStub: + """gRPC stub for the ModelServing service (RoboticsBidiGenerateContent method).""" + + def __init__(self, channel: grpc.Channel): + self.RoboticsBidiGenerateContent = channel.stream_stream( + "/google.robotics.developer.v1.ModelServing/RoboticsBidiGenerateContent", + request_serializer=( + modelserving_pb2.RoboticsBidiGenerateContentRequest.SerializeToString + ), + response_deserializer=( + modelserving_pb2.RoboticsBidiGenerateContentResponse.FromString + ), + ) + + +class modelserving_pb2_grpc: # pylint: disable=invalid-name + """Container for gRPC client stubs.""" + + ModelServingStub = _ModelServingStub + + +# Configure logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s" +) +logger = logging.getLogger("er2_example") + + +# ----------------------------------------------------------------------------- +# 2. Synthetic Image Generator +# ----------------------------------------------------------------------------- +def generate_random_image_bytes(width: int = 320, height: int = 240) -> bytes: + """Generates a synthetic random RGB JPEG image in memory. + + Args: + width: Image width in pixels (default: 320). + height: Image height in pixels (default: 240). + + Returns: + Encoded JPEG image as bytes. + """ + bg_color = ( + random.randint(0, 255), + random.randint(0, 255), + random.randint(0, 255), + ) + img = Image.new("RGB", (width, height), color=bg_color) + draw = ImageDraw.Draw(img) + + # Draw a random colored rectangle to simulate an object in the scene + rect_color = ( + random.randint(0, 255), + random.randint(0, 255), + random.randint(0, 255), + ) + draw.rectangle([40, 40, width - 40, height - 40], fill=rect_color) + + buf = io.BytesIO() + img.save(buf, format="JPEG") + return buf.getvalue() + + +# ----------------------------------------------------------------------------- +# 3. Thread-Safe gRPC Streaming Request Queue +# ----------------------------------------------------------------------------- +class RequestIterator: + """Thread-safe iterator yielding gRPC request messages to the streaming RPC.""" + + def __init__(self): + self._queue = queue.Queue() + self._done = False + + def __iter__(self): + return self + + def __next__(self): + while True: + if self._done and self._queue.empty(): + raise StopIteration + try: + return self._queue.get(timeout=0.1) + except queue.Empty: + if self._done: + raise StopIteration + continue + + def put_dict(self, msg_dict: dict[str, Any]) -> None: + """Converts a JSON-compatible Python dict into a gRPC request packet.""" + client_msg = generative_service.BidiGenerateContentClientMessage() + json_format.ParseDict(msg_dict, client_msg._pb, ignore_unknown_fields=True) + serialized_bytes = ( + generative_service.BidiGenerateContentClientMessage.serialize( + client_msg + ) + ) + request = modelserving_pb2.RoboticsBidiGenerateContentRequest( + message=serialized_bytes + ) + self._queue.put(request) + + def close(self) -> None: + self._done = True + + +# ----------------------------------------------------------------------------- +# 4. Main Inference Function +# ----------------------------------------------------------------------------- +def run_er2_inference( + api_key: str, + model: str, + prompt: str, + effort_level: float = 0.75, + timeout_seconds: float = 15.0, +): + """Establishes gRPC stream with ER2 model, sends inputs, and prints response. + + Args: + api_key: Gemini API Key for authentication. + model: Target ER2 model name. + prompt: User prompt string. + effort_level: Silent thinking effort budget (0.0 to 1.0). + timeout_seconds: Max seconds to wait for model response completion. + """ + model_name = model if model.startswith("models/") else f"models/{model}" + server_addr = "dns:///roboticsdeveloper.googleapis.com:443" + + logger.info("Connecting to Robotics Developer API at %s...", server_addr) + channel_creds = grpc.ssl_channel_credentials() + channel = grpc.secure_channel( + server_addr, + channel_creds, + options=[("grpc.service_config_disable_resolution", 1)], + ) + stub = modelserving_pb2_grpc.ModelServingStub(channel) + + request_iterator = RequestIterator() + + # Step A: Construct session setup message + system_instruction = ( + f"SPECIAL INSTRUCTION: think silently if needed. EFFORT LEVEL:" + f" {effort_level:.2f}." + ) + + # Function declaration for robot instruction execution + tools = [{ + "functionDeclarations": [{ + "name": "run_instruction", + "description": ( + "Execute a natural language action/instruction for the robot." + ), + "parameters": { + "type": "OBJECT", + "properties": { + "instruction": { + "type": "STRING", + "description": ( + "Atomic action instruction (e.g. 'pick up the blue" + " cube')" + ), + } + }, + "required": ["instruction"], + }, + }] + }] + + setup_msg = { + "setup": { + "model": model_name, + "systemInstruction": { + "role": "system", + "parts": [{"text": system_instruction}], + }, + "generationConfig": {"responseModalities": ["TEXT"]}, + "tools": tools, + } + } + + # Step B: Generate synthetic camera frame + image_bytes = generate_random_image_bytes(320, 240) + image_b64 = base64.b64encode(image_bytes).decode("utf-8") + logger.info("Generated random synthetic JPEG image (%d bytes)", len(image_bytes)) + + realtime_image_msg = { + "realtimeInput": { + "video": { + "mimeType": "image/jpeg", + "data": image_b64, + } + } + } + + # Step C: Construct user text & image turn + user_turn_msg = { + "clientContent": { + "turns": [{ + "role": "user", + "parts": [ + {"inlineData": {"mimeType": "image/jpeg", "data": image_b64}}, + {"text": prompt}, + ], + }], + "turnComplete": True, + } + } + + # Enqueue requests: setup -> image -> user turn + request_iterator.put_dict(setup_msg) + request_iterator.put_dict(realtime_image_msg) + request_iterator.put_dict(user_turn_msg) + + metadata = [("x-goog-api-key", api_key)] + logger.info("Sending setup, synthetic image frame, and prompt: %r", prompt) + + try: + grpc_stream = stub.RoboticsBidiGenerateContent( + request_iterator, metadata=metadata + ) + except grpc.RpcError as e: + logger.error("Failed to initiate gRPC stream: %s (%s)", e.details(), e.code()) + return + + # Listener thread to print incoming model responses + def response_listener(): + try: + for response in grpc_stream: + server_msg = ( + generative_service.BidiGenerateContentServerMessage.deserialize( + response.message + ) + ) + msg_dict = json_format.MessageToDict( + server_msg._pb, preserving_proto_field_name=False + ) + print("\n--- Incoming Model Response Message ---") + print(json.dumps(msg_dict, indent=2)) + + # Log parsed tool calls if present + if "toolCall" in msg_dict: + calls = msg_dict["toolCall"].get("functionCalls", []) + for fc in calls: + logger.info(">>> ER2 Model Tool Call: %s(%s)", fc.get("name"), fc.get("args")) + + if msg_dict.get("serverContent", {}).get("turnComplete"): + logger.info("Turn completed by ER2 model.") + break + except grpc.RpcError as e: + logger.error("gRPC stream error: code=%s, details=%s", e.code(), e.details()) + except Exception as e: + logger.exception("Error in response listener: %s", e) + + listener_thread = threading.Thread(target=response_listener, daemon=True) + listener_thread.start() + + # Wait for turn completion or timeout + start_time = time.time() + while listener_thread.is_alive() and (time.time() - start_time < timeout_seconds): + time.sleep(0.2) + + request_iterator.close() + listener_thread.join(timeout=2.0) + logger.info("Finished ER2 inference session.") + + +# ----------------------------------------------------------------------------- +# 5. CLI Entry Point +# ----------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser( + description="Inference ER2 model via Google Robotics Developer API (gRPC)" + ) + parser.add_argument( + "--api_key", + default=os.getenv("GEMINI_API_KEY", ""), + help="Gemini API Key (or set GEMINI_API_KEY env var)", + ) + parser.add_argument( + "--model", + default="robotics_er_live_text_only_2p0_no_safety_classifiers", + help="ER2 model name", + ) + parser.add_argument( + "--prompt", + default="Pick up the blue cube and place it on the red tray.", + help="User input prompt", + ) + parser.add_argument( + "--effort_level", + type=float, + default=0.75, + help="Thinking effort level (0.0 to 1.0)", + ) + args = parser.parse_args() + + if not args.api_key: + sys.stderr.write( + "Error: Gemini API Key is required. Provide --api_key or set" + " GEMINI_API_KEY environment variable.\n" + ) + sys.exit(1) + + run_er2_inference( + api_key=args.api_key, + model=args.model, + prompt=args.prompt, + effort_level=args.effort_level, + ) + + +if __name__ == "__main__": + main() diff --git a/live-api/agent/server.py b/live-api/agent/server.py new file mode 100644 index 0000000..1cdbf7e --- /dev/null +++ b/live-api/agent/server.py @@ -0,0 +1,832 @@ +# /// script +# dependencies = [ +# "fastapi", +# "uvicorn", +# "websockets", +# "websocket-client", +# "httpx", +# "numpy", +# "pillow", +# "protobuf", +# "grpcio", +# "google-ai-generativelanguage", +# ] +# /// + +r"""Proactive Agent — FastAPI server. + +This is the main entry point for the agent. It provides: + - WebSocket /ws endpoint for bidirectional Gemini Live API sessions + - Static UI serving + - Minimal REST API for camera proxy and inter-agent messaging + +No external frameworks, SSOT, or internal RPC infrastructure. +""" + +import argparse +import asyncio +import base64 +import dataclasses +import json +import logging +import os +import re +import signal +import tempfile +import time +import zipfile + +import fastapi +from fastapi import responses as fastapi_responses +from fastapi import staticfiles as fastapi_staticfiles +from fastapi.middleware import cors as cors_middleware +import uvicorn + +import session_config +import session_manager +from agent import agent as agent_lib +from embodiment import human as human_embodiment_lib +from embodiment.spot import robot_client as spot_robot_client_lib +from embodiment.spot import spot_embodiment as spot_embodiment_lib +from embodiment.tinybot import tinybot_embodiment as tinybot_embodiment_lib +from model import tts_client as tts_client_lib +from tool import tools as tools_lib + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Server configuration +# --------------------------------------------------------------------------- +@dataclasses.dataclass +class ServerConfig: + """Server-level configuration from CLI flags.""" + + model: str = "models/gemini-robotics-er-2-streaming-preview" + robot_url: str = "http://localhost:8888" + api_key: str | None = None + tts_api_key: str | None = None + use_tts: bool = True + tts_voice: str = "en-US-Chirp3-HD-Puck" + tts_language_code: str = "en-US" + tts_audio_gain: float = 1.0 + response_modality: str = "AUDIO" + dump_video_dir: str = "" + heartbeat_interval_seconds: float = 2.0 + heartbeat_min_delay_seconds: float = 2.0 + heartbeat_enabled: bool = True + use_event_driven_heartbeat: bool = True + media_resolution: str = "low" + enable_send_message_to_user: bool = True + mock_robot: bool = False + agent_peers: dict[str, str] = dataclasses.field(default_factory=dict) + custom_si: dict[str, str] = dataclasses.field(default_factory=dict) + custom_di: dict[str, str] = dataclasses.field(default_factory=dict) + custom_heartbeat_text: dict[str, str] = dataclasses.field( + default_factory=dict + ) + port: int = 8000 + + +# --------------------------------------------------------------------------- +# UI directory resolution (handles .par packaging) +# --------------------------------------------------------------------------- +def _resolve_ui_dir(): + """Find the ui/ directory.""" + this_dir = os.path.dirname(os.path.abspath(__file__)) + candidate = os.path.join(this_dir, "ui") + if os.path.isdir(candidate): + logger.info("Found UI directory: %s", candidate) + return candidate + return None + +# --------------------------------------------------------------------------- +# Thinking level resolution +# --------------------------------------------------------------------------- +def resolve_thinking_level( + model_name: str, + session_override: str | None = None, +) -> str: + """Resolve thinking level: session override > model default > global default.""" + if session_override is not None: + return session_override + normalized = model_name.removeprefix("models/") + spec = session_config.KNOWN_MODELS.get( + normalized, session_config.DEFAULT_MODEL_SPEC + ) + return spec.thinking_level + + +# --------------------------------------------------------------------------- +# Type conversion utility for query parameter parsing +# --------------------------------------------------------------------------- +def _convert_type(value: str, type_hint) -> object: + """Convert a string value to the expected type hint.""" + if type_hint is bool or type_hint == "bool": + return value.lower() in ("true", "1", "yes") + if type_hint is int or type_hint == "int": + return int(value) + if type_hint is float or type_hint == "float": + return float(value) + return value + + +# --------------------------------------------------------------------------- +# FastAPI app setup +# --------------------------------------------------------------------------- +router = fastapi.APIRouter() + + +@router.get("/") +async def root(request: fastapi.Request): + ui_dir = request.app.state.ui_dir + if not ui_dir: + return fastapi_responses.PlainTextResponse("UI not found") + return fastapi_responses.FileResponse(os.path.join(ui_dir, "index.html")) + + +@router.get("/api/camera") +async def api_camera(request: fastapi.Request): + """Return the latest camera frame as MJPEG stream (robot camera proxy).""" + poller = request.app.state.active_poller_ref + if not poller: + return fastapi.Response(status_code=204) + + async def _mjpeg_gen(): + async for frame in poller.get_stream(): + yield ( + b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + frame + b"\r\n" + ) + + return fastapi.responses.StreamingResponse( + _mjpeg_gen(), + media_type="multipart/x-mixed-replace; boundary=frame", + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, + ) + + +@router.post("/api/send") +async def api_send(request: fastapi.Request): + """Inject text into the active session (inter-agent messaging).""" + body = await request.json() + text = body.get("text", "") + active_session = request.app.state.active_session + if not active_session or not text: + return fastapi.responses.JSONResponse( + status_code=400, content={"error": "No active session or empty text"} + ) + await active_session.get_text_queue().put(text) + return {"status": "ok"} + + +@router.get("/api/server_defaults") +async def api_server_defaults(request: fastapi.Request): + """Return available models and server defaults for the UI.""" + config = request.app.state.config + return { + "models": list(session_config.KNOWN_MODELS.keys()), + "default_model": config.model, + "default_modality": config.response_modality, + "use_tts": config.use_tts, + } + + +def _validate_agent_name(name: object) -> str: + if not isinstance(name, str) or not name.strip(): + raise fastapi.HTTPException(status_code=400, detail="agent_name is required") + name = name.strip() + try: + agent_lib.Agent.from_name(name) + except ValueError as exc: + raise fastapi.HTTPException(status_code=400, detail=str(exc)) from exc + return name + + +def _set_optional_override( + overrides: dict[str, str], + agent_name: str, + value: object, +) -> None: + if value is None or value == "": + overrides.pop(agent_name, None) + return + if not isinstance(value, str): + raise fastapi.HTTPException(status_code=400, detail="Instruction values must be strings") + overrides[agent_name] = value + + +@router.post("/api/config/instructions") +async def update_instructions(request: fastapi.Request): + """Update runtime instructions used when the next agent session starts.""" + body = await request.json() + agent_name = _validate_agent_name(body.get("agent_name")) + config = request.app.state.config + + _set_optional_override( + config.custom_si, agent_name, body.get("system_instruction") + ) + _set_optional_override( + config.custom_di, agent_name, body.get("developer_instruction") + ) + _set_optional_override( + config.custom_heartbeat_text, agent_name, body.get("heartbeat_text") + ) + + for field in ("heartbeat_interval_seconds", "heartbeat_min_delay_seconds"): + value = body.get(field) + if value is None: + continue + try: + value = float(value) + except (TypeError, ValueError) as exc: + raise fastapi.HTTPException( + status_code=400, detail=f"{field} must be a number" + ) from exc + if value <= 0: + raise fastapi.HTTPException( + status_code=400, detail=f"{field} must be greater than zero" + ) + setattr(config, field, value) + + if "use_event_driven_heartbeat" in body: + value = body["use_event_driven_heartbeat"] + if not isinstance(value, bool): + raise fastapi.HTTPException( + status_code=400, + detail="use_event_driven_heartbeat must be a boolean", + ) + config.use_event_driven_heartbeat = value + + return { + "success": True, + "agent_name": agent_name, + "requires_reconnect": request.app.state.active_session is not None, + } + + +@router.delete("/api/config/instructions") +async def reset_instructions(request: fastapi.Request, agent_name: str): + """Reset one agent's overrides and heartbeat settings to startup values.""" + agent_name = _validate_agent_name(agent_name) + config = request.app.state.config + config.custom_si.pop(agent_name, None) + config.custom_di.pop(agent_name, None) + config.custom_heartbeat_text.pop(agent_name, None) + + defaults = request.app.state.instruction_defaults + config.heartbeat_interval_seconds = defaults["heartbeat_interval_seconds"] + config.heartbeat_min_delay_seconds = defaults["heartbeat_min_delay_seconds"] + config.use_event_driven_heartbeat = defaults["use_event_driven_heartbeat"] + return { + "success": True, + "agent_name": agent_name, + "requires_reconnect": request.app.state.active_session is not None, + } + + +@router.get("/api/agent_config/{name}") +async def api_agent_config( + request: fastapi.Request, + name: str, + endpoint_type: str = "gemini_live_api", + model: str | None = None, +): + """Return the resolved agent configuration.""" + config = request.app.state.config + try: + agent = agent_lib.Agent.from_name(name) + agent_si = config.custom_si.get(name, "") + agent_di = config.custom_di.get(name, "") + agent_hb = ( + config.custom_heartbeat_text.get(name, "") + or session_manager.get_default_heartbeat_text() + ) + if agent_si: + agent.system_instruction = agent_si + if agent_di: + agent.developer_instruction = agent_di + except ValueError: + raise fastapi.HTTPException( + status_code=400, detail=f"Unknown agent name: {name}" + ) + + resolved_tools = agent.tools + + + decls = [] + for t in resolved_tools: + if "functionDeclarations" in t: + decls.extend(t["functionDeclarations"]) + else: + decls.append(t) + + service_address = "generativelanguage.googleapis.com" + + return { + "name": agent.name, + "system_instruction": agent.system_instruction, + "developer_instruction": agent.developer_instruction, + "is_modified": bool( + agent_si or agent_di or config.custom_heartbeat_text.get(name, "") + ), + "tools": decls, + "model": config.model, + "response_modality": config.response_modality, + "use_tts": config.use_tts, + "tts_voice": config.tts_voice, + "tts_language_code": config.tts_language_code, + "endpoint_type": endpoint_type, + "service_address": service_address, + "heartbeat_text": agent_hb, + "heartbeat_interval_seconds": config.heartbeat_interval_seconds, + "heartbeat_min_delay_seconds": config.heartbeat_min_delay_seconds, + "use_event_driven_heartbeat": config.use_event_driven_heartbeat, + "thinking_level": resolve_thinking_level(model or config.model), + } + + +# --------------------------------------------------------------------------- +# WebSocket endpoint — the heart of the system +# --------------------------------------------------------------------------- +@router.websocket("/ws") +async def websocket_endpoint( + websocket: fastapi.WebSocket, + agent_name: str = "human", + custom_si: str = "", + enabled_tools: str = "", + response_modality: str | None = None, + model: str | None = None, + use_tts: bool | None = None, + thinking_level: str | None = None, + endpoint_type: str = "gemini_live_api", +): + """WebSocket: Proactive Agent session with Gemini Live API.""" + app = websocket.app + config = app.state.config + + await websocket.accept() + logger.info("WebSocket connected") + + async def ui_callback(event): + await websocket.send_json(event) + + async with app.state.session_lock: + if app.state.active_session is not None: + await websocket.close( + code=1008, reason="Only one active session allowed" + ) + return + + if agent_name == "spot": + current_embodiment = spot_embodiment_lib.SpotEmbodiment( + robot_url=config.robot_url + ) + await current_embodiment.initialize() + app.state.active_poller_ref = current_embodiment.poller + elif agent_name == "tinybot": + current_embodiment = tinybot_embodiment_lib.TinybotEmbodiment( + robot_url=config.robot_url + ) + app.state.active_poller_ref = current_embodiment.poller + else: + # Default to human (local webcam/mic) embodiment + current_embodiment = human_embodiment_lib.HumanEmbodiment() + + # Build SessionConfig from query params + session_cfg = session_config.SessionConfig( + agent_name=agent_name, + custom_si=custom_si, + enabled_tools=enabled_tools.split(",") if enabled_tools else [], + response_modality=response_modality, + model=model or config.model, + use_tts=use_tts, + thinking_level=thinking_level, + endpoint_type=endpoint_type, + ) + logger.info("[SERVER] Resolved session config: %s", session_cfg) + + # Resolve agent + custom_si_override = session_cfg.custom_si or config.custom_si.get( + session_cfg.agent_name, "" + ) + custom_di_override = session_cfg.custom_di or config.custom_di.get( + session_cfg.agent_name, "" + ) + heartbeat_text_override = ( + session_cfg.heartbeat_text + or config.custom_heartbeat_text.get(session_cfg.agent_name, "") + ) + resolved_model = session_cfg.model or config.model + assert resolved_model is not None + + resolved_thinking_level = resolve_thinking_level( + resolved_model, session_cfg.thinking_level + ) + agent = agent_lib.Agent.from_name( + session_cfg.agent_name, + ) + if custom_si_override: + agent.system_instruction = custom_si_override + if custom_di_override: + agent.developer_instruction = custom_di_override + + # Determine tools (agent tools, optionally filtered) + agent_tools = ( + current_embodiment.get_tools() + if agent_name in ("spot", "tinybot") + else agent.tools + ) + if session_cfg.enabled_tools: + enabled = set(session_cfg.enabled_tools) + agent_tools = [ + { + **group, + "functionDeclarations": [ + declaration + for declaration in group.get("functionDeclarations", []) + if declaration.get("name") in enabled + ], + } + for group in agent_tools + if group.get("functionDeclarations") + ] + agent_tools = [ + group for group in agent_tools if group["functionDeclarations"] + ] + + modality = session_cfg.response_modality or config.response_modality + resolved_use_tts = ( + session_cfg.use_tts + if session_cfg.use_tts is not None + else config.use_tts + ) + + logger.info( + "Session: agent=%s, model=%s, modality=%s, tts=%s, thinking_level=%s", + session_cfg.agent_name, + resolved_model, + modality, + resolved_use_tts, + resolved_thinking_level, + ) + + app.state.active_session = session_manager.SessionManager( + model=resolved_model, + embodiment_instance=current_embodiment, + tools=agent_tools, + system_instruction=agent.system_instruction, + developer_instruction=agent.developer_instruction, + response_modality=modality, + dump_video_dir=config.dump_video_dir or None, + api_key=config.api_key, + heartbeat_interval_seconds=config.heartbeat_interval_seconds, + heartbeat_enabled=config.heartbeat_enabled, + heartbeat_min_delay_seconds=config.heartbeat_min_delay_seconds, + agent_peers=config.agent_peers, + peer_name=session_cfg.agent_name, + use_event_driven_heartbeat=config.use_event_driven_heartbeat, + media_resolution=config.media_resolution, + heartbeat_text=heartbeat_text_override, + enable_send_message_to_user=config.enable_send_message_to_user, + endpoint_type=session_cfg.endpoint_type, + thinking_level=resolved_thinking_level, + ) + app.state.session_agent_name = session_cfg.agent_name + + try: + + async def audio_output(data): + try: + await websocket.send_bytes(data) + except Exception: # pylint: disable=broad-except + logger.warning("Failed to send audio output to WebSocket") + + # TTS callback + tts_client = None + text_output_cb = None + if resolved_use_tts: + tts_client = tts_client_lib.Tts3pClient( + voice_name=config.tts_voice, + language_code=config.tts_language_code, + api_key=config.tts_api_key, + audio_gain=config.tts_audio_gain, + ) + logger.info("TTS client initialized with voice: %s", config.tts_voice) + + async def text_output(text): + filtered_text = re.sub(r"```.*?```", "", text, flags=re.DOTALL) + if not filtered_text.strip(): + return + + start_sec = time.time() + logger.info("Synthesizing TTS for: %.50s", filtered_text) + total_bytes = 0 + first_chunk_sent = False + t_start = 0.0 + try: + async for chunk in tts_client.synthesize_stream(filtered_text): + if not first_chunk_sent: + t_start = time.time() + first_chunk_sent = True + if chunk: + await audio_output(chunk) + total_bytes += len(chunk) + except Exception as e: + logger.error("TTS streaming failed: %s", e) + + if total_bytes > 0 and t_start > 0.0: + total_playback_secs = total_bytes / 48000.0 + elapsed_secs = time.time() - t_start + remaining_secs = max(0.0, total_playback_secs - elapsed_secs) + await asyncio.sleep(remaining_secs) + + text_output_cb = text_output + + async def receive_from_client(): + """Read audio/text/image from browser WebSocket.""" + try: + while True: + message = await websocket.receive() + if message.get("type") == "websocket.disconnect": + logger.info("WebSocket disconnect event received") + break + if message.get("bytes"): + await current_embodiment.get_audio_queue().put(message["bytes"]) + elif message.get("text"): + text = message["text"] + try: + payload = json.loads(text) + if isinstance(payload, dict): + if payload.get("type") == "image": + data = base64.b64decode(payload["data"]) + await current_embodiment.get_video_queue().put(data) + continue + except json.JSONDecodeError: + pass + await current_embodiment.get_text_queue().put(text) + except fastapi.WebSocketDisconnect: + logger.info("WebSocket disconnected") + except asyncio.CancelledError: + pass + except Exception as e: # pylint: disable=broad-except + logger.error("Receive error: %s", e) + + async def run_session(): + try: + assert app.state.active_session is not None + async for event in app.state.active_session.start_session( + audio_output_callback=audio_output, + text_output_callback=text_output_cb, + ): + if event: + success = True + try: + await websocket.send_json(event) + except (fastapi.WebSocketDisconnect, RuntimeError): + logger.info("WebSocket disconnected during send_json") + success = False + + if not success: + break + except Exception as e: + logger.error("Error in run_session: %s", e) + + receive_task = asyncio.create_task(receive_from_client()) + session_task = asyncio.create_task(run_session()) + + done, pending = await asyncio.wait( + [session_task, receive_task], + return_when=asyncio.FIRST_COMPLETED, + ) + for t in pending: + t.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + except Exception as e: # pylint: disable=broad-except + import traceback + traceback.print_exc() + logger.error("Session error: %s", e, exc_info=True) + finally: + app.state.active_poller_ref = None + try: + await current_embodiment.close() + except Exception as exc: # pylint: disable=broad-except + logger.warning("Failed to close embodiment: %s", exc) + async with app.state.session_lock: + app.state.active_session = None + try: + await websocket.close() + except Exception: # pylint: disable=broad-except + pass + + +# --------------------------------------------------------------------------- +# App factory +# --------------------------------------------------------------------------- +def create_app(config: ServerConfig) -> fastapi.FastAPI: + """Creates a FastAPI app instance with the given configuration.""" + ui_dir = _resolve_ui_dir() + + application = fastapi.FastAPI() + + @application.on_event("startup") + def startup_event(): + # Force our package loggers to INFO and ensure they write to stdout + loggers = [ + logging.getLogger(), + logging.getLogger("__main__"), + ] + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + ch.setFormatter(formatter) + for l in loggers: + l.setLevel(logging.INFO) + l.addHandler(ch) + l.propagate = False # Avoid duplicate logs if root logger starts working + + application.state.config = config + application.state.ui_dir = ui_dir + application.state.active_poller_ref = None + application.state.active_session = None + application.state.session_agent_name = None + application.state.session_lock = asyncio.Lock() + application.state.instruction_defaults = { + "heartbeat_interval_seconds": config.heartbeat_interval_seconds, + "heartbeat_min_delay_seconds": config.heartbeat_min_delay_seconds, + "use_event_driven_heartbeat": config.use_event_driven_heartbeat, + } + + application.add_middleware( + cors_middleware.CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + if ui_dir: + application.mount( + "/static", + fastapi_staticfiles.StaticFiles(directory=ui_dir, follow_symlink=True), + name="static", + ) + logger.info("UI mounted from %s", ui_dir) + else: + logger.warning("UI directory not found.") + + application.include_router(router) + return application + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(description="Proactive Agent Server") + parser.add_argument( + "--model", default="models/gemini-robotics-er-2-streaming-preview", help="Model name." + ) + parser.add_argument( + "--robot_url", default="http://localhost:8888", help="Robot URL." + ) + parser.add_argument( + "--api_key", + default=os.getenv("GEMINI_API_KEY"), + help="Gemini API key (or set GEMINI_API_KEY env var).", + ) + parser.add_argument( + "--tts_api_key", + default=os.getenv("TTS_API_KEY"), + help="TTS API key (or set TTS_API_KEY env var).", + ) + parser.add_argument( + "--use_tts", action="store_true", default=True, help="Enable TTS." + ) + parser.add_argument("--no_tts", dest="use_tts", action="store_false") + parser.add_argument( + "--tts_voice", + default="en-US-Chirp3-HD-Puck", + help="TTS voice name.", + ) + parser.add_argument( + "--tts_language_code", default="en-US", help="TTS language code." + ) + parser.add_argument( + "--tts_audio_gain", type=float, default=1.0, help="TTS audio gain." + ) + parser.add_argument( + "--response_modality", default="AUDIO", help="Response modality." + ) + parser.add_argument("--dump_video_dir", default="", help="Dump video dir.") + parser.add_argument("--port", type=int, default=8000, help="Server port.") + parser.add_argument( + "--heartbeat_interval_seconds", + type=float, + default=2.0, + help="Heartbeat interval.", + ) + parser.add_argument( + "--heartbeat_min_delay_seconds", + type=float, + default=2.0, + help="Min heartbeat delay.", + ) + parser.add_argument( + "--heartbeat_enabled", action="store_true", default=True + ) + parser.add_argument( + "--no_heartbeat", dest="heartbeat_enabled", action="store_false" + ) + parser.add_argument( + "--use_event_driven_heartbeat", action="store_true", default=True + ) + parser.add_argument( + "--media_resolution", + default="low", + choices=["low", "medium", "high", "ultra_high"], + ) + parser.add_argument( + "--enable_send_message_to_user", action="store_true", default=True + ) + parser.add_argument( + "--mock_robot", action="store_true", default=False, help="Mock robot." + ) + parser.add_argument( + "--agent_peers", + default="", + help="Comma-separated peers: name=url,name=url", + ) + + args = parser.parse_args() + + # Try to load API key from file if not provided + api_key = args.api_key + if not api_key: + for p in [ + os.path.expanduser("~/.config/gemini/API_KEY"), + os.path.expanduser("~/.config/safari_sdk/API_KEY"), + ]: + if os.path.isfile(p): + with open(p, "r") as f: + api_key = f.read().strip() + logger.info("Loaded API key from %s", p) + break + + # Parse agent peers + agent_peers = {} + if args.agent_peers: + for pair in args.agent_peers.split(","): + pair = pair.strip() + if "=" in pair: + name, url = pair.split("=", 1) + agent_peers[name.strip()] = url.strip() + + config = ServerConfig( + model=args.model, + robot_url=args.robot_url, + api_key=api_key, + tts_api_key=args.tts_api_key, + use_tts=args.use_tts, + tts_voice=args.tts_voice, + tts_language_code=args.tts_language_code, + tts_audio_gain=args.tts_audio_gain, + response_modality=args.response_modality, + dump_video_dir=args.dump_video_dir, + heartbeat_interval_seconds=args.heartbeat_interval_seconds, + heartbeat_min_delay_seconds=args.heartbeat_min_delay_seconds, + heartbeat_enabled=args.heartbeat_enabled, + use_event_driven_heartbeat=args.use_event_driven_heartbeat, + media_resolution=args.media_resolution, + enable_send_message_to_user=args.enable_send_message_to_user, + mock_robot=args.mock_robot, + agent_peers=agent_peers, + port=args.port, + ) + + logger.info("Server config: %s", config) + app_instance = create_app(config) + + uvicorn_config = uvicorn.Config( + app_instance, host="0.0.0.0", port=config.port, log_level="info" + ) + server = uvicorn.Server(uvicorn_config) + + def _handle_shutdown(sig, frame): + del sig, frame + logger.info("Received shutdown signal, stopping server...") + server.should_exit = True + + signal.signal(signal.SIGINT, _handle_shutdown) + signal.signal(signal.SIGTERM, _handle_shutdown) + + server.run() + + +if __name__ == "__main__": + main() diff --git a/live-api/agent/session_config.py b/live-api/agent/session_config.py new file mode 100644 index 0000000..e0f3a76 --- /dev/null +++ b/live-api/agent/session_config.py @@ -0,0 +1,103 @@ +"""Session configuration and model specification for Proactive Agent. + +This module defines the per-session configuration and model abstraction layer. +ThinkingMode, ModelSpec, and KNOWN_MODELS live here because thinking mode is a +session-specific concern, not a server-level one. +""" + +import dataclasses +import enum +import logging + +logger = logging.getLogger(__name__) + + +class EndpointType(enum.Enum): + GEMINI_LIVE_API = "gemini_live_api" + + +# Supported thinking levels for the Gemini API. +# "none" disables thinking (no thinkingConfig sent). +THINKING_LEVELS = ("none", "minimal", "low", "medium", "high") +DEFAULT_THINKING_LEVEL = "low" + + + +@dataclasses.dataclass +class ModelSpec: + """Specification for a model and its default attributes. + + Wraps the model name together with model-specific defaults (e.g. thinking + budget). Use KNOWN_MODELS to register defaults for specific models. Users can + still override individual attributes via CLI flags or ORCA eval query params. + """ + + name: str + thinking_level: str = DEFAULT_THINKING_LEVEL + # Future attributes: temperature, max_output_tokens, etc. + + +# Registry of known models and their default attributes. +# Only public Gemini Live API models are included for Lite. +# thinking_level: "none" = disabled, "minimal"/"low"/"medium"/"high" = API thinking levels. +KNOWN_MODELS: dict[str, ModelSpec] = { + "gemini-3.1-flash-live-preview": ModelSpec( + name="gemini-3.1-flash-live-preview", + thinking_level="low", + ), + "gemini-robotics-er-2-streaming-preview": ModelSpec( + name="gemini-robotics-er-2-streaming-preview", + thinking_level="low", + ), + "robotics_er_live_text_only_2p0_no_safety_classifiers": ModelSpec( + name="robotics_er_live_text_only_2p0_no_safety_classifiers", + thinking_level="low", + ), + "robotics_er_live_text_only_2p0_2_no_safety_classifiers": ModelSpec( + name="robotics_er_live_text_only_2p0_2_no_safety_classifiers", + thinking_level="low", + ), + "robotics_er_live_text_only_optimized": ModelSpec( + name="robotics_er_live_text_only_optimized", + thinking_level="low", + ), +} + +# Fallback spec for unknown models. +DEFAULT_MODEL_SPEC = ModelSpec(name="default", thinking_level=DEFAULT_THINKING_LEVEL) + + +@dataclasses.dataclass +class SessionConfig: + """Configuration for a specific session/agent instance at runtime.""" + + # Agent name (e.g., "human", "apollo", "custom") + agent_name: str = "human" + + # Custom system instruction (used if agent_name is "custom") + custom_si: str = "" + + # Custom developer instruction + custom_di: str = "" + + # List of tool names to enable. If empty, all tools from ServerConfig are enabled. + enabled_tools: list[str] = dataclasses.field(default_factory=list) + + # Response modality override (AUDIO or TEXT). If None, use ServerConfig default. + response_modality: str | None = None + + # Model override. If None, use ServerConfig default. + model: str | None = None + + # Enable TTS override. If None, use ServerConfig default. + use_tts: bool | None = None + + # Thinking level override. None = use model default from KNOWN_MODELS. + # "none" = disabled, "minimal"/"low"/"medium"/"high" = API thinking levels. + thinking_level: str | None = None + + # Heartbeat text override. If None, use ServerConfig default or built-in default. + heartbeat_text: str | None = None + + # Endpoint type (gemini_live_api) + endpoint_type: str = "gemini_live_api" diff --git a/live-api/agent/session_config_test.py b/live-api/agent/session_config_test.py new file mode 100644 index 0000000..5ce0a3f --- /dev/null +++ b/live-api/agent/session_config_test.py @@ -0,0 +1,58 @@ +"""Tests for thinking level resolution.""" + +import unittest + +import server +import session_config + + +class ThinkingLevelResolutionTest(unittest.TestCase): + + def test_model_default_level_is_used(self): + for model_name, spec in session_config.KNOWN_MODELS.items(): + with self.subTest(model=model_name): + self.assertEqual( + spec.thinking_level, + server.resolve_thinking_level(model_name), + ) + + def test_session_override_takes_precedence(self): + model = next(iter(session_config.KNOWN_MODELS)) + override = "high" + self.assertEqual( + override, + server.resolve_thinking_level(model, session_override=override), + ) + + def test_unknown_model_gets_global_default(self): + self.assertEqual( + session_config.DEFAULT_THINKING_LEVEL, + server.resolve_thinking_level("some-unknown-model"), + ) + + def test_low_level_is_default(self): + """Ensure 'low' is the default thinking level.""" + model = next(iter(session_config.KNOWN_MODELS)) + self.assertEqual( + "low", + server.resolve_thinking_level(model), + ) + + def test_models_prefix_stripped(self): + """Ensure models/ prefix is stripped for lookup.""" + model = next(iter(session_config.KNOWN_MODELS)) + self.assertEqual( + session_config.KNOWN_MODELS[model].thinking_level, + server.resolve_thinking_level(f"models/{model}"), + ) + + def test_all_levels_are_valid(self): + """All THINKING_LEVELS should be accepted.""" + for level in session_config.THINKING_LEVELS: + with self.subTest(level=level): + result = server.resolve_thinking_level("any-model", session_override=level) + self.assertEqual(level, result) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/agent/session_manager.py b/live-api/agent/session_manager.py new file mode 100644 index 0000000..0a170c7 --- /dev/null +++ b/live-api/agent/session_manager.py @@ -0,0 +1,730 @@ +r"""Manages a Gemini Live bidirectional streaming session (Lite version). + +This is a simplified version of the session manager that uses only +the public Gemini Live API (WebSocket + JSON) with zero external dependencies. +""" + +import asyncio +import base64 +import datetime +import json +import logging +import time +from typing import Any, Mapping + +from core import audio_handler +from core import decision_making +from core import event_bus +from core import observation +from core import tool_call_handler +from model import live_api_client + + +logger = logging.getLogger(__name__) + + +def get_default_heartbeat_text() -> str: + """Returns the default heartbeat text to send to the model.""" + return ( + "[HEARTBEAT] If no task is active, call 'ack' and wait for user" + " input. If a task is active: observe the scene. If the current" + " step is progressing correctly, call 'ack'. If the current step" + " is complete, call 'run_instruction' with the next step. If the" + " overall goal is achieved, call 'reset' and inform the user." + ) + + +class SessionManager: + """Manages a Gemini Live bidirectional streaming session. + + Uses an EventBus for typed pub/sub event dispatching. Components register + as self-subscribing handlers; the session manager bridges UI-relevant + events to the WebSocket via a ui_queue. + """ + + _DEFAULT_SYSTEM_INSTRUCTION = ( + "You are a helpful AI assistant. Keep your responses" + " concise. You can see the user's camera or screen" + " which is shared as realtime input images with you." + ) + + def __init__( + self, + model: str, + embodiment_instance, + input_sample_rate: int = 16000, + tools=None, + system_instruction: str | None = None, + developer_instruction: str | None = None, + response_modality: str = "AUDIO", + dump_video_dir: str | None = None, + api_key: str | None = None, + heartbeat_interval_seconds: float = 2.0, + heartbeat_enabled: bool = True, + heartbeat_min_delay_seconds: float = 0.0, + agent_peers: dict[str, str] | None = None, + peer_name: str = "unknown", + use_event_driven_heartbeat: bool = False, + media_resolution: str = "low", + heartbeat_text: str = "", + enable_send_message_to_user: bool = False, + endpoint_type: str = "gemini_live_api", + thinking_level: str = "none", + ): + self.model = model + self.embodiment = embodiment_instance + self.input_sample_rate = input_sample_rate + self.tools = tools or [] + self.system_instruction = ( + system_instruction or self._DEFAULT_SYSTEM_INSTRUCTION + ) + self.developer_instruction = developer_instruction or "" + self.thinking_level = thinking_level + self.response_modality = response_modality + self.dump_video_dir = dump_video_dir + self.heartbeat_interval_seconds = heartbeat_interval_seconds + self.heartbeat_enabled = heartbeat_enabled + self.heartbeat_min_delay_seconds = heartbeat_min_delay_seconds + self._agent_peers = agent_peers or {} + self._peer_name = peer_name + self.use_event_driven_heartbeat = use_event_driven_heartbeat + self.heartbeat_text = heartbeat_text + self._enable_send_message_to_user = enable_send_message_to_user + self._last_heartbeat_sent_time = 0.0 + self._interrupted = False + + self._heartbeat_sent_count = 0 + self._turn_completed_count = 0 + self._heartbeat_in_flight = False + + self.api_key = api_key + self._endpoint_type = endpoint_type + + # Create the API client. + self.client = live_api_client.GeminiLiveApiClient(api_key=api_key) + self.stream = None + self._stream_lock = asyncio.Lock() + self.loop = asyncio.get_running_loop() + + # Create the event bus. + self.bus = event_bus.EventBus() + + # Initialize Observation early to expose queues. + self.observation = observation.Observation( + self.bus, + self.loop, + self, # SessionManager provides send_message for stream I/O + 0, # Session start time not available yet + self.embodiment, + self.input_sample_rate, + self.dump_video_dir, + media_resolution=media_resolution, + ) + self.decision_making = None + self.tool_handler = None + self.audio_handler = None + + def _get_heartbeat_text(self) -> str: + if self.heartbeat_text: + return self.heartbeat_text + return get_default_heartbeat_text() + + def set_embodiment(self, embodiment): + """Updates the embodiment dynamically.""" + self.embodiment = embodiment + self.observation.set_embodiment(embodiment) + if self.tool_handler: + self.tool_handler.set_embodiment(embodiment) + + async def send_message(self, msg: Any) -> None: + """Send a JSON message to the Gemini Live API stream. + + Serializes all writes through an asyncio.Lock to prevent concurrent + stream.Send calls from different tasks. + """ + assert self.stream is not None, "Stream is not initialized" + async with self._stream_lock: + await asyncio.get_running_loop().run_in_executor( + None, self.stream.Send, msg + ) + + async def send_text_with_fresh_video(self, text: str) -> None: + """Atomically send a current Spot frame followed by user text.""" + poller = getattr(self.embodiment, "poller", None) + chunk = b"" + if poller is not None: + try: + chunk = await asyncio.wait_for( + poller.wait_for_next_frame(), timeout=2.0 + ) + except asyncio.TimeoutError: + logger.warning("Timed out waiting for a pre-text camera frame") + + messages = [] + if chunk: + await self.observation.dump_frame(chunk) + messages.append({ + "realtimeInput": { + "video": { + "mimeType": "image/jpeg", + "data": base64.b64encode(chunk).decode("utf-8"), + } + } + }) + messages.append({ + "clientContent": { + "turns": [{ + "role": "user", + "parts": [{"text": text}], + }], + "turnComplete": True, + } + }) + + assert self.stream is not None, "Stream is not initialized" + async with self._stream_lock: + for message in messages: + await asyncio.get_running_loop().run_in_executor( + None, self.stream.Send, message + ) + if chunk: + logger.info("Sent synchronized fresh video frame with user text") + + async def send_latest_video_frame(self, timeout: float = 2.0) -> bool: + """Send the first poller frame captured after this method is called.""" + poller = getattr(self.embodiment, "poller", None) + if poller is None: + return False + try: + chunk = await asyncio.wait_for( + poller.wait_for_next_frame(), timeout=timeout + ) + except asyncio.TimeoutError: + logger.warning("Timed out waiting for a post-tool camera frame") + return False + if not chunk: + logger.warning("Camera poller stopped before producing a post-tool frame") + return False + + await self.observation.dump_frame(chunk) + await self.send_message({ + "realtimeInput": { + "video": { + "mimeType": "image/jpeg", + "data": base64.b64encode(chunk).decode("utf-8"), + } + } + }) + logger.info("Sent synchronized fresh video frame") + return True + + def get_audio_queue(self) -> asyncio.Queue: + return self.observation.get_audio_queue() + + def get_video_queue(self) -> asyncio.Queue: + return self.observation.get_video_queue() + + def get_text_queue(self) -> asyncio.Queue: + return self.observation.get_text_queue() + + def _on_message(self, msg): + """Callback for incoming Gemini stream messages (JSON dicts).""" + if msg is not None: + logger.debug("Received message: %s", msg) + self.loop.call_soon_threadsafe( + self.bus.publish_nowait, + event_bus.Event( + type=event_bus.EventType.MODEL_RESPONSE, + source=event_bus.EventSource.ASSISTANT, + data=msg, + ), + ) + + def _on_done(self): + """Callback when the Gemini stream ends.""" + logger.error("[Session] on_done fired — stream disconnected") + self.loop.call_soon_threadsafe( + self.bus.publish_nowait, + event_bus.Event( + type=event_bus.EventType.SESSION_DONE, + source=event_bus.EventSource.ASSISTANT, + ), + ) + + async def _send_setup(self) -> None: + """Sends the JSON setup message to the Live API stream.""" + model_name = self.model + if not model_name.startswith("models/"): + model_name = f"models/{model_name}" + + # Combine SI and DI into a single system instruction. + combined_prompt = self.system_instruction + if self.developer_instruction: + combined_prompt += "\\n\\n" + self.developer_instruction + + modality_str = self.response_modality.upper() + if modality_str == "AUDIO": + modality = "AUDIO" + elif modality_str == "TEXT": + modality = "TEXT" + else: + raise ValueError(f"Unsupported response modality: {self.response_modality}") + + setup_dict = { + "setup": { + "model": model_name, + "systemInstruction": { + "role": "system", + "parts": [{"text": combined_prompt}] + }, + "generationConfig": { + "responseModalities": [modality] + }, + "tools": [] + } + } + + # Add thinking config if a level is set. + if self.thinking_level and self.thinking_level != "none": + setup_dict["setup"]["generationConfig"]["thinkingConfig"] = { + "thinkingLevel": self.thinking_level.upper(), + } + + # Add tools. + for tool in self.tools: + setup_dict["setup"]["tools"].append(tool) + + logger.info( + "[Session] Sending setup: model=%s, modality=%s, tools=%d", + model_name, + modality_str, + len(self.tools), + ) + + assert self.stream is not None + await asyncio.get_running_loop().run_in_executor( + None, self.stream.Send, setup_dict + ) + + async def _reconnect(self, max_retries: int = 3) -> bool: + """Attempts to reconnect the Gemini stream with exponential backoff.""" + import time + now = time.monotonic() + last_rc = getattr(self, '_last_reconnect_time', 0) + rapid_count = getattr(self, '_rapid_reconnect_count', 0) + + if now - last_rc < 5.0: + rapid_count += 1 + else: + rapid_count = 0 + + self._last_reconnect_time = now + self._rapid_reconnect_count = rapid_count + + if rapid_count >= max_retries: + logger.error("Stream disconnecting too rapidly (%d times). Aborting.", rapid_count) + return False + + for attempt in range(max_retries): + wait_time = 2**attempt + logger.warning( + "Stream disconnected. Reconnecting in %ds (attempt %d/%d)...", + wait_time, + attempt + 1, + max_retries, + ) + await asyncio.sleep(wait_time) + try: + self.stream = self.client.create_stream() + self.stream.Start( + lambda msg: self._on_message(msg), + lambda: self._on_done(), + ) + await self._send_setup() + logger.info("Reconnected successfully on attempt %d.", attempt + 1) + return True + except Exception as e: # pylint: disable=broad-except + logger.error("Reconnection attempt %d failed: %s", attempt + 1, e) + logger.error("All %d reconnection attempts failed.", max_retries) + return False + + async def start_session( + self, + audio_output_callback, + audio_interrupt_callback=None, + text_output_callback=None, + ): + self.session_start_time = self.loop.time() + self.observation.session_start_time = self.session_start_time + + # --- Create the Gemini stream --- + self.stream = self.client.create_stream() + self.stream.Start( + lambda msg: self._on_message(msg), + lambda: self._on_done(), + ) + + # --- Register handlers on the bus --- + + # DecisionMaking: routes MODEL_RESPONSE → typed events + self.decision_making = decision_making.DecisionMaking( + self.bus, + text_output_callback=text_output_callback, + audio_interrupt_callback=audio_interrupt_callback, + ) + + # AudioResponseHandler: accumulates AUDIO_CHUNK → AUDIO_RESPONSE + self.audio_handler = audio_handler.AudioResponseHandler( + self.bus, + audio_output_callback=audio_output_callback, + ) + + # ToolCallHandler: executes TOOL_CALL → TOOL_RESULT + self.tool_handler = tool_call_handler.ToolCallHandler( + self.bus, + self, # session_manager for send_message + self.embodiment, + agent_peers=self._agent_peers, + peer_name=self._peer_name, + text_output_callback=text_output_callback, + enable_send_message_to_user=self._enable_send_message_to_user, + blocking_tools=tool_call_handler.blocking_tool_names(self.tools), + ) + + # --- UI queue bridge: events → async generator yield --- + + ui_queue: asyncio.Queue = asyncio.Queue() + + _UI_EVENT_TYPES = [ + event_bus.EventType.GEMINI_TEXT, + event_bus.EventType.GEMINI_THOUGHT, + event_bus.EventType.USER_TRANSCRIPT, + event_bus.EventType.AUDIO_RESPONSE, + event_bus.EventType.TOOL_RESULT, + event_bus.EventType.TURN_COMPLETE, + event_bus.EventType.TELEMETRY, + event_bus.EventType.INTERRUPTED, + event_bus.EventType.ERROR, + event_bus.EventType.LOG, + event_bus.EventType.SESSION_DONE, + event_bus.EventType.TEXT_INPUT, + event_bus.EventType.TRANSPARENT_HISTORY, + ] + + async def _ui_handler(event: event_bus.Event): + await ui_queue.put(event) + + self.bus.subscribe(_UI_EVENT_TYPES, _ui_handler) + + # --- Send Setup Message --- + + await self._send_setup() + + # --- Start input tasks and heartbeat --- + + has_poller = getattr(self.embodiment, "poller", None) is not None + self.observation.start_input_tasks( + skip_video=self.use_event_driven_heartbeat and has_poller + ) + + heartbeat_trigger = None + if self.use_event_driven_heartbeat: + heartbeat_signal = asyncio.Event() + _trigger_source = [""] + + async def _on_interrupt(event: event_bus.Event): + self._interrupted = True + + async def _on_heartbeat_trigger(event: event_bus.Event): + if event.type == event_bus.EventType.TURN_COMPLETE: + self._heartbeat_in_flight = False + if self._interrupted: + self._interrupted = False + return + + now = asyncio.get_running_loop().time() + if now - self._last_heartbeat_sent_time < 0.5: + return + + if event.type == event_bus.EventType.TURN_COMPLETE: + had_tool_call = ( + event.data.get("had_tool_call", False) + if isinstance(event.data, dict) + else False + ) + if had_tool_call: + return + + source = ( + event.data.get("source", event.type.value) + if event.data and isinstance(event.data, dict) + else event.type.value + ) + _trigger_source[0] = source + heartbeat_signal.set() + + if self.heartbeat_enabled: + self.bus.subscribe( + [ + event_bus.EventType.TURN_COMPLETE, + event_bus.EventType.HEARTBEAT_TRIGGER, + ], + _on_heartbeat_trigger, + ) + self.bus.subscribe( + [event_bus.EventType.INTERRUPTED], + _on_interrupt, + ) + + async def _heartbeat_loop(): + try: + _trigger_source[0] = "session_start" + heartbeat_signal.set() + + while True: + try: + await asyncio.wait_for( + heartbeat_signal.wait(), timeout=10.0 + ) + trigger = _trigger_source[0] + except asyncio.TimeoutError: + trigger = "safety_timeout" + logger.warning( + "No model response within 10s, sending recovery heartbeat" + ) + + try: + if ( + self.tool_handler is not None + and self.tool_handler.tool_executing.is_set() + ): + heartbeat_signal.clear() + logger.info( + "Suppressing %s heartbeat while blocking tool executes", + trigger, + ) + continue + + if has_poller: + chunk = await self.observation.get_video_queue().get() + video_msg = { + "realtimeInput": { + "video": { + "mimeType": "image/jpeg", + "data": base64.b64encode(chunk).decode("utf-8"), + } + } + } + await self.observation.dump_frame(chunk) + await self.send_message(video_msg) + logger.info("Sent video frame with heartbeat") + else: + try: + await asyncio.wait_for( + self.observation.wait_for_next_frame(), timeout=2.0 + ) + except asyncio.TimeoutError: + logger.warning( + "Timeout waiting for webcam frame, proceeding" + ) + + heartbeat_signal.clear() + + self._heartbeat_sent_count += 1 + self._heartbeat_in_flight = True + heartbeat_msg = { + "clientContent": { + "turns": [{ + "role": "user", + "parts": [{"text": self._get_heartbeat_text()}], + }], + "turnComplete": True, + } + } + await self.send_message(heartbeat_msg) + self._last_heartbeat_sent_time = ( + asyncio.get_running_loop().time() + ) + logger.info("Sent heartbeat (trigger=%s)", trigger) + await ui_queue.put({"type": "heartbeat_sent"}) + except Exception as e: + logger.error("Heartbeat send failed: %s", e) + await asyncio.sleep(1.0) + except asyncio.CancelledError: + pass + except Exception as e: + logger.error("Heartbeat loop failed: %s", e) + + heartbeat_task = ( + asyncio.create_task(_heartbeat_loop()) + if self.heartbeat_enabled + else None + ) + + else: + # Legacy fixed-interval heartbeat with turn_complete chaining. + heartbeat_trigger = asyncio.Event() + + async def _heartbeat_loop(): + try: + await asyncio.sleep(2.0) + while True: + if ( + self.tool_handler is not None + and self.tool_handler.tool_executing.is_set() + ): + await asyncio.sleep(0.5) + continue + while ( + self.decision_making is not None + and self.decision_making.speaking.is_set() + ): + await asyncio.sleep(1.0) + + heartbeat_msg = { + "clientContent": { + "turns": [{ + "role": "user", + "parts": [{"text": self._get_heartbeat_text()}], + }], + "turnComplete": True, + } + } + heartbeat_trigger.clear() + await self.send_message(heartbeat_msg) + hb_send_time = time.monotonic() + logger.info("Sent heartbeat") + self._heartbeat_sent_count += 1 + await ui_queue.put({"type": "heartbeat_sent"}) + try: + await asyncio.wait_for( + heartbeat_trigger.wait(), + timeout=self.heartbeat_interval_seconds, + ) + except asyncio.TimeoutError: + pass + elapsed = time.monotonic() - hb_send_time + remaining = self.heartbeat_min_delay_seconds - elapsed + if remaining > 0: + await asyncio.sleep(remaining) + except asyncio.CancelledError: + pass + except Exception as e: + logger.error("Heartbeat loop failed: %s", e) + + heartbeat_task = ( + asyncio.create_task(_heartbeat_loop()) + if self.heartbeat_enabled + else None + ) + + # --- Start the event bus --- + + self.bus.start() + + # --- Main loop: drain UI queue and yield to WebSocket --- + + try: + while True: + event = await ui_queue.get() + + if isinstance(event, dict): + yield event + continue + + if event.type == event_bus.EventType.SESSION_DONE: + if await self._reconnect(): + continue + break + + if event.type == event_bus.EventType.TURN_COMPLETE: + self._turn_completed_count += 1 + + # Suppress heartbeat-caused INTERRUPTED events. + if self._enable_send_message_to_user: + _suppress = ( + event.type == event_bus.EventType.INTERRUPTED + and event.source == event_bus.EventSource.ASSISTANT + and self._heartbeat_sent_count > self._turn_completed_count + and not ( + isinstance(event.data, dict) + and event.data.get("tts_preempt") + ) + ) + else: + _suppress = ( + event.type == event_bus.EventType.INTERRUPTED + and self._heartbeat_in_flight + and not ( + isinstance(event.data, dict) + and event.data.get("tts_preempt") + ) + ) + if _suppress: + logger.info("Suppressing INTERRUPTED — heartbeat-caused") + continue + + ui_event = self._to_ui_event(event) + yield ui_event + if ( + ui_event.get("type") == "turn_complete" + and not self.use_event_driven_heartbeat + ): + if heartbeat_trigger is not None: + heartbeat_trigger.set() + + finally: + if heartbeat_task: + heartbeat_task.cancel() + await self.observation.stop_input_tasks() + self.observation.close() + await self.bus.shutdown() + if self.stream: + try: + self.stream.Shutdown() + except Exception: # pylint: disable=broad-except + pass + self.stream = None + + def _to_ui_event(self, event: event_bus.Event) -> Mapping[str, Any]: + """Convert a typed Event to the UI dict format for the WebSocket.""" + + if event.type == event_bus.EventType.GEMINI_TEXT: + return {"type": "gemini", "text": event.data.get("text", "")} + elif event.type == event_bus.EventType.GEMINI_THOUGHT: + return {"type": "gemini_thought", "text": event.data.get("text", "")} + elif event.type == event_bus.EventType.TEXT_INPUT: + return {"type": "text_input", "text": event.data} + elif event.type == event_bus.EventType.USER_TRANSCRIPT: + return {"type": "user_transcript", "text": event.data.get("text", "")} + elif event.type == event_bus.EventType.AUDIO_RESPONSE: + return { + "type": "audio_response", + "data": event.data.get("audio_data", ""), + } + elif event.type == event_bus.EventType.TOOL_RESULT: + return event.data # Already a dict + elif event.type == event_bus.EventType.TURN_COMPLETE: + d: dict[str, Any] = {"type": "turn_complete"} + if isinstance(event.data, dict) and "had_tool_call" in event.data: + d["had_tool_call"] = event.data["had_tool_call"] + return d + elif event.type == event_bus.EventType.TELEMETRY: + d: dict[str, Any] = {"type": "telemetry"} + if isinstance(event.data, dict): + for key in ["token_usage", "server_ttft_ms"]: + if key in event.data: + d[key] = event.data[key] + return d + elif event.type == event_bus.EventType.INTERRUPTED: + return {"type": "interrupted"} + elif event.type == event_bus.EventType.ERROR: + return {"type": "error", "error": event.data} + elif event.type == event_bus.EventType.TRANSPARENT_HISTORY: + # In Lite, transparent history data is already a dict (no proto). + return {"type": "transparent_history", "data": event.data} + elif event.type == event_bus.EventType.LOG: + return {"type": "log", "data": event.data} + else: + logger.warning("Unknown UI event type: %s", event.type) + return {"type": "unknown", "data": str(event.data)} diff --git a/live-api/agent/setup.sh b/live-api/agent/setup.sh new file mode 100755 index 0000000..6559b2b --- /dev/null +++ b/live-api/agent/setup.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# setup.sh - Setup virtual environment and install dependencies for physical-agent + +set -e + +# Get the directory of the script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR" + +VENV_DIR=".venv" + +echo "=== Physical Agent Setup ===" + +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment in $VENV_DIR..." + python3 -m venv "$VENV_DIR" +else + echo "Virtual environment already exists." +fi + +echo "Activating virtual environment..." +source "$VENV_DIR"/bin/activate + +echo "Upgrading pip..." +pip install --index-url https://pypi.org/simple --upgrade pip + +echo "Installing setuptools..." +pip install --index-url https://pypi.org/simple "setuptools>=61.0.0" + +echo "Installing physical-agent in editable mode with dependencies..." +pip install --index-url https://pypi.org/simple --no-build-isolation -e . + +echo "=== Setup Complete ===" +echo "To activate the virtual environment, run:" +echo "source .venv/bin/activate" diff --git a/live-api/agent/tool/__init__.py b/live-api/agent/tool/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/live-api/agent/tool/__init__.py @@ -0,0 +1 @@ + diff --git a/live-api/agent/tool/tools.py b/live-api/agent/tool/tools.py new file mode 100644 index 0000000..e7bb931 --- /dev/null +++ b/live-api/agent/tool/tools.py @@ -0,0 +1,855 @@ +"""Tool declarations for Proactive Agent. + +All tools are declared as plain Python dicts matching the Gemini API JSON +format. No protobuf dependencies. + +Tool sets return raw dicts matching the Gemini API. + +Typical usage: + + from tool import tools + + my_tools = tools.robot_tools() +""" + +from typing import Any + + +# --------------------------------------------------------------------------- +# Individual tool declarations +# --------------------------------------------------------------------------- + + +def _tool( + name: str, + description: str, + parameters: dict[str, Any], + behavior: str | None = None, +) -> dict[str, Any]: + """Helper to build a single FunctionDeclaration dict.""" + d: dict[str, Any] = { + "name": name, + "description": description, + "parameters": parameters, + } + if behavior is not None: + d["behavior"] = behavior + return d + + +def run_instruction_tool() -> dict[str, Any]: + """Tool to send a natural language instruction to the robot.""" + return _tool( + name="run_instruction", + description=( + "When you call this tool, the robot will start executing the" + " instruction. The robot continues executing until you call" + " `run_instruction` again with a new instruction. Observe the scene" + " visually to determine if the current step is complete." + ), + parameters={ + "type": "OBJECT", + "properties": { + "instruction": { + "type": "STRING", + "description": ( + "A specific, unambiguous, atomic (involving single" + " action) natural language instruction for the robot" + " (e.g. 'bring the cup to the table', 'put the dice in" + " the green tray'). This instruction should be specific" + " enough to describe the exact object, e.g. 'the white" + " cup on the center of the table'. Always use 'pick and" + " place' instead of just 'pick'. Always give concrete" + " from and to locations. It probably should not contain" + " the word 'and' except in 'pick and place'." + ), + }, + }, + "required": ["instruction"], + }, + behavior="BLOCKING", + ) + + +def draw_points_tool() -> dict[str, Any]: + """Tool to draw labeled point overlays on the camera image.""" + return _tool( + name="draw_points", + description=( + "Draw one or more labeled blue point overlays on the camera image" + " to highlight points or objects in the scene. Coordinates are" + " integers 0-1000 where (0,0) is top-left and (1000,1000) is" + " bottom-right." + ), + parameters={ + "type": "OBJECT", + "properties": { + "points": { + "type": "ARRAY", + "description": "List of points to draw.", + "items": { + "type": "OBJECT", + "properties": { + "x": { + "type": "NUMBER", + "description": ( + "Horizontal position, 0-1000" + " (0=left, 1000=right)." + ), + }, + "y": { + "type": "NUMBER", + "description": ( + "Vertical position, 0-1000" + " (0=top, 1000=bottom)." + ), + }, + "label": { + "type": "STRING", + "description": ( + "Text label displayed next to the point." + ), + }, + }, + "required": ["x", "y"], + }, + }, + }, + "required": ["points"], + }, + behavior="NON_BLOCKING", + ) + + +def stop_tool() -> dict[str, Any]: + """Tool to stop all robot motion immediately.""" + return _tool( + name="stop", + description="Stop all robot motion immediately.", + parameters={"type": "OBJECT", "properties": {}}, + ) + + +def ack_tool() -> dict[str, Any]: + """Tool to acknowledge the heartbeat without taking action.""" + return _tool( + name="ack", + description=( + "Call this tool to acknowledge the regular heartbeat and indicate" + " that no new intervention is needed. This is the default action when" + " no other specific action is required." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def make_gesture_tool() -> dict[str, Any]: + """Tool to make the robot perform a gesture.""" + return _tool( + name="make_gesture", + description="Make the robot perform a gesture.", + parameters={ + "type": "OBJECT", + "properties": { + "gesture": { + "type": "STRING", + "description": ( + "The name of the gesture to perform. Valid gestures" + " are: 'wave', 'thinking_pose', 'dance_move'," + " 'alive', 'left_hand_wave', 'right_hand_wave'," + " 'idle_nod', 'idle_hands', 'greeting_crowd'," + " 'look_right', 'addresses_audience'," + " 'addressing_audience_on_the_left'," + " 'addressing_the_audience_on_the_right'," + " 'agrees_with_someone', 'arms_wide_open'," + " 'both_arms_in_one_direction'," + " 'bringing_hands_together'," + " 'calming_audience_on_the_left'," + " 'calming_down_audience'," + " 'came_up_with_an_idea', 'celebrates'," + " 'comes_up_with_an_idea', 'disappointed'," + " 'greets_with_both_hands', 'hands_up'," + " 'has_a_question', 'is_afraid_of_something'," + " 'is_demanding_sth', 'looks_around'," + " 'looks_around_starts_by_turning_its_head_left'," + " 'looks_down', 'looks_up_raises_hands'," + " 'makes_a_point', 'open_arm_welcoming'," + " 'opens_its_arms_looking_at_the_scene_in_front_of_it'," + " 'pointing_to_the_left_and_explaining_sth'," + " 'points_at_something'," + " 'points_somewhere_and_agrees'," + " 'raises_hand_and_agrees', 'goodbye'," + " 'hello', 'hello_1', 'no'," + " 'showing_scary_move', 'thinks_about_sth'," + " 'thinks_how_to_solve_a_problem'," + " 'thumbs_up'," + " 'turns_its_head_right_and_notices_sth'," + " 'turns_left_and_then_pays_attention_to_sth_in_center'," + " 'victory_sign'," + " 'want_audience_to_calm_down'," + " 'wants_audience_to_calm_down'." + ), + }, + }, + "required": ["gesture"], + }, + behavior="NON_BLOCKING", + ) + + +def send_message_tool( + enable_send_message_to_user: bool = False, +) -> dict[str, Any]: + """Tool to send a message to another robot agent (or the user).""" + if enable_send_message_to_user: + desc = ( + "Send a text message to the user or another robot agent in the fleet." + " Use this to delegate tasks or coordinate with other robots or to" + " talk to the user." + ) + target_desc = ( + "The target robot agent name (e.g. 'duo', 'apollo', 'user')." + ) + else: + desc = ( + "Send a text message to another robot agent in the fleet." + " Use this to delegate tasks or coordinate with other robots." + ) + target_desc = "The target robot agent name (e.g. 'duo', 'apollo')." + + return _tool( + name="send_message", + description=desc, + parameters={ + "type": "OBJECT", + "properties": { + "target": { + "type": "STRING", + "description": target_desc, + }, + "message": { + "type": "STRING", + "description": ( + "The message to send. Can be a task instruction, " + "status update, or coordination signal. Do not send " + "emojis in the message." + ), + }, + }, + "required": ["target", "message"], + }, + behavior="NON_BLOCKING", + ) + + +def get_calendar_tool() -> dict[str, Any]: + """Tool to retrieve upcoming events from the shared calendar.""" + return _tool( + name="get_calendar", + description=( + "Retrieve upcoming events from the shared family calendar. Use this" + " to check for planned activities." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def turn_head_to_uv_tool() -> dict[str, Any]: + """Tool to turn the robot head to a normalized pixel coordinate.""" + return _tool( + name="turn_head_to_uv", + description=( + "Turn the robot head to look at a specific point in the current" + " camera view. Takes two arguments: `u` (horizontal, 0=left, 1=right)" + " and `v` (vertical, 0=top, 1=bottom). Both are normalized pixel" + " coordinates between 0 and 1." + ), + parameters={ + "type": "OBJECT", + "properties": { + "u": { + "type": "NUMBER", + "description": ( + "Horizontal pixel coordinate, normalized between 0 and" + " 1. 0 is the left edge, 1 is the right edge." + ), + }, + "v": { + "type": "NUMBER", + "description": ( + "Vertical pixel coordinate, normalized between 0 and" + " 1. 0 is the top edge, 1 is the bottom edge." + ), + }, + }, + "required": ["u", "v"], + }, + behavior="NON_BLOCKING", + ) + + +# --------------------------------------------------------------------------- +# Tool sets — return lists of dicts wrapped in {"functionDeclarations": [...]} +# --------------------------------------------------------------------------- + +def _wrap(decls: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Wrap a list of function declarations in the Gemini API format.""" + return [{"functionDeclarations": decls}] + + +def robot_tools() -> list[dict[str, Any]]: + """Motion control tools: run_instruction, stop, ack.""" + return _wrap([run_instruction_tool(), stop_tool(), ack_tool()]) + + +def human_tools() -> list[dict[str, Any]]: + """Human/local mode tools: run_instruction, stop, ack.""" + return _wrap([ + run_instruction_tool(), + stop_tool(), + ack_tool(), + send_message_tool(enable_send_message_to_user=True), + ]) + + +def all_tools() -> list[dict[str, Any]]: + """All available tools (robot + UI + gesture + ack).""" + return _wrap([ + run_instruction_tool(), + draw_points_tool(), + stop_tool(), + ack_tool(), + make_gesture_tool(), + ]) + + +def collab_tools() -> list[dict[str, Any]]: + """Robot + collaboration tools for multi-agent coordination.""" + return _wrap([ + run_instruction_tool(), + stop_tool(), + ack_tool(), + send_message_tool(), + ]) + + +def navigate_tool() -> dict[str, Any]: + """Tool to navigate to a named waypoint.""" + return _tool( + name="navigate", + description=( + "Navigate the robot to a named waypoint using GraphNav. The robot" + " will walk to the specified waypoint and report when it arrives." + " Call only after get_waypoints returns navigation_ready=true, use" + " an exact name from that result, and call stow before navigating." + ), + parameters={ + "type": "OBJECT", + "properties": { + "waypoint": { + "type": "STRING", + "description": ( + "Exact waypoint name from the latest successful" + " get_waypoints result. Never invent, normalize, or reuse" + " a name from an earlier session." + ), + }, + }, + "required": ["waypoint"], + }, + behavior="BLOCKING", + ) + + +def drive_tool() -> dict[str, Any]: + """Tool for short body-relative movement without GraphNav waypoints.""" + return _tool( + name="drive", + description=( + "Drive Spot without a waypoint for a short, bounded interval using" + " body-frame velocities. Positive v_x moves forward, positive v_y" + " moves left, and positive v_rot turns counterclockwise. The robot" + " powers on and stands automatically. The current camera-arm joint" + " pose is held relative to the body, so the gripper camera moves" + " with Spot. Use small commands near people, furniture, stairs, or" + " other obstacles." + ), + parameters={ + "type": "OBJECT", + "properties": { + "v_x": { + "type": "NUMBER", + "minimum": -0.8, + "maximum": 0.8, + "description": ( + "Forward velocity in meters per second, from -0.8" + " backward to 0.8 forward." + ), + }, + "v_y": { + "type": "NUMBER", + "minimum": -0.5, + "maximum": 0.5, + "description": ( + "Sideways velocity in meters per second, from -0.5" + " right to 0.5 left." + ), + }, + "v_rot": { + "type": "NUMBER", + "minimum": -1.0, + "maximum": 1.0, + "description": ( + "Yaw velocity in radians per second, from -1.0" + " clockwise to 1.0 counterclockwise." + ), + }, + "duration": { + "type": "NUMBER", + "minimum": 0.1, + "maximum": 2.0, + "description": "How long to drive, in seconds (0.1 to 2.0).", + }, + }, + "required": ["v_x", "v_y", "v_rot", "duration"], + }, + behavior="BLOCKING", + ) + + +def stop_spot_tool() -> dict[str, Any]: + """Tool to immediately stop Spot motion and freeze the arm.""" + return _tool( + name="stop", + description=( + "Immediately cancel base motion and navigation and freeze the arm." + " Call this whenever motion is unsafe or the user asks Spot to stop." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def look_tool() -> dict[str, Any]: + """Tool to aim the gripper camera by rotating only the arm.""" + return _tool( + name="look", + description=( + "Aim Spot's gripper-mounted camera by rotating only the arm; the" + " robot body and feet remain stationary. Use small steps and" + " inspect the fresh camera frame returned after each move." + ), + parameters={ + "type": "OBJECT", + "properties": { + "direction": { + "type": "STRING", + "enum": ["up", "down", "left", "right"], + "description": "Direction to aim the camera view.", + }, + "angle_rad": { + "type": "NUMBER", + "minimum": 0.05, + "maximum": 0.35, + "description": ( + "Positive angular step in radians. Use 0.15 for a" + " small adjustment and 0.3 for a larger adjustment." + ), + }, + }, + "required": ["direction"], + }, + behavior="BLOCKING", + ) + + +def detect_tool() -> dict[str, Any]: + """Detect an object or placement location and store a one-time target.""" + return _tool( + name="detect", + description=( + "Use the Spot backend's Gemini Robotics detector to locate exactly" + " one language-specified object or placement location in the stable" + " hand-camera image." + " When the user asks to pick up an object, assume it is in the" + " current view and call this tool instead of claiming it is not" + " visible without attempting detection." + " The backend returns and stores the grasp location and the UI" + " overlays it. Call only after the camera view has remained stable" + " for at least three seconds. After a correct detection, call pick" + " or place with no arguments before any camera or robot movement." + ), + parameters={ + "type": "OBJECT", + "properties": { + "instruction": { + "type": "STRING", + "description": ( + "A concise description identifying exactly one visible" + ' object, such as "the red cube on the floor".' + ), + }, + }, + "required": ["instruction"], + }, + behavior="BLOCKING", + ) + + +def pick_tool() -> dict[str, Any]: + """Grasp the target stored by the latest detect call.""" + return _tool( + name="pick", + description=( + "Grasp the one-time location stored by the most recent successful" + " detect call. This tool accepts no pixel coordinates. Call it only" + " after confirming the detection overlay is on the requested object" + " and before any robot or camera motion. The target is consumed" + " after one attempt. A completed call still requires visual success" + " verification. The backend reduces the gripper hold torque after" + " the native Spot grasp succeeds." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def deploy_arm_tool() -> dict[str, Any]: + """Deploy arm.""" + return _tool( + name="deploy_arm", + description=( + "Deploy the robot arm from its stowed position. Must be called" + " before any arm or gripper operations." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def stow_arm_tool() -> dict[str, Any]: + """Stow arm.""" + return _tool( + name="stow_arm", + description=( + "Stow the robot arm back into its carry position. Call this before" + " navigating to avoid collisions." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def open_gripper_tool() -> dict[str, Any]: + """Open gripper.""" + return _tool( + name="open_gripper", + description=( + "Open the robot gripper. Optionally specify the fraction to open to." + ), + parameters={ + "type": "OBJECT", + "properties": { + "fraction": { + "type": "NUMBER", + "description": ( + "How far to open (0=closed, 1=fully open). Default is" + " 1.0." + ), + }, + }, + }, + behavior="NON_BLOCKING", + ) + + +def close_gripper_tool() -> dict[str, Any]: + """Close gripper.""" + return _tool( + name="close_gripper", + description="Close the robot gripper to grasp an object.", + parameters={"type": "OBJECT", "properties": {}}, + behavior="NON_BLOCKING", + ) + + +def stand_tool() -> dict[str, Any]: + """Stand up.""" + return _tool( + name="stand", + description=( + "Power on the robot and stand up. Call this to wake the robot from" + " sitting." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def sit_tool() -> dict[str, Any]: + """Sit down.""" + return _tool( + name="sit", + description=( + "Sit the robot down and power off its motors. Call this only when" + " the user's current instruction explicitly asks Spot to sit; never" + " sit automatically when a task ends or Spot becomes idle." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def get_battery_tool() -> dict[str, Any]: + """Get battery.""" + return _tool( + name="get_battery", + description="Check the robot battery level and power state.", + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def get_waypoints_tool() -> dict[str, Any]: + """Get waypoints.""" + return _tool( + name="get_waypoints", + description=( + "List registered navigation destinations and report GraphNav" + " readiness. Use only when the user asks for destinations or" + " requests waypoint navigation; do not call at startup or for" + " manipulation, camera, status, or local-drive tasks. If the user" + " only asks for names, return them without discussing localization." + " For navigation, use a returned name only when navigation_ready is" + " true. If false, explain that GraphNav must be loaded and localized." + " If null or absent, say readiness could not be verified; do not" + " assume localization is missing and do not call navigate." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def health_check_tool() -> dict[str, Any]: + """Tool to check the robot connection status, lease holder state, and battery levels.""" + return _tool( + name="health_check", + description=( + "Check robot connection, lease holder, and battery state. Use when" + " the user asks for status, before physical motion when control" + " state is unknown, or after evidence of a connection, lease, or" + " battery problem. Do not call automatically at startup or while" + " idle." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def place_tool() -> dict[str, Any]: + """Place the held object at the target stored by detect.""" + return _tool( + name="place", + description=( + "Place the currently held object at the one-time location stored by" + " the most recent successful detect call. This tool accepts no" + " coordinates. Call it immediately after detecting the intended" + " placement surface, before any robot or camera movement. The" + " backend moves the hand to the projected 3D target and releases" + " only after arm arrival is confirmed; a failed approach leaves the" + " gripper closed." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def wait_for_pick_up_tool() -> dict[str, Any]: + """Wait for a recipient to lift and take the held object.""" + return _tool( + name="wait_for_pick_up", + description=( + "Wait for a person to pick up the object currently held by Spot." + " Call from carry pose after reaching the recipient. The tool" + " monitors upward hand motion; when the hand rises by the threshold," + " it opens the gripper, waits briefly, closes the gripper, and stows" + " the arm. A timeout leaves the object held." + ), + parameters={ + "type": "OBJECT", + "properties": { + "monitor_sec": { + "type": "NUMBER", + "minimum": 1.0, + "maximum": 120.0, + "description": "Maximum seconds to wait for pickup.", + }, + "upward_threshold_m": { + "type": "NUMBER", + "minimum": 0.005, + "maximum": 0.2, + "description": "Upward hand displacement that triggers release; default 0.02 m.", + }, + "open_duration_sec": { + "type": "NUMBER", + "minimum": 0.1, + "maximum": 10.0, + "description": "Seconds to keep the gripper open; default 3 seconds.", + }, + }, + }, + behavior="BLOCKING", + ) + + +def stow_tool() -> dict[str, Any]: + """Tool to stow the arm safely, choosing carry pose if holding an object.""" + return _tool( + name="stow", + description=( + "Stow the arm safely. If the robot is holding an object, it will" + " move the arm to a stable carry pose to prevent collision. If it" + " is not holding anything, it will stow the arm completely." + ), + parameters={"type": "OBJECT", "properties": {}}, + behavior="BLOCKING", + ) + + +def spot_tools() -> list[dict[str, Any]]: + """Spot navigation + manipulation tools.""" + return _wrap([ + health_check_tool(), + get_waypoints_tool(), + navigate_tool(), + drive_tool(), + stop_spot_tool(), + look_tool(), + detect_tool(), + pick_tool(), + place_tool(), + wait_for_pick_up_tool(), + stand_tool(), + sit_tool(), + stow_tool(), + ack_tool(), + send_message_tool(enable_send_message_to_user=True), + ]) + + +def tinybot_make_gesture_tool() -> dict[str, Any]: + """Tool to make the tinybot perform a gesture.""" + return _tool( + name="make_gesture", + description="Make the tinybot perform a gesture.", + parameters={ + "type": "OBJECT", + "properties": { + "gesture": { + "type": "STRING", + "description": ( + "The name of the gesture to perform. Valid gestures" + " are: 'nod', 'no', 'home', 'home_pose'." + ), + }, + "speed": { + "type": "INTEGER", + "description": "Speed of gesture (default: 150).", + }, + }, + "required": ["gesture"], + }, + behavior="NON_BLOCKING", + ) + + +def move_absolute_tool() -> dict[str, Any]: + """Tool to move tinybot joints to absolute angles.""" + return _tool( + name="move_absolute", + description="Move tinybot joints to absolute angles in degrees.", + parameters={ + "type": "OBJECT", + "properties": { + "angles": { + "type": "ARRAY", + "items": {"type": "NUMBER"}, + "description": "Target joint angles in degrees.", + }, + "speeds": { + "type": "ARRAY", + "items": {"type": "INTEGER"}, + "description": "Movement speeds for each joint.", + }, + "blocking": { + "type": "BOOLEAN", + "description": ( + "Whether to block until movement is complete (default:" + " true)." + ), + }, + }, + "required": ["angles", "speeds"], + }, + behavior="BLOCKING", + ) + + +def move_relative_tool() -> dict[str, Any]: + """Tool to move tinybot joints by relative angles.""" + return _tool( + name="move_relative", + description="Move tinybot joints by relative angles in degrees.", + parameters={ + "type": "OBJECT", + "properties": { + "relative_angles": { + "type": "ARRAY", + "items": {"type": "NUMBER"}, + "description": "Relative angles in degrees.", + }, + "speeds": { + "type": "ARRAY", + "items": {"type": "INTEGER"}, + "description": "Movement speeds for each joint.", + }, + "blocking": { + "type": "BOOLEAN", + "description": ( + "Whether to block until movement is complete (default:" + " true)." + ), + }, + }, + "required": ["relative_angles", "speeds"], + }, + behavior="BLOCKING", + ) + + +def clear_overlay_tool() -> dict[str, Any]: + """Tool to clear point overlays on the camera image.""" + return _tool( + name="clear_overlay", + description="Clear all drawn point overlays from the camera image.", + parameters={"type": "OBJECT", "properties": {}}, + behavior="NON_BLOCKING", + ) + + +def tinybot_tools() -> list[dict[str, Any]]: + """Tinybot motion + drawing tools.""" + return _wrap([ + turn_head_to_uv_tool(), + tinybot_make_gesture_tool(), + move_absolute_tool(), + move_relative_tool(), + draw_points_tool(), + clear_overlay_tool(), + stop_tool(), + ack_tool(), + send_message_tool(enable_send_message_to_user=True), + ]) + diff --git a/live-api/agent/ui/agent-client.js b/live-api/agent/ui/agent-client.js new file mode 100644 index 0000000..6d8f1b1 --- /dev/null +++ b/live-api/agent/ui/agent-client.js @@ -0,0 +1,78 @@ +/** + * GeminiClient: Handles WebSocket communication + */ +class GeminiClient { + constructor(config) { + this.websocket = null; + this.onOpen = config.onOpen; + this.onMessage = config.onMessage; + this.onClose = config.onClose; + this.onError = config.onError; + } + + connect(params = {}) { + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + let wsUrl = `${protocol}//${window.location.host}/ws`; + + // Append query parameters if provided + const queryParts = []; + for (const [key, value] of Object.entries(params)) { + if (value != null && value !== '') { + queryParts.push( + `${encodeURIComponent(key)}=${encodeURIComponent(value)}`); + } + } + if (queryParts.length > 0) { + wsUrl += '?' + queryParts.join('&'); + } + + + this.websocket = new WebSocket(wsUrl); + this.websocket.binaryType = 'arraybuffer'; + + this.websocket.onopen = () => { + if (this.onOpen) this.onOpen(); + }; + + this.websocket.onmessage = (event) => { + if (this.onMessage) this.onMessage(event); + }; + + this.websocket.onclose = (event) => { + if (this.onClose) this.onClose(event); + }; + + this.websocket.onerror = (event) => { + if (this.onError) this.onError(event); + }; + } + + send(data) { + if (this.websocket && this.websocket.readyState === WebSocket.OPEN) { + this.websocket.send(data); + } + } + + sendText(text) { + this.send(text); + } + + sendImage(base64Data, mimeType = 'image/jpeg') { + this.send(JSON.stringify({ + type: 'image', + mime_type: mimeType, + data: base64Data, + })); + } + + disconnect() { + if (this.websocket) { + this.websocket.close(); + this.websocket = null; + } + } + + isConnected() { + return this.websocket && this.websocket.readyState === WebSocket.OPEN; + } +} diff --git a/live-api/agent/ui/agent-ui.js b/live-api/agent/ui/agent-ui.js new file mode 100644 index 0000000..dab0896 --- /dev/null +++ b/live-api/agent/ui/agent-ui.js @@ -0,0 +1,1538 @@ +// --- Main Application Logic (Proactive Agent) --- +let localStorage = window.localStorage; + +let statusLabel = document.getElementById('status-label'); +let micBtn = document.getElementById('micBtn'); +let textInput = document.getElementById('textInput'); +let sendBtn = document.getElementById('sendBtn'); +let videoPreview = document.getElementById('video-preview'); +let videoPlaceholder = document.getElementById('video-placeholder'); +let robotCamImg = document.getElementById('robot-cam-img'); +let connectBtn = document.getElementById('connectBtn'); +let chatLog = document.getElementById('chat-log'); +let overlayCanvas = document.getElementById('overlay-canvas'); +let chatToggle = document.getElementById('chatToggle'); +let chatSidebar = document.getElementById('chat-sidebar'); + +// --- Model Selector & Runtime Configs --- +let modelSelectorBtn = document.getElementById('modelSelectorBtn'); +let modelSelectorLabel = document.getElementById('modelSelectorLabel'); +let modelDropdown = document.getElementById('modelDropdown'); +let customModelModal = document.getElementById('custom-model-modal'); +let customModelInput = document.getElementById('custom-model-input'); +let customModalitySelect = document.getElementById('custom-modality-select'); +let customModelCancel = document.getElementById('custom-model-cancel'); +let customModelApply = document.getElementById('custom-model-apply'); + +// Endpoint Selector DOM +let endpointSelectorBtn = document.getElementById('endpointSelectorBtn'); +let endpointSelectorLabel = document.getElementById('endpointSelectorLabel'); +let endpointDropdown = document.getElementById('endpointDropdown'); + +// Audio Toggle DOM +let audioToggleBtn = document.getElementById('audioToggleBtn'); + +// Agent Selector DOM +let agentSelectorBtn = document.getElementById('agentSelectorBtn'); +let agentSelectorLabel = document.getElementById('agentSelectorLabel'); +let agentDropdown = document.getElementById('agentDropdown'); + +const ENDPOINT_MODELS = { + gemini_live_api: [ + { model: 'models/gemini-robotics-er-2-streaming-preview', label: 'models/gemini-robotics-er-2-streaming-preview', modality: 'TEXT', textOnly: true }, + { model: 'models/gemini-3.1-flash-live-preview', label: 'models/gemini-3.1-flash-live-preview', modality: 'AUDIO', textOnly: false }, + { model: 'models/gemini-2.5-flash-native-audio-latest', label: 'models/gemini-2.5-flash-native-audio-latest', modality: 'AUDIO', textOnly: false } + ] +}; + +let selectedEndpointType = + localStorage.getItem('lite_endpoint_type') || 'gemini_live_api'; + +let selectedModel = + localStorage.getItem('lite_model') || + ENDPOINT_MODELS[selectedEndpointType][0].model; + +let selectedAudioEnabled = localStorage.getItem('lite_audio_enabled') !== + 'false'; // defaults to true +let customBaseModality = + localStorage.getItem('lite_custom_modality') || 'TEXT'; + +// Re-validate selectedModel against available options for the selected endpoint +let availableModels = ENDPOINT_MODELS[selectedEndpointType] || []; +let modelExists = availableModels.some(m => m.model === selectedModel) || selectedModel === 'custom'; +if (!modelExists && availableModels.length > 0) { + selectedModel = availableModels[0].model; + localStorage.setItem('lite_model', selectedModel); +} + +// Populate models dropdown dynamically +function populateModels(endpoint) { + if (!modelDropdown) return; + modelDropdown.innerHTML = ''; + const models = ENDPOINT_MODELS[endpoint] || []; + models.forEach(m => { + const opt = document.createElement('div'); + opt.className = 'model-option'; + if (m.model === selectedModel) { + opt.classList.add('active'); + if (modelSelectorLabel) modelSelectorLabel.textContent = m.label; + } + opt.dataset.model = m.model; + opt.dataset.modality = m.modality; + opt.dataset.textOnly = m.textOnly ? 'true' : 'false'; + opt.textContent = m.label; + bindModelOption(opt); + modelDropdown.appendChild(opt); + }); + + // Add Custom Option + const customOpt = document.createElement('div'); + customOpt.className = 'model-option model-option-custom'; + if (selectedModel === 'custom') { + customOpt.classList.add('active'); + } + customOpt.dataset.model = 'custom'; + customOpt.textContent = 'Custom...'; + bindModelOption(customOpt); + modelDropdown.appendChild(customOpt); +} + +// Binds the click handler to a model option element. +function bindModelOption(opt) { + opt.onclick = () => { + const model = opt.dataset.model; + if (model === 'custom') { + modelDropdown.classList.add('hidden'); + if (customModelInput) customModelInput.value = selectedModel; + if (customModalitySelect) customModalitySelect.value = customBaseModality; + customModelModal.classList.remove('hidden'); + if (customModelInput) customModelInput.focus(); + return; + } + selectedModel = model; + localStorage.setItem('lite_model', selectedModel); + modelSelectorLabel.textContent = opt.textContent; + + modelDropdown.querySelectorAll('.model-option') + .forEach(o => o.classList.remove('active')); + opt.classList.add('active'); + modelDropdown.classList.add('hidden'); + updateAudioUI(); + }; +} + +// Initialize endpoint and model selectors +if (endpointSelectorLabel) { + if (endpointDropdown) { + const activeOpt = Array.from(endpointDropdown.querySelectorAll('.model-option')) + .find(opt => opt.dataset.endpoint === selectedEndpointType); + if (activeOpt) { + endpointSelectorLabel.textContent = activeOpt.textContent; + endpointDropdown.querySelectorAll('.model-option').forEach(o => { + o.classList.toggle('active', o.dataset.endpoint === selectedEndpointType); + }); + } + } +} + +populateModels(selectedEndpointType); + +// Toggle endpoint dropdown +if (endpointSelectorBtn) { + endpointSelectorBtn.onclick = (e) => { + e.stopPropagation(); + endpointDropdown.classList.toggle('hidden'); + }; +} + +// Toggle model dropdown +if (modelSelectorBtn) { + modelSelectorBtn.onclick = (e) => { + e.stopPropagation(); + modelDropdown.classList.toggle('hidden'); + }; +} + +// Handle endpoint selection +if (endpointDropdown) { + endpointDropdown.querySelectorAll('.model-option').forEach(opt => { + opt.onclick = () => { + const ep = opt.dataset.endpoint; + selectedEndpointType = ep; + localStorage.setItem('lite_endpoint_type', ep); + if (endpointSelectorLabel) { + endpointSelectorLabel.textContent = opt.textContent; + } + endpointDropdown.querySelectorAll('.model-option').forEach(o => o.classList.remove('active')); + opt.classList.add('active'); + endpointDropdown.classList.add('hidden'); + + // Refresh model dropdown + populateModels(ep); + // Select the first model as default + const defaultModel = ENDPOINT_MODELS[ep][0].model; + selectModelVisual(defaultModel); + }; + }); +} + +// Close dropdowns on outside click +document.addEventListener('click', (e) => { + if (modelDropdown && !modelDropdown.contains(e.target) && e.target !== modelSelectorBtn) { + modelDropdown.classList.add('hidden'); + } + if (endpointDropdown && !endpointDropdown.contains(e.target) && e.target !== endpointSelectorBtn) { + endpointDropdown.classList.add('hidden'); + } +}); + +// Custom model modal handlers +if (customModelCancel) { + customModelCancel.onclick = () => customModelModal.classList.add('hidden'); +} +if (customModelApply) { + customModelApply.onclick = () => { + const customName = customModelInput.value.trim(); + if (!customName) return; + selectedModel = customName; + localStorage.setItem('lite_model', selectedModel); + modelSelectorLabel.textContent = selectedModel; + if (modelDropdown) { + modelDropdown.querySelectorAll('.model-option') + .forEach(o => o.classList.remove('active')); + } + // Custom modality from select + if (customModalitySelect) { + customBaseModality = customModalitySelect.value; + localStorage.setItem('lite_custom_modality', customBaseModality); + } + customModelModal.classList.add('hidden'); + updateAudioUI(); + }; +} + +// Handle Audio Toggle button clicks +if (audioToggleBtn) { + audioToggleBtn.onclick = () => { + if (audioToggleBtn.disabled) return; + selectedAudioEnabled = !selectedAudioEnabled; + localStorage.setItem('lite_audio_enabled', selectedAudioEnabled); + updateAudioUI(); + }; +} + +function updateAudioUI() { + if (!audioToggleBtn) return; + + const ICON_AUDIO_ON = ''; + const ICON_AUDIO_OFF = ''; + + // Resolve base modality of selected model + let baseModality = 'TEXT'; + if (selectedModel === 'custom') { + baseModality = customBaseModality; + } else if (modelDropdown) { + const selectedOpt = + Array.from(modelDropdown.querySelectorAll('.model-option')) + .find(opt => opt.dataset.model === selectedModel); + if (selectedOpt) { + baseModality = selectedOpt.dataset.modality || 'TEXT'; + } + } + + if (baseModality === 'AUDIO') { + // Natively AUDIO model -> force Speaker to ON and disable it + audioToggleBtn.disabled = true; + audioToggleBtn.classList.add('active'); + audioToggleBtn.innerHTML = ICON_AUDIO_ON; + audioToggleBtn.title = 'Audio is required for this model'; + } else { + // Natively TEXT model -> restore user's toggle state and enable it + audioToggleBtn.disabled = false; + audioToggleBtn.classList.toggle('active', selectedAudioEnabled); + audioToggleBtn.innerHTML = selectedAudioEnabled ? ICON_AUDIO_ON : ICON_AUDIO_OFF; + audioToggleBtn.title = + selectedAudioEnabled ? 'Click to mute audio' : 'Click to unmute audio'; + } +} + +// Run initial update +updateAudioUI(); + +function selectModelVisual(model) { + selectedModel = model; + localStorage.setItem('lite_model', selectedModel); + + if (modelDropdown) { + let activeText = selectedModel; + modelDropdown.querySelectorAll('.model-option') + .forEach(o => { + const isActive = o.dataset.model === selectedModel; + o.classList.toggle('active', isActive); + if (isActive) activeText = o.textContent; + }); + if (modelSelectorLabel) modelSelectorLabel.textContent = activeText; + } else if (modelSelectorLabel) { + modelSelectorLabel.textContent = selectedModel; + } + updateAudioUI(); +} + + + +// Helper to dynamically resolve session configs — always Live API +function getResolvedSessionConfig() { + let baseModality = 'TEXT'; + if (selectedModel === 'custom') { + baseModality = customBaseModality; + } else if (modelDropdown) { + const selectedOpt = + Array.from(modelDropdown.querySelectorAll('.model-option')) + .find(opt => opt.dataset.model === selectedModel); + if (selectedOpt) { + baseModality = selectedOpt.dataset.modality || 'TEXT'; + } + } + + let resolvedModality = 'TEXT'; + let resolvedUseTts = false; + + if (baseModality === 'AUDIO') { + resolvedModality = 'AUDIO'; + resolvedUseTts = selectedAudioEnabled; + } else { + resolvedModality = 'TEXT'; + resolvedUseTts = selectedAudioEnabled; + } + + return { + model: selectedModel, + response_modality: resolvedModality, + use_tts: resolvedUseTts, + endpoint_type: selectedEndpointType + }; +} + +// Fetch server defaults if not in localStorage + +async function initServerDefaults() { + try { + const resp = await fetch('/api/server_defaults'); + if (!resp.ok) throw new Error(); + const defaults = await resp.json(); + // Dynamically rebuild the model dropdown based on available server models + if (defaults.available_models && modelDropdown) { + modelDropdown.innerHTML = ''; + const allKnown = Object.values(ENDPOINT_MODELS).flat(); + defaults.available_models.forEach(modelName => { + const found = allKnown.find(m => m.model === modelName); + const label = found ? found.label : modelName; + const modality = found ? + found.modality : + (modelName.includes('audio') || modelName.includes('live') ? 'AUDIO' : 'TEXT'); + const textOnly = found ? found.textOnly : false; + + const opt = document.createElement('div'); + opt.className = 'model-option'; + opt.dataset.model = modelName; + opt.dataset.modality = modality; + opt.dataset.textOnly = textOnly ? 'true' : 'false'; + opt.textContent = label; + bindModelOption(opt); + modelDropdown.appendChild(opt); + }); + // Add custom option back + const customOpt = document.createElement('div'); + customOpt.className = 'model-option model-option-custom'; + customOpt.dataset.model = 'custom'; + customOpt.innerHTML = 'Custom…'; + bindModelOption(customOpt); + modelDropdown.appendChild(customOpt); + } + + // Always fetch server default model if nothing explicitly preferred by user + // or if the preferred model is not available + const localModel = localStorage.getItem('lite_model'); + if (!localModel || + (defaults.available_models && + !defaults.available_models.includes(localModel))) { + selectedModel = defaults.model; + localStorage.setItem('lite_model', selectedModel); + } else { + selectedModel = localModel; + } + + // Set default audio enabled based on server-configured use_tts + if (!localStorage.getItem('lite_audio_enabled_set')) { + selectedAudioEnabled = defaults.use_tts; + localStorage.setItem('lite_audio_enabled', selectedAudioEnabled); + localStorage.setItem('lite_audio_enabled_set', 'true'); + } + + if (modelDropdown) { + let activeText = selectedModel; + modelDropdown.querySelectorAll('.model-option').forEach(opt => { + const isActive = opt.dataset.model === selectedModel; + opt.classList.toggle('active', isActive); + if (isActive) activeText = opt.textContent; + }); + if (modelSelectorLabel) modelSelectorLabel.textContent = activeText; + } + updateAudioUI(); + } catch (e) { + console.warn('Failed to fetch server defaults, using local defaults'); + } +} +initServerDefaults(); + +let currentGeminiMessageDiv = null; +let currentUserMessageDiv = null; +let currentThinkingContentDiv = null; +let videoSource = 'computer'; // "computer" or "robot" +let toolCallCounter = 0; +let lastAckRow = null; // Reference to the current aggregated ack heartbeat row +let ackCount = 0; // Number of consecutive ack calls in the current row + +// --- Latency Tracking --- +let MAX_HISTORY = 50; // Sliding window for CI computation +let querySendTime = null; +let ttftRecorded = false; +let ttfcRecorded = false; + +// Client-measured latency histories (sliding window, milliseconds). +let ttftHistory = []; // Time to First Token (any model response) +let ttfcHistory = []; // Time to First Function Call (tool invocation) +let ttltHistory = []; // Time to Last Token (turn_complete) +// Server-reported latency breakdown histories (milliseconds). +let srvClientHistory = []; +let srvServerHistory = []; +let srvPrefillHistory = []; +let srvDecodeHistory = []; +let srvTtftHistory = []; +let srvClientTtftHistory = []; + +function pushCapped(arr, val) { + arr.push(val); + if (arr.length > MAX_HISTORY) arr.shift(); +} + +function resetLatencyTimer() { + querySendTime = performance.now(); + ttftRecorded = false; + ttfcRecorded = false; +} + +function computeCI(values) { + const n = values.length; + if (n === 0) return null; + const mean = values.reduce((a, b) => a + b, 0) / n; + if (n === 1) return { mean, lo: mean, hi: mean, n }; + const variance = values.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1); + const stderr = Math.sqrt(variance / n); + const z = 1.96; // 95% CI + return { mean, lo: mean - z * stderr, hi: mean + z * stderr, n }; +} + +function formatCI(ci) { + if (!ci) return '--'; + const m = (ci.mean / 1000).toFixed(2); + if (ci.n === 1) return m + 's'; + const lo = (Math.max(0, ci.lo) / 1000).toFixed(2); + const hi = (ci.hi / 1000).toFixed(2); + return m + 's [' + lo + ', ' + hi + ']'; +} + +function formatCIms(ci) { + if (!ci) return '--'; + const m = Math.round(ci.mean); + if (ci.n === 1) return m + 'ms'; + const lo = Math.round(Math.max(0, ci.lo)); + const hi = Math.round(ci.hi); + return m + 'ms [' + lo + ', ' + hi + ']'; +} + +function recordTTFT() { + if (!ttftRecorded && querySendTime !== null) { + const ttft = performance.now() - querySendTime; + pushCapped(ttftHistory, ttft); + const ci = computeCI(ttftHistory); + const el = document.getElementById('ttft-val'); + const countEl = document.getElementById('ttft-count'); + if (el) el.textContent = formatCI(ci); + if (countEl) countEl.textContent = 'n=' + ci.n; + ttftRecorded = true; + } +} + +function recordTTFC() { + if (!ttfcRecorded && querySendTime !== null) { + const ttfc = performance.now() - querySendTime; + pushCapped(ttfcHistory, ttfc); + const ci = computeCI(ttfcHistory); + const el = document.getElementById('ttfc-val'); + const countEl = document.getElementById('ttfc-count'); + if (el) el.textContent = formatCI(ci); + if (countEl) countEl.textContent = 'n=' + ci.n; + ttfcRecorded = true; + } +} + +function recordTTLT() { + if (querySendTime !== null) { + const ttlt = performance.now() - querySendTime; + pushCapped(ttltHistory, ttlt); + const ci = computeCI(ttltHistory); + const el = document.getElementById('ttlt-val'); + const countEl = document.getElementById('ttlt-count'); + if (el) el.textContent = formatCI(ci); + if (countEl) countEl.textContent = 'n=' + ci.n; + } +} + +// Helper to update a metric's DOM elements. +function updateMetric(prefix, history) { + const ci = computeCI(history); + const el = document.getElementById(prefix + '-val'); + const countEl = document.getElementById(prefix + '-count'); + if (el) el.textContent = ci ? formatCIms(ci) : '--'; + if (countEl) countEl.textContent = ci ? 'n=' + ci.n : ''; +} + +function updateServerLatency(sl, serverTtftMs) { + // Only use client TTFT if it was freshly recorded for the current turn. + const clientTtftMs = ttftRecorded && ttftHistory.length > 0 ? + ttftHistory[ttftHistory.length - 1] : + null; + + // ① Client Overhead = Client TTFT - Server TTFT + if (clientTtftMs !== null && serverTtftMs > 0) { + const val = Math.max(0, clientTtftMs - serverTtftMs); + pushCapped(srvClientHistory, val); + updateMetric('srv-client', srvClientHistory); + } + + // ② Server Overhead = Server TTFT - known inference + if (serverTtftMs > 0 && sl.request_ttft_ms > 0) { + const known = (sl.prefill_queue_ms || 0) + (sl.prefill_ms || 0) + + (sl.decode_queue_ms || 0) + (sl.decode_ttft_ms || 0); + const val = Math.max(0, serverTtftMs - known); + pushCapped(srvServerHistory, val); + updateMetric('srv-server', srvServerHistory); + } + + // ③ Prefill + if (sl.prefill_ms > 0) { + pushCapped(srvPrefillHistory, sl.prefill_ms); + updateMetric('srv-prefill', srvPrefillHistory); + } + + // ④ Decode + if (sl.decode_ttft_ms > 0) { + pushCapped(srvDecodeHistory, sl.decode_ttft_ms); + updateMetric('srv-decode', srvDecodeHistory); + } + + // Reference: Srv TTFT + if (sl.request_ttft_ms > 0) { + pushCapped(srvTtftHistory, sl.request_ttft_ms); + updateMetric('srv-ttft', srvTtftHistory); + } + + // Reference: Client TTFT + if (clientTtftMs !== null) { + pushCapped(srvClientTtftHistory, clientTtftMs); + updateMetric('srv-client-ttft', srvClientTtftHistory); + } +} + +function formatTokenCount(n) { + if (n >= 1000) return (n / 1000).toFixed(1) + 'K'; + return String(n); +} + +function updateTokenUsage(tu) { + if (!tu) return; + + // Footer bar: CTX utilization %, input, and cumulative output tokens + const ctxEl = document.getElementById('tok-ctx-pct'); + const inputBarEl = document.getElementById('tok-input-bar'); + const outputBarEl = document.getElementById('tok-output-bar'); + if (ctxEl) { + const pct = tu.context_window_utilization_pct; + ctxEl.textContent = pct != null ? pct.toFixed(1) + '%' : '--'; + // Color-code: green < 50%, yellow 50-80%, red > 80% + if (pct != null) { + ctxEl.style.color = pct >= 80 ? '#e74c3c' : pct >= 50 ? '#f39c12' : ''; + } + } + if (inputBarEl) { + inputBarEl.textContent = formatTokenCount(tu.prompt_token_count); + } + if (outputBarEl) { + outputBarEl.textContent = formatTokenCount(tu.cumulative_output_tokens); + } + + // Detail panel: per-field breakdown + const setEl = (id, val) => { + const el = document.getElementById(id); + if (el) el.textContent = val; + }; + setEl('tok-prompt-val', formatTokenCount(tu.prompt_token_count)); + setEl('tok-response-val', formatTokenCount(tu.response_token_count)); + setEl('tok-thoughts-val', formatTokenCount(tu.thoughts_token_count)); + setEl( + 'tok-total-val', + formatTokenCount(tu.total_token_count) + ' / ' + + formatTokenCount(tu.context_window_limit)); + setEl('tok-cumulative-val', formatTokenCount(tu.cumulative_output_tokens)); + + // Per-modality breakdown (compact format) + const modality = tu.prompt_tokens_by_modality || {}; + const parts = Object.entries(modality) + .filter(([, v]) => v > 0) + .map(([k, v]) => k + ':' + formatTokenCount(v)); + setEl('tok-modality-val', parts.length > 0 ? parts.join(' ') : '--'); +} + +let mediaHandler = new MediaHandler(); +let audioFlushPending = false; +let geminiClient = new GeminiClient({ + onOpen: () => { + statusLabel.className = 'status-label connected'; + connectBtn.classList.add('danger'); + connectBtn.title = 'Disconnect from Agent'; + connectBtn.disabled = false; + connectBtn.textContent = 'Disconnect'; + if (modelSelectorBtn) modelSelectorBtn.disabled = true; + if (audioToggleBtn) audioToggleBtn.disabled = true; + if (agentSelectorBtn) agentSelectorBtn.disabled = true; + + // Apply the current video source mode + applyVideoSource(videoSource); + + // Start session timer + startSessionTimer(); + }, + onMessage: (event) => { + if (typeof event.data === 'string') { + try { + const msg = JSON.parse(event.data); + handleJsonMessage(msg); + } catch (e) { + console.error('Parse error:', e); + } + } else { + if (audioFlushPending) { + mediaHandler.stopAudioPlayback(); + audioFlushPending = false; + } + mediaHandler.playAudio(event.data); + } + }, + onClose: (e) => { + console.log('WS Closed:', e); + statusLabel.className = 'status-label disconnected'; + statusLabel.textContent = 'Disconnected'; + stopSessionTimer(); + resetUI(); + }, + onError: (e) => { + console.error('WS Error:', e); + statusLabel.className = 'status-label error'; + statusLabel.textContent = 'Error'; + }, +}); + +// --- Video Source Toggle --- + +function applyVideoSource(source) { + videoSource = source; + + // Tell the server which mode we're in + if (geminiClient.isConnected()) { + geminiClient.send(JSON.stringify({ type: 'video_source', source: source })); + } + + const videoContainer = robotCamImg.parentElement; + + if (source === 'computer') { + // Computer mode: webcam → Gemini directly + robotCamImg.classList.add('hidden'); + robotCamImg.src = ''; + videoContainer.classList.remove('atari-aspect'); + videoContainer.classList.remove('spot-aspect'); + + // Start webcam + videoPlaceholder.classList.add('hidden'); + videoPreview.classList.remove('hidden'); + if (!mediaHandler.videoStream) { + mediaHandler + .startVideo( + videoPreview, + (base64Data) => { + if (geminiClient.isConnected()) { + geminiClient.sendImage(base64Data); + } + }) + .catch(e => console.warn('Camera start failed:', e)); + } + } else { + // Robot / Atari mode: camera poller → Gemini + mediaHandler.stopVideo(videoPreview); + videoPreview.classList.add('hidden'); + videoPlaceholder.classList.add('hidden'); + + // Atari uses 8/5 aspect ratio + if (source === 'atari') { + videoContainer.classList.add('atari-aspect'); + videoContainer.classList.remove('spot-aspect'); + } else { + videoContainer.classList.remove('atari-aspect'); + videoContainer.classList.toggle('spot-aspect', source === 'spot'); + } + + // Show robot cam (stitched MJPEG from camera poller) + robotCamImg.classList.remove('hidden'); + robotCamImg.src = '/api/camera?' + Date.now(); + + // Retry after a short delay + if (geminiClient.isConnected()) { + setTimeout(() => { + robotCamImg.src = '/api/camera?' + Date.now(); + }, 2000); + } + } +} + +// --- Agent Selector Click Bindings --- +if (agentSelectorBtn) { + agentSelectorBtn.onclick = (e) => { + e.stopPropagation(); + agentDropdown.classList.toggle('hidden'); + }; +} + +document.addEventListener('click', (e) => { + if (agentDropdown && !agentDropdown.contains(e.target) && + e.target !== agentSelectorBtn) { + agentDropdown.classList.add('hidden'); + } +}); + +if (agentDropdown) { + agentDropdown.querySelectorAll('.agent-option').forEach(opt => { + opt.onclick = () => { + const agent = opt.dataset.agent; + selectAgent(agent); + + if (agent === 'human') { + applyVideoSource('computer'); + } else if (agent === 'spot') { + applyVideoSource('spot'); + } else if (agent === 'tinybot') { + applyVideoSource('tinybot'); + } + + agentDropdown.classList.add('hidden'); + }; + }); +} + +function selectAgent(type) { + if (typeof selectedAgentName !== 'undefined') { + selectedAgentName = type; + } + + // Clear active state from all agent dropdown options + if (agentDropdown) { + agentDropdown.querySelectorAll('.agent-option') + .forEach(o => o.classList.remove('active')); + const activeOpt = + Array.from(agentDropdown.querySelectorAll('.agent-option')) + .find(o => o.dataset.agent === type); + if (activeOpt) { + activeOpt.classList.add('active'); + if (agentSelectorLabel) { + agentSelectorLabel.textContent = activeOpt.textContent; + } + } + } +} + +// --- Chat Sidebar Toggle --- +chatToggle.onclick = () => { + chatSidebar.classList.toggle('collapsed'); +}; + +// --- Message Handling --- + +function handleJsonMessage(msg) { + if (msg.type === 'heartbeat_sent') { + resetLatencyTimer(); + return; + } + if (msg.type === 'interrupted') { + audioFlushPending = true; + currentGeminiMessageDiv = null; + currentUserMessageDiv = null; + currentThinkingContentDiv = null; + } else if (msg.type === 'turn_complete') { + recordTTLT(); + currentGeminiMessageDiv = null; + currentUserMessageDiv = null; + currentThinkingContentDiv = null; + } else if (msg.type === 'telemetry') { + if (msg.server_latency || msg.server_ttft_ms) { + updateServerLatency( + msg.server_latency || {}, msg.server_ttft_ms || 0); + } + if (msg.token_usage) { + updateTokenUsage(msg.token_usage); + } + + } else if (msg.type === 'draw_points') { + recordTTFT(); + drawPointsOverlay(msg.points); + } else if (msg.type === 'clear_overlay') { + clearOverlay(); + } else if (msg.type === 'user' || msg.type === 'user_transcript') { + // User message breaks ack streak + lastAckRow = null; + ackCount = 0; + currentThinkingContentDiv = null; + // Reset latency timer on voice input (user_transcript from server). + if (msg.type === 'user_transcript' && querySendTime === null) { + resetLatencyTimer(); + } + if (currentUserMessageDiv) { + currentUserMessageDiv.textContent += msg.text; + const isAtBottomUser = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < 50; + if (isAtBottomUser) chatLog.scrollTop = chatLog.scrollHeight; + } else { + currentUserMessageDiv = appendMessage('user', msg.text); + } + } else if (msg.type === 'text_input') { + if (typeof msg.text === 'string' && + msg.text.trimStart().startsWith('[HEARTBEAT]')) { + return; + } + currentThinkingContentDiv = null; + appendMessage('system', `📨 ${msg.text}`); + } else if (msg.type === 'gemini' || msg.type === 'gemini_transcript') { + // Gemini message breaks ack streak + lastAckRow = null; + ackCount = 0; + currentThinkingContentDiv = null; + recordTTFT(); + if (currentGeminiMessageDiv) { + currentGeminiMessageDiv.textContent += msg.text; + const isAtBottomGemini = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < 50; + if (isAtBottomGemini) chatLog.scrollTop = chatLog.scrollHeight; + } else { + currentGeminiMessageDiv = appendMessage('gemini', msg.text); + } + } else if (msg.type === 'gemini_thought') { + // Gemini thought message breaks ack streak + lastAckRow = null; + ackCount = 0; + recordTTFT(); + currentGeminiMessageDiv = null; + currentUserMessageDiv = null; + if (currentThinkingContentDiv) { + currentThinkingContentDiv.textContent += msg.text; + const isAtBottom = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < 50; + if (isAtBottom) chatLog.scrollTop = chatLog.scrollHeight; + } else { + currentThinkingContentDiv = appendThoughtMessage(msg.text); + } + } else if (msg.type === 'tool_call') { + currentThinkingContentDiv = null; + recordTTFT(); + recordTTFC(); + toolCallCounter++; + if (msg.name === 'ack') { + // Aggregate consecutive ack calls into a single heartbeat row + ackCount++; + if (lastAckRow) { + // Update existing heartbeat row + const countSpan = lastAckRow.querySelector('.hb-count'); + if (countSpan) countSpan.textContent = `x${ackCount}`; + // Scroll if at bottom + const isAtBottom = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < + 50; + if (isAtBottom) chatLog.scrollTop = chatLog.scrollHeight; + } else { + // Create new heartbeat row + const row = document.createElement('div'); + row.className = 'message tool_call heartbeat-row'; + row.innerHTML = `❤️` + + `x${ackCount}`; + const isAtBottom = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < + 50; + chatLog.appendChild(row); + if (isAtBottom) chatLog.scrollTop = chatLog.scrollHeight; + lastAckRow = row; + } + } else { + // Non-ack tool call — reset ack aggregation + lastAckRow = null; + ackCount = 0; + const args = msg.args ? JSON.stringify(msg.args) : ''; + const result = + msg.result !== undefined ? formatFunctionResult(msg.result) : ''; + + if (result) { + // Render as collapsible details + const details = document.createElement('details'); + details.className = 'message tool_call_details'; + details.open = true; // Open by default + + const summary = document.createElement('summary'); + summary.className = 'tool_call_summary'; + summary.textContent = `🔧 [${toolCallCounter}] ${msg.name}(${args})`; + details.appendChild(summary); + + const contentDiv = document.createElement('div'); + contentDiv.className = 'tool_response_content'; + contentDiv.textContent = result; + details.appendChild(contentDiv); + + const isAtBottom = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < 50; + chatLog.appendChild(details); + if (isAtBottom) chatLog.scrollTop = chatLog.scrollHeight; + } else { + // Fallback if no result yet + appendMessage( + 'tool_call', `🔧 [${toolCallCounter}] ${msg.name}(${args})`); + } + + if (msg.name === 'run_instruction') { + updateRobotStatus( + 'executing', 'Executing: ' + (msg.args?.instruction || '...')); + } else if (msg.name === 'stop') { + updateRobotStatus('idle', 'Stopped'); + } else if (msg.name === 'reset') { + updateRobotStatus('idle', 'Resetting...'); + } + } + } else if (msg.type === 'tool_response') { + currentThinkingContentDiv = null; + // Tool response breaks ack streak + lastAckRow = null; + ackCount = 0; + appendMessage( + 'tool_response', `✅ ${msg.name}: ${formatFunctionResult(msg.result)}`); + } +} + +// --- Overlay Drawing --- + +function drawPointsOverlay(points) { + const rect = overlayCanvas.parentElement.getBoundingClientRect(); + overlayCanvas.width = rect.width; + overlayCanvas.height = rect.height; + + // Compute actual image bounds within the container (object-fit: contain). + let imgWidth = rect.width; + let imgHeight = rect.height; + let offsetX = 0; + let offsetY = 0; + + if (!robotCamImg.classList.contains('hidden') && + robotCamImg.naturalWidth > 0 && robotCamImg.naturalHeight > 0) { + const imgAspect = robotCamImg.naturalWidth / robotCamImg.naturalHeight; + const containerAspect = rect.width / rect.height; + + if (containerAspect > imgAspect) { + imgHeight = rect.height; + imgWidth = rect.height * imgAspect; + offsetX = (rect.width - imgWidth) / 2; + } else { + imgWidth = rect.width; + imgHeight = rect.width / imgAspect; + offsetY = (rect.height - imgHeight) / 2; + } + } + + const ctx = overlayCanvas.getContext('2d'); + const POINT_RADIUS = 7; + + for (const c of points) { + const cx = offsetX + (c.x / 1000) * imgWidth; + const cy = offsetY + (c.y / 1000) * imgHeight; + + // Draw filled point with contrast outline + ctx.beginPath(); + ctx.arc(cx, cy, POINT_RADIUS, 0, 2 * Math.PI); + ctx.fillStyle = 'rgba(0, 120, 255, 0.9)'; + ctx.fill(); + ctx.strokeStyle = 'rgba(255, 255, 255, 0.95)'; + ctx.lineWidth = 2.5; + ctx.stroke(); + + // Draw label with background pill + if (c.label) { + ctx.font = 'bold 13px sans-serif'; + const labelX = cx + POINT_RADIUS + 8; + const labelY = cy + 4; + const metrics = ctx.measureText(c.label); + const padX = 5, padY = 3; + + // Background pill + ctx.fillStyle = 'rgba(0, 0, 0, 0.65)'; + const pillX = labelX - padX; + const pillY = labelY - 10 - padY; + const pillW = metrics.width + padX * 2; + const pillH = 14 + padY * 2; + ctx.beginPath(); + ctx.roundRect(pillX, pillY, pillW, pillH, 4); + ctx.fill(); + + // Label text + ctx.fillStyle = 'rgba(255, 255, 255, 0.95)'; + ctx.fillText(c.label, labelX, labelY); + } + } +} + +function clearOverlay() { + const rect = overlayCanvas.parentElement.getBoundingClientRect(); + overlayCanvas.width = rect.width; + overlayCanvas.height = rect.height; + // Setting width/height clears the canvas. +} + +let robotState = 'idle'; +let robotStateText = ''; + +function updateRobotStatus(state, text) { + robotState = state; + robotStateText = text; + refreshStatusLabel(); +} + +function refreshStatusLabel() { + if (!statusLabel) return; + // Robot executing takes priority over connection state display + if (robotState === 'executing' && geminiClient.isConnected()) { + statusLabel.className = 'status-label executing'; + statusLabel.textContent = robotStateText; + return; + } + // Otherwise show connection state (timer is updated separately) + if (geminiClient.isConnected()) { + statusLabel.className = 'status-label connected'; + if (!sessionStartTime) { + statusLabel.textContent = 'Connected'; + } + } +} + +function appendMessage(type, text) { + const msgDiv = document.createElement('div'); + msgDiv.className = `message ${type}`; + msgDiv.textContent = text; + const isAtBottom = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < 50; + chatLog.appendChild(msgDiv); + if (isAtBottom) { + chatLog.scrollTop = chatLog.scrollHeight; + } + return msgDiv; +} + +function appendThoughtMessage(text) { + const details = document.createElement('details'); + details.className = 'message thought'; + details.open = false; // Start collapsed + + const summary = document.createElement('summary'); + summary.textContent = 'Thinking Process'; + details.appendChild(summary); + + const contentDiv = document.createElement('div'); + contentDiv.className = 'thought-content'; + contentDiv.textContent = text; + details.appendChild(contentDiv); + + const isAtBottom = + chatLog.scrollHeight - chatLog.scrollTop - chatLog.clientHeight < 50; + chatLog.appendChild(details); + if (isAtBottom) { + chatLog.scrollTop = chatLog.scrollHeight; + } + return contentDiv; +} + +// --- Connect --- + +connectBtn.onclick = async () => { + if (geminiClient.isConnected()) { + geminiClient.disconnect(); + } else { + // Clear previous conversation history when starting a new connection + chatLog.innerHTML = ''; + const memoryLog = document.getElementById('memory-log'); + if (memoryLog) memoryLog.innerHTML = ''; + + statusLabel.className = 'status-label connecting'; + statusLabel.textContent = 'Connecting…'; + connectBtn.disabled = true; + + try { + await mediaHandler.initializeAudio(); + + const sessionCfg = getResolvedSessionConfig(); + const connParams = { + agent_name: selectedAgentName, + model: sessionCfg.model, + response_modality: sessionCfg.response_modality, + use_tts: sessionCfg.use_tts, + endpoint_type: sessionCfg.endpoint_type, + thinking_level: document.getElementById('config-thinking-level') ? + document.getElementById('config-thinking-level').value : undefined, + }; + geminiClient.connect(connParams); + } catch (error) { + console.error('Connection error:', error); + statusLabel.className = 'status-label error'; + statusLabel.textContent = 'Error'; + connectBtn.disabled = false; + } + } +}; + +micBtn.onclick = async () => { + if (mediaHandler.isRecording) { + mediaHandler.stopAudio(); + micBtn.classList.remove('active'); + micBtn.title = 'Start Microphone'; + } else { + try { + await mediaHandler.startAudio((data) => { + if (geminiClient.isConnected()) { + geminiClient.send(data); + } + }); + micBtn.classList.add('active'); + micBtn.title = 'Stop Microphone'; + } catch (e) { + alert('Could not start audio capture'); + } + } +}; + +sendBtn.onclick = sendText; +textInput.onkeypress = (e) => { + if (e.key === 'Enter') sendText(); +}; + +function sendText() { + const text = textInput.value; + if (text && geminiClient.isConnected()) { + resetLatencyTimer(); + geminiClient.sendText(text); + appendMessage('user', text); + textInput.value = ''; + } +} + +function resetUI() { + connectBtn.classList.remove('danger'); + connectBtn.textContent = 'Connect'; + connectBtn.title = 'Connect to Agent'; + if (modelSelectorBtn) modelSelectorBtn.disabled = false; + if (audioToggleBtn) audioToggleBtn.disabled = false; + if (agentSelectorBtn) agentSelectorBtn.disabled = false; + updateAudioUI(); + + mediaHandler.stopAudio(); + mediaHandler.stopVideo(videoPreview); + robotCamImg.classList.add('hidden'); + robotCamImg.src = ''; + videoPlaceholder.classList.remove('hidden'); + + micBtn.classList.remove('active'); + micBtn.title = 'Start Microphone'; + connectBtn.disabled = false; + updateRobotStatus('idle', 'Stopped'); + toolCallCounter = 0; + lastAckRow = null; + ackCount = 0; +} + +// --- Settings Modal and Theme Toggle --- + +let settingsBtn = document.getElementById('settings-btn'); +let settingsModal = document.getElementById('settings-modal'); +let closeModalBtn = document.getElementById('close-modal-btn'); +let themeRadios = document.querySelectorAll('input[name="theme"]'); +let modalTabs = document.querySelectorAll('.modal-sidebar li[data-tab]'); + +let selectedAgentName = 'human'; // Track the currently selected agent + +// Tab switching +modalTabs.forEach(tab => { + tab.onclick = () => { + modalTabs.forEach(t => t.classList.remove('active')); + tab.classList.add('active'); + const tabName = tab.dataset.tab; + document.querySelectorAll('.modal-panel') + .forEach(p => p.classList.add('hidden')); + document.getElementById(`modal-panel-${tabName}`) + .classList.remove('hidden'); + }; +}); + +// Fetch and display agent config +async function loadAgentConfig(name, model) { + try { + let url = `/api/agent_config/${encodeURIComponent(name)}?endpoint_type=${encodeURIComponent(selectedEndpointType)}`; + if (model) { + url += `&model=${encodeURIComponent(model)}`; + } + const resp = await fetch(url); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + const cfg = await resp.json(); + + document.getElementById('config-title').textContent = + `${cfg.name} Agent Config`; + const sessionCfg = getResolvedSessionConfig(); + document.getElementById('config-model').textContent = sessionCfg.model; + document.getElementById('config-modality').textContent = + sessionCfg.response_modality; + document.getElementById('config-tts').textContent = + sessionCfg.use_tts ? 'Enabled' : 'Disabled'; + document.getElementById('config-tts-voice').textContent = cfg.tts_voice; + document.getElementById('config-endpoint').textContent = cfg.endpoint_type; + document.getElementById('config-service').textContent = cfg.service_address; + + // Tools + const toolsDiv = document.getElementById('config-tools'); + toolsDiv.innerHTML = ''; + if (cfg.tools && cfg.tools.length > 0) { + cfg.tools.forEach(tool => { + const chip = document.createElement('span'); + chip.className = 'tool-chip'; + chip.textContent = tool.name; + chip.title = tool.description || ''; + toolsDiv.appendChild(chip); + }); + } else { + toolsDiv.textContent = 'None'; + } + + // Instructions + const devPromptEl = document.getElementById('config-developer-prompt'); + if (devPromptEl) devPromptEl.value = cfg.developer_instruction || ''; + + const hbTextEl = document.getElementById('config-heartbeat-text'); + if (hbTextEl) hbTextEl.value = cfg.heartbeat_text || ''; + + const thinkingLevelEl = document.getElementById('config-thinking-level'); + if (thinkingLevelEl && cfg.thinking_level) { + thinkingLevelEl.value = cfg.thinking_level; + } + + document.getElementById('config-heartbeat-interval').value = + cfg.heartbeat_interval_seconds !== undefined ? cfg.heartbeat_interval_seconds : ''; + document.getElementById('config-heartbeat-min-delay').value = + cfg.heartbeat_min_delay_seconds !== undefined ? cfg.heartbeat_min_delay_seconds : ''; + document.getElementById('config-use-event-driven-heartbeat').checked = + !!cfg.use_event_driven_heartbeat; + + } catch (e) { + console.error('Failed to load agent config:', e); + const devPromptEl = document.getElementById('config-developer-prompt'); + if (devPromptEl) devPromptEl.value = 'Error loading config'; + } +} + +if (settingsBtn) { + settingsBtn.onclick = () => { + settingsModal.classList.remove('hidden'); + // Reset to Agent Config tab + modalTabs.forEach(t => t.classList.remove('active')); + document.getElementById('modal-tab-config').classList.add('active'); + document.querySelectorAll('.modal-panel') + .forEach(p => p.classList.add('hidden')); + document.getElementById('modal-panel-config').classList.remove('hidden'); + loadAgentConfig(selectedAgentName, selectedModel); + }; +} + +let updatePromptBtn = document.getElementById('update-prompt-btn'); +if (updatePromptBtn) { + updatePromptBtn.onclick = async () => { + const newDeveloperPrompt = + document.getElementById('config-developer-prompt') ? document.getElementById('config-developer-prompt').value : ''; + const newHeartbeatText = + document.getElementById('config-heartbeat-text') ? document.getElementById('config-heartbeat-text').value : ''; + const heartbeatInterval = + document.getElementById('config-heartbeat-interval').value; + const heartbeatMinDelay = + document.getElementById('config-heartbeat-min-delay').value; + const useEventDrivenHeartbeat = + document.getElementById('config-use-event-driven-heartbeat').checked; + updatePromptBtn.disabled = true; + updatePromptBtn.textContent = 'Updating...'; + try { + const resp = await fetch('/api/config/instructions', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + agent_name: selectedAgentName, + developer_instruction: newDeveloperPrompt, + heartbeat_text: newHeartbeatText, + heartbeat_interval_seconds: heartbeatInterval !== '' ? parseFloat(heartbeatInterval) : null, + heartbeat_min_delay_seconds: heartbeatMinDelay !== '' ? parseFloat(heartbeatMinDelay) : null, + use_event_driven_heartbeat: useEventDrivenHeartbeat, + }), + }); + const data = await resp.json(); + if (data.success) { + updatePromptBtn.textContent = 'Updated ✅'; + setTimeout(() => { + loadAgentConfig(selectedAgentName, selectedModel); + }, 1000); + } else { + updatePromptBtn.textContent = 'Failed ❌'; + } + } catch (e) { + console.error('Failed to update instructions:', e); + updatePromptBtn.textContent = 'Error ❌'; + } + setTimeout(() => { + updatePromptBtn.disabled = false; + updatePromptBtn.textContent = 'Update Instructions'; + }, 2000); + }; +} + +let resetPromptBtn = document.getElementById('reset-prompt-btn'); +if (resetPromptBtn) { + resetPromptBtn.onclick = async () => { + resetPromptBtn.disabled = true; + resetPromptBtn.textContent = 'Resetting...'; + try { + const resp = await fetch(`/api/config/instructions?agent_name=${encodeURIComponent(selectedAgentName)}`, { + method: 'DELETE', + }); + const data = await resp.json(); + if (data.success) { + resetPromptBtn.textContent = 'Reset ✅'; + setTimeout(() => { + loadAgentConfig(selectedAgentName, selectedModel); + }, 1000); + } else { + resetPromptBtn.textContent = 'Failed ❌'; + } + } catch (e) { + console.error('Failed to reset instructions:', e); + resetPromptBtn.textContent = 'Error ❌'; + } + setTimeout(() => { + resetPromptBtn.disabled = false; + resetPromptBtn.textContent = 'Reset to Default'; + }, 2000); + }; +} + +if (closeModalBtn) { + closeModalBtn.onclick = () => { + settingsModal.classList.add('hidden'); + }; +} + +// Close modal on click outside +window.onclick = (event) => { + if (event.target === settingsModal) { + settingsModal.classList.add('hidden'); + } +}; + +themeRadios.forEach(radio => { + radio.onchange = (e) => { + const theme = e.target.value; + applyTheme(theme); + localStorage.setItem('theme', theme); + }; +}); + +function applyTheme(theme) { + if (theme === 'dark') { + document.body.classList.add('dark-theme'); + } else { + document.body.classList.remove('dark-theme'); + } +} + +// Load theme on startup +let savedTheme = localStorage.getItem('theme') || 'light'; +applyTheme(savedTheme); + +// Update radio button state +themeRadios.forEach(radio => { + if (radio.value === savedTheme) { + radio.checked = true; + } +}); + +// --- Keyboard Shortcuts --- +document.addEventListener('keydown', (e) => { + // Don't intercept when typing in input fields + if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') { + if (e.key === 'Escape') { + e.target.blur(); + } + return; + } + + switch (e.key) { + case ' ': + e.preventDefault(); + micBtn.click(); + break; + case 'Escape': + if (!settingsModal.classList.contains('hidden')) { + settingsModal.classList.add('hidden'); + } else if (geminiClient.isConnected()) { + // Send stop command + geminiClient.sendText('stop'); + } + break; + case '1': + selectAgent('human'); + applyVideoSource('computer'); + break; + case '2': + selectAgent('spot'); + applyVideoSource('spot'); + break; + case 't': + case 'T': + const currentTheme = + document.body.classList.contains('dark-theme') ? 'light' : 'dark'; + applyTheme(currentTheme); + localStorage.setItem('theme', currentTheme); + themeRadios.forEach(r => { + r.checked = r.value === currentTheme; + }); + break; + case '?': + const helpModal = document.getElementById('shortcuts-modal'); + if (helpModal) helpModal.classList.toggle('hidden'); + break; + } +}); + +function escapeHtml(str) { + const div = document.createElement('div'); + div.textContent = str; + return div.innerHTML; +} + +function formatFunctionResult(value, depth = 0) { + if (value === null || value === undefined) return ''; + if (typeof value !== 'object') return String(value); + + const indent = ' '.repeat(depth); + if (Array.isArray(value)) { + return value.map(item => { + const text = formatFunctionResult(item, depth + 1); + const lines = text.split('\n'); + return `${indent}- ${lines[0]}${lines.slice(1).map(line => `\n${indent} ${line}`).join('')}`; + }).join('\n'); + } + + return Object.entries(value).map(([key, item]) => { + const label = key.replaceAll('_', ' '); + if (item !== null && typeof item === 'object') { + return `${indent}${label}:\n${formatFunctionResult(item, depth + 1)}`; + } + return `${indent}${label}: ${formatFunctionResult(item)}`; + }).join('\n'); +} + +// --- Session Timer (integrated into status label) --- +let sessionStartTime = null; +let sessionTimerInterval = null; + +function startSessionTimer() { + sessionStartTime = Date.now(); + updateSessionTimer(); + sessionTimerInterval = setInterval(updateSessionTimer, 1000); +} + +function stopSessionTimer() { + if (sessionTimerInterval) { + clearInterval(sessionTimerInterval); + sessionTimerInterval = null; + } + sessionStartTime = null; +} + +function updateSessionTimer() { + if (!sessionStartTime || !statusLabel) return; + const elapsed = Math.floor((Date.now() - sessionStartTime) / 1000); + const h = Math.floor(elapsed / 3600); + const m = Math.floor((elapsed % 3600) / 60); + const s = elapsed % 60; + const pad = n => String(n).padStart(2, '0'); + const time = h > 0 ? `${pad(h)}:${pad(m)}:${pad(s)}` : `${pad(m)}:${pad(s)}`; + statusLabel.textContent = `Connected ${time}`; +} + +// --- Transparent History Viewer --- + +function prettyPrintPartJs(part) { + if (!part) return ''; + const isThought = part.thought || part.raw_thought || part.rawThought; + const prefix = isThought ? 'Thought: ' : ''; + if (part.text) { + return `${prefix}${part.text}`; + } else if (part.inlineData) { + const mime = part.inlineData.mimeType || 'unknown'; + const len = (part.inlineData.data || '').length; + if (mime.startsWith('image/') || mime.includes('jpeg') || + mime.includes('png')) { + return `
🖼️ Frame
`; + } + return `📦 Data [InlineData: ${mime} (${len} bytes)]`; + } else if (part.audioTranscription) { + return `🗣️ Transcript "${part.audioTranscription.text}"`; + } else if (part.audio_transcription) { + return `🗣️ Transcript "${part.audio_transcription.text}"`; + } else if (part.functionCall) { + const fc = part.functionCall; + const args = JSON.stringify(fc.args || {}); + return `🔧 Call ${fc.name}(${args})`; + } else if (part.functionResponse) { + const fr = part.functionResponse; + const resp = escapeHtml(formatFunctionResult(fr.response || {})) + .replaceAll('\n', '
'); + return `✅ Result ${fr.name} -> ${resp}`; + } else if (part.fileData) { + return `📁 File [FileData: ${part.fileData.mimeType}]`; + } + return `[Unknown Part]`; +} + + + +// --- Latency Detail Toggle --- +let latencyDetailToggle = document.getElementById('latencyDetailToggle'); +let latencyDetail = document.getElementById('latency-detail'); +if (latencyDetailToggle) { + latencyDetailToggle.onclick = () => { + latencyDetail.classList.toggle('hidden'); + }; +} diff --git a/live-api/agent/ui/index.html b/live-api/agent/ui/index.html new file mode 100644 index 0000000..55ed185 --- /dev/null +++ b/live-api/agent/ui/index.html @@ -0,0 +1,267 @@ + + + + + + Proactive Agent — Live Robot Control + + + +
+
+
+ + +
+
+ +
+
+ +
+
+
+
+ +
+
+
+
Start camera to send video
+ + + + +
+
Disconnected
+ +
+
+
+ + +
+
+ + +
+ + + +
+
+
+
+
+ + +
+
+ +
+
Chat
+
+
+
+
+ + +
+
+
+
+ + + + + + + + + + + + + + + + + + diff --git a/live-api/agent/ui/media-handler.js b/live-api/agent/ui/media-handler.js new file mode 100644 index 0000000..091e7ea --- /dev/null +++ b/live-api/agent/ui/media-handler.js @@ -0,0 +1,268 @@ +/** + * MediaHandler: Manages Audio/Video capture and playback + */ +class MediaHandler { + constructor() { + this.audioContext = null; + this.mediaStream = null; + this.audioWorkletNode = null; + this.videoStream = null; + this.videoInterval = null; + this.nextStartTime = 0; + this.scheduledSources = []; + this.isRecording = false; + this.videoCanvas = document.createElement('canvas'); + this.canvasCtx = this.videoCanvas.getContext('2d'); + } + + async initializeAudio() { + if (!this.audioContext) { + this.audioContext = + new (window.AudioContext || window.webkitAudioContext)(); + await this.audioContext.audioWorklet.addModule( + '/static/pcm-processor.js'); + } + if (this.audioContext.state === 'suspended') { + await this.audioContext.resume(); + } + } + + async startAudio(onAudioData) { + await this.initializeAudio(); + + try { + this.mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: true, + }); + const source = + this.audioContext.createMediaStreamSource(this.mediaStream); + this.audioWorkletNode = + new AudioWorkletNode(this.audioContext, 'pcm-processor'); + + this.audioWorkletNode.port.onmessage = (event) => { + if (this.isRecording) { + const downsampled = this.downsampleBuffer( + event.data, this.audioContext.sampleRate, 16000); + const pcm16 = this.convertFloat32ToInt16(downsampled); + onAudioData(pcm16); + } + }; + + source.connect(this.audioWorkletNode); + // Mute local feedback + const muteGain = this.audioContext.createGain(); + muteGain.gain.value = 0; + this.audioWorkletNode.connect(muteGain); + muteGain.connect(this.audioContext.destination); + + this.isRecording = true; + } catch (e) { + console.error('Error starting audio:', e); + throw e; + } + } + + stopAudio() { + this.isRecording = false; + if (this.mediaStream) { + this.mediaStream.getTracks().forEach((t) => t.stop()); + this.mediaStream = null; + } + if (this.audioWorkletNode) { + this.audioWorkletNode.disconnect(); + this.audioWorkletNode = null; + } + } + + async startVideo(videoElement, onFrame) { + try { + this.videoStream = await navigator.mediaDevices.getUserMedia({ + video: true, + }); + videoElement.srcObject = this.videoStream; + + this.videoInterval = setInterval(() => { + this.captureFrame(videoElement, onFrame); + }, 1000); // 1 FPS + } catch (e) { + console.error('Error starting video:', e); + throw e; + } + } + + /** + * Start video and also upload frames to the robot backend at 5 FPS. + * This feeds the fake backend so the robot camera pipeline works. + */ + async startVideoWithRobotFeed(videoElement, onFrame) { + try { + this.videoStream = await navigator.mediaDevices.getUserMedia({ + video: true, + }); + videoElement.srcObject = this.videoStream; + + // 1 FPS for Gemini direct image input + this.videoInterval = setInterval(() => { + this.captureFrame(videoElement, onFrame); + }, 1000); + + // 5 FPS upload to fake backend via proxy + const uploadCanvas = document.createElement('canvas'); + uploadCanvas.width = 384; + uploadCanvas.height = 384; + const uploadCtx = uploadCanvas.getContext('2d'); + + this._robotFeedInterval = setInterval(() => { + if (!this.videoStream) return; + uploadCtx.drawImage(videoElement, 0, 0, 384, 384); + uploadCanvas.toBlob(async (blob) => { + if (!blob) return; + try { + await fetch('/api/upload_frame', { + method: 'POST', + headers: {'Content-Type': 'image/jpeg'}, + body: blob, + }); + } catch (e) { + // silent — backend may not be ready yet + } + }, 'image/jpeg', 0.8); + }, 200); // 5 FPS + + } catch (e) { + console.error('Error starting video:', e); + throw e; + } + } + + async startScreen(videoElement, onFrame, onEnded) { + try { + this.videoStream = await navigator.mediaDevices.getDisplayMedia({ + video: true, + }); + videoElement.srcObject = this.videoStream; + + // Handle stream ending (e.g. user clicks "Stop sharing" in browser UI) + this.videoStream.getVideoTracks()[0].onended = () => { + this.stopVideo(videoElement); + if (onEnded) onEnded(); + }; + + this.videoInterval = setInterval(() => { + this.captureFrame(videoElement, onFrame); + }, 1000); // 1 FPS + } catch (e) { + console.error('Error starting screen share:', e); + throw e; + } + } + + stopVideo(videoElement) { + if (this.videoStream) { + this.videoStream.getTracks().forEach((t) => t.stop()); + this.videoStream = null; + } + if (this.videoInterval) { + clearInterval(this.videoInterval); + this.videoInterval = null; + } + if (this._robotFeedInterval) { + clearInterval(this._robotFeedInterval); + this._robotFeedInterval = null; + } + if (videoElement) { + videoElement.srcObject = null; + } + } + + captureFrame(videoElement, onFrame) { + if (!this.videoStream) return; + this.videoCanvas.width = 640; + this.videoCanvas.height = 480; + this.canvasCtx.drawImage(videoElement, 0, 0, 640, 480); + const base64 = this.videoCanvas.toDataURL('image/jpeg', 0.7).split(',')[1]; + onFrame(base64); + } + + playAudio(arrayBuffer) { + if (!this.audioContext) return; + if (this.audioContext.state === 'suspended') { + this.audioContext.resume(); + } + + let bufferToUse = arrayBuffer; + if (arrayBuffer.byteLength % 2 !== 0) { + console.warn(`Audio buffer length is odd (${ + arrayBuffer.byteLength}), slicing to even length.`); + bufferToUse = arrayBuffer.slice(0, arrayBuffer.byteLength - 1); + } + const pcmData = new Int16Array(bufferToUse); + const float32Data = new Float32Array(pcmData.length); + for (let i = 0; i < pcmData.length; i++) { + float32Data[i] = pcmData[i] / 32768.0; + } + + const buffer = this.audioContext.createBuffer(1, float32Data.length, 24000); + buffer.getChannelData(0).set(float32Data); + + const source = this.audioContext.createBufferSource(); + source.buffer = buffer; + source.connect(this.audioContext.destination); + + const now = this.audioContext.currentTime; + this.nextStartTime = Math.max(now, this.nextStartTime); + source.start(this.nextStartTime); + this.nextStartTime += buffer.duration; + + this.scheduledSources.push(source); + source.onended = () => { + const idx = this.scheduledSources.indexOf(source); + if (idx > -1) this.scheduledSources.splice(idx, 1); + }; + } + + stopAudioPlayback() { + this.scheduledSources.forEach((s) => { + try { + s.stop(); + } catch (e) { + } + }); + this.scheduledSources = []; + if (this.audioContext) { + this.nextStartTime = this.audioContext.currentTime; + } + } + + // Utils + downsampleBuffer(buffer, sampleRate, outSampleRate) { + if (outSampleRate === sampleRate) return buffer; + const ratio = sampleRate / outSampleRate; + const newLength = Math.round(buffer.length / ratio); + const result = new Float32Array(newLength); + let offsetResult = 0; + let offsetBuffer = 0; + while (offsetResult < result.length) { + const nextOffsetBuffer = Math.round((offsetResult + 1) * ratio); + let accum = 0, count = 0; + for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; + i++) { + accum += buffer[i]; + count++; + } + result[offsetResult] = accum / count; + offsetResult++; + offsetBuffer = nextOffsetBuffer; + } + return result; + } + + convertFloat32ToInt16(buffer) { + let l = buffer.length; + const buf = new Int16Array(l); + while (l--) { + buf[l] = Math.min(1, Math.max(-1, buffer[l])) * 0x7fff; + } + return buf.buffer; + } +} diff --git a/live-api/agent/ui/pcm-processor.js b/live-api/agent/ui/pcm-processor.js new file mode 100644 index 0000000..820e758 --- /dev/null +++ b/live-api/agent/ui/pcm-processor.js @@ -0,0 +1,28 @@ +class PCMProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.bufferSize = 4096; + this.buffer = new Float32Array(this.bufferSize); + this.bufferIndex = 0; + } + + process(inputs, outputs, parameters) { + const input = inputs[0]; + if (!input || !input.length) return true; + + const channelData = input[0]; + + for (let i = 0; i < channelData.length; i++) { + this.buffer[this.bufferIndex++] = channelData[i]; + + if (this.bufferIndex >= this.bufferSize) { + this.port.postMessage(this.buffer); + this.bufferIndex = 0; + } + } + + return true; + } +} + +registerProcessor('pcm-processor', PCMProcessor); diff --git a/live-api/agent/ui/style.css b/live-api/agent/ui/style.css new file mode 100644 index 0000000..d3e716e --- /dev/null +++ b/live-api/agent/ui/style.css @@ -0,0 +1,1709 @@ +:root { + --bg-dark: #f8f9fa; + --bg-surface: #ffffff; + --bg-panel: #ffffff; + --text-primary: #202124; + --text-secondary: #5f6368; + --border-color: #dadce0; + --accent: #202124; + --accent-bg: #f1f3f4; + --red: #d93025; + --radius-md: 6px; + --radius-lg: 12px; +} + +.dark-theme { + --bg-dark: #0f0f0f; + --bg-surface: #1a1a1a; + --bg-panel: #242424; + --text-primary: #f0f0f0; + --text-secondary: #888888; + --border-color: #333333; + --accent: #ffffff; + --accent-bg: #2a2a2a; + --red: #ff453a; +} + +* { box-sizing: border-box; } + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + font-size: 13px; + background-color: var(--bg-dark); + color: var(--text-primary); + margin: 0; + padding: 0; + line-height: 1.5; + display: flex; + justify-content: center; + height: 100vh; + overflow: hidden; +} + +.container { + width: 100%; + height: calc(100vh - 32px); + margin: 0; + background: var(--bg-surface); + padding: 24px; + border: none; + border-radius: 0; + display: flex; + flex-direction: column; + overflow: hidden; +} + +#sidebar { + width: 250px; + min-width: 250px; + border-right: 1px solid var(--border-color); + padding-right: 20px; + padding-bottom: 8px; + display: flex; + flex-direction: column; + gap: 20px; + overflow-y: auto; + transition: width 0.2s ease, min-width 0.2s ease, padding 0.2s ease, opacity 0.2s ease; +} + +#sidebar.collapsed { + width: 0; + min-width: 0; + padding-right: 0; + border-right: none; + overflow: hidden; + opacity: 0; +} + +.sidebar-toggle { + font-size: 16px; + padding: 0.3rem 0.6rem; + line-height: 1; +} + +.see-all-link { + color: var(--text-secondary); + font-size: 13px; + cursor: pointer; + text-decoration: none; + padding: 4px 12px; + transition: color 0.15s ease; +} + +.see-all-link:hover { + color: var(--text-primary); +} + +#sidebar h2 { + font-size: 13px; + font-weight: 600; + margin-top: 0; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +#sidebar ul { + list-style: none; + padding: 0; + margin: 0; +} + +#sidebar li { + padding: 8px 12px; + cursor: pointer; + border-radius: var(--radius-md); + font-size: 13px; + transition: background 0.15s ease; +} + +#sidebar li:hover { + background: var(--accent-bg); +} + +#sidebar li.active { + background: var(--accent); + color: var(--bg-dark); + font-weight: 500; +} + +#main-content { + flex: 1; + padding-left: 20px; + display: flex; + flex-direction: row; + overflow: hidden; +} + +.main-body { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; + min-width: 0; +} + +main { + flex: 1; + overflow-y: auto; +} + +header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; + padding-bottom: 0.75rem; + border-bottom: 1px solid var(--border-color); + position: relative; +} + +header h1 { + font-weight: 600; + font-size: 16px; + letter-spacing: -0.02em; + margin: 0; + position: absolute; + left: 50%; + transform: translateX(-50%); +} + +/* Status Label (unified connection + robot state + timer) */ +.status-label { + display: flex; + align-items: center; + justify-content: flex-start; + gap: 6px; + padding: 6px 12px; + font-size: 13px; + color: var(--text-secondary); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.status-label::before { + content: ''; + width: 8px; + height: 8px; + border-radius: 50%; + flex-shrink: 0; +} + +.status-label.disconnected::before { + background: var(--text-secondary); +} + +.status-label.connected::before { + background: #34a853; + box-shadow: 0 0 6px rgba(52, 168, 83, 0.4); +} + +.status-label.error::before { + background: var(--red); + box-shadow: 0 0 6px rgba(217, 48, 37, 0.4); +} + +.status-label.connecting::before { + background: #fbbc04; + animation: pulse-dot 1s ease-in-out infinite; +} + +@keyframes pulse-dot { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.3; } +} + +.hidden { + display: none !important; +} + +.video-container { + width: 100%; + aspect-ratio: 1/1; + max-height: calc(100vh - 280px); + background: var(--bg-dark); + border-radius: var(--radius-md); + overflow: hidden; + margin-bottom: 0.5rem; + position: relative; + border: 1px solid var(--border-color); +} + +.video-container.atari-aspect { + aspect-ratio: 8/5; +} + +.video-container.spot-aspect { + aspect-ratio: 4/3; +} + +#video-placeholder { + position: absolute; + top: 0; left: 0; width: 100%; height: 100%; + display: flex; + align-items: center; + justify-content: center; + color: var(--text-secondary); + font-size: 13px; + z-index: 1; +} + +video { + width: 100%; + height: 100%; + object-fit: cover; +} + +.robot-cam-img { + width: 100%; + height: 100%; + object-fit: contain; + position: absolute; + top: 0; left: 0; + z-index: 2; + background: var(--bg-dark); +} + +#overlay-canvas { + position: absolute; + top: 0; left: 0; + width: 100%; + height: 100%; + z-index: 10; + pointer-events: none; +} + +.controls-grid { + display: flex; + flex-direction: column; + gap: 0.5rem; + margin-bottom: 1rem; +} + +.controls-row { + display: flex; + gap: 0.5rem; + flex-wrap: wrap; + align-items: center; +} + +.btn { + padding: 0.5rem 1rem; + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + background: var(--bg-panel); + color: var(--text-primary); + cursor: pointer; + font-weight: 500; + font-size: 13px; + transition: all 0.15s ease; +} + +.btn:hover { + background: var(--accent-bg); + border-color: #555; +} + +.btn:active { + transform: scale(0.98); +} + +.btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.btn.danger { + color: var(--red); + border-color: rgba(255, 69, 58, 0.3); +} + +.btn.danger:hover { + background: rgba(255, 69, 58, 0.1); + border-color: var(--red); +} + +.btn.rerun-connect-btn { + background-color: #ffffff !important; + color: #202124 !important; + border-color: #dadce0 !important; +} + +.btn.rerun-connect-btn:hover { + background-color: #f8f9fa !important; + border-color: #999999 !important; + color: #202124 !important; +} + +.video-source-toggle { + display: flex; + align-items: center; + background: var(--bg-panel); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + overflow: hidden; + padding: 2px; + width: fit-content; +} + +.segment { + padding: 0.4rem 0.8rem; + font-size: 13px; + font-weight: 500; + color: var(--text-secondary); + background: transparent; + border: none; + border-radius: 4px; + cursor: pointer; + transition: all 0.2s ease; + display: flex; + align-items: center; + gap: 0.25rem; +} + +.segment:hover { + color: var(--text-primary); +} + +.segment.active { + color: var(--bg-dark); + background: var(--text-primary); +} + +.chat-log { + flex: 1; + overflow-y: auto; + padding: 1rem; + background: var(--bg-panel); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + margin-bottom: 0.8rem; + max-height: 400px; + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.chat-log::-webkit-scrollbar { + width: 6px; +} + +.chat-log::-webkit-scrollbar-thumb { + background: var(--border-color); + border-radius: 3px; +} + +.message { + padding: 0.5rem 0.75rem; + border-radius: var(--radius-md); + max-width: 85%; + word-wrap: break-word; + font-size: 13px; + white-space: pre-wrap; +} + +.message.user { + align-self: flex-end; + background: var(--accent-bg); + border: 1px solid var(--border-color); +} + +.message.gemini { + align-self: flex-start; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-primary); +} + +.message.tool_call { + align-self: flex-start; + color: var(--text-secondary); + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 12px; +} + +/* Heartbeat row: aggregated ack tool calls */ +.heartbeat-row { + display: flex; + align-items: center; + gap: 6px; + padding: 3px 10px; + max-width: 100%; +} + +.hb-icon { + font-size: 13px; + animation: hb-pulse 1.2s ease-in-out infinite; + flex-shrink: 0; +} + +@keyframes hb-pulse { + 0%, 100% { opacity: 1; transform: scale(1); } + 50% { opacity: 0.4; transform: scale(0.9); } +} + +.hb-count { + color: var(--text-secondary); + font-size: 11px; + flex-shrink: 0; + opacity: 0.8; +} + +.message.system { + align-self: flex-start; + color: var(--text-secondary); + font-style: italic; + font-size: 12px; +} + +.message.tool_response { + align-self: flex-start; + color: var(--text-secondary); + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 11px; + background: var(--bg-panel); + border: 1px solid var(--border-color); + margin-top: 2px; + margin-bottom: 2px; + max-width: 90%; +} + +.input-area { + display: flex; + gap: 0.5rem; + flex-shrink: 0; +} + +input[type="text"], input[type="number"] { + flex: 1; + padding: 0.5rem 0.75rem; + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + background: var(--bg-panel); + color: var(--text-primary); + font-size: 13px; +} + +input[type="text"]:focus, input[type="number"]:focus { + outline: none; + border-color: #666; +} + +input[type="text"]::placeholder, input[type="number"]::placeholder { + color: var(--text-secondary); +} + +.app-grid { + display: flex; + flex-direction: column; + gap: 1rem; + align-items: stretch; +} + +/* Chat Sidebar (right side, collapsible) */ +.chat-sidebar { + width: 500px; + min-width: 500px; + border-left: 1px solid var(--border-color); + padding-left: 20px; + display: flex; + flex-direction: column; + gap: 0.5rem; + transition: width 0.2s ease, min-width 0.2s ease, padding 0.2s ease, opacity 0.2s ease; +} + +.chat-sidebar.collapsed { + width: 0; + min-width: 0; + padding-left: 0; + border-left: none; + overflow: hidden; + opacity: 0; +} + +@media (max-width: 768px) { + .chat-sidebar { + width: 100%; + min-width: 100%; + border-left: none; + border-top: 1px solid var(--border-color); + padding-left: 0; + padding-top: 12px; + } +} + +.chat-sidebar .chat-log { + flex: 1; + min-height: 0; + max-height: none; +} + +.sidebar-tabs { + display: flex; + border-bottom: 1px solid var(--border-color); + margin-bottom: 8px; + gap: 4px; +} + +.tab-btn { + flex: 1; + background: none; + border: none; + border-bottom: 2px solid transparent; + padding: 8px 12px; + font-size: 12px; + font-weight: 600; + color: var(--text-secondary); + cursor: pointer; + transition: all 0.15s ease; + border-radius: var(--radius-md) var(--radius-md) 0 0; +} + +.tab-btn:hover { + color: var(--text-primary); + background: var(--accent-bg); +} + +.tab-btn.active { + color: var(--text-primary); + border-bottom-color: var(--text-primary); + background: var(--accent-bg); +} + +.hidden { + display: none !important; +} + +/* --- Event Card (MCAP Log Visualization) --- */ + +.event-card { + background-color: var(--bg-panel); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + padding: 15px; + margin-bottom: 12px; + font-size: 14px; +} + +.event-card .header { + display: flex; + justify-content: space-between; + color: var(--text-secondary); + margin-bottom: 10px; + font-size: 12px; +} + +.event-card .topic { + font-weight: bold; + color: #5e9eff; +} + +.event-card .data { + white-space: pre-wrap; + word-break: break-all; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + background-color: var(--bg-dark); + padding: 10px; + border-radius: 4px; + font-size: 12px; + max-height: 300px; + overflow-y: auto; +} + +.event-card .data.image-container { + max-height: none; +} + +.event-card .data.json { + color: #d4a849; +} + +.event-card .data img { + max-width: 100%; + border-radius: 4px; +} + +#log-viewer-section { + padding: 0 4px; +} + +#log-viewer-section h2 { + font-weight: 500; + margin-top: 0; + margin-bottom: 16px; + color: var(--text-secondary); +} + +/* --- Settings Modal --- */ +.modal { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0, 0, 0, 0.5); + display: flex; + justify-content: center; + align-items: center; + z-index: 1000; +} + +.modal.hidden { + display: none !important; +} + +.modal-content { + background: var(--bg-surface); + border: 1px solid var(--border-color); + border-radius: var(--radius-lg); + width: 600px; + max-width: 90%; + height: 400px; + display: flex; + overflow: hidden; + max-height: 90vh; +} + +.modal-sidebar { + width: 200px; + border-right: 1px solid var(--border-color); + background: var(--bg-dark); + padding: 20px; +} + +.modal-sidebar ul { + list-style: none; + padding: 0; + margin: 0; +} + +.modal-sidebar li { + padding: 8px 12px; + cursor: pointer; + border-radius: var(--radius-md); + font-size: 13px; + color: var(--text-secondary); +} + +.modal-sidebar li.active { + background: var(--accent-bg); + color: var(--text-primary); + font-weight: 500; +} + +.modal-body { + flex: 1; + padding: 20px; + display: flex; + flex-direction: column; + gap: 20px; + min-height: 0; + overflow: hidden; +} + +.modal-body h2 { + margin-top: 0; + font-size: 16px; + color: var(--text-primary); +} + +.theme-options { + display: flex; + gap: 20px; +} + +.theme-options label { + display: flex; + align-items: center; + gap: 8px; + cursor: pointer; + color: var(--text-primary); +} + +#close-modal-btn { + align-self: flex-end; + margin-top: auto; +} + +/* Settings Button in Sidebar */ +#sidebar-footer { + margin-top: auto; + padding-top: 10px; + border-top: 1px solid var(--border-color); +} + +#settings-btn { + display: flex; + align-items: center; + gap: 10px; + color: var(--text-secondary); + padding: 8px 12px; + cursor: pointer; + border-radius: var(--radius-md); + font-size: 13px; + transition: background 0.15s ease; +} + +#settings-btn:hover { + background: var(--accent-bg); + color: var(--text-primary); +} + +/* --- Config Display in Settings Modal --- */ + +.modal-content-lg { + width: 800px; + max-width: 95%; + height: 80vh; +} + +.modal-panel { + flex: 1; + overflow-y: auto; + min-height: 0; +} + +.modal-panel.hidden { + display: none !important; +} + +.modal-body h3 { + font-size: 14px; + font-weight: 600; + color: var(--text-secondary); + margin-bottom: 8px; + margin-top: 16px; +} + +.config-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 10px; +} + +.config-item { + display: flex; + flex-direction: column; + gap: 2px; +} + +.config-item label { + font-size: 11px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.config-item code { + font-family: inherit; + font-size: 12px; + color: var(--text-primary); + background: var(--accent-bg); + padding: 4px 8px; + border-radius: 4px; + word-break: break-all; +} + +.config-tools { + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +.tool-chip { + display: inline-block; + padding: 3px 10px; + font-size: 13px; + font-family: inherit; + background: var(--accent-bg); + border: 1px solid var(--border-color); + border-radius: 12px; + color: var(--text-primary); + cursor: default; +} + +.config-prompt { + font-family: inherit; + font-size: 12px; + background: var(--bg-dark); + color: var(--text-primary); + padding: 12px; + border-radius: var(--radius-md); + border: 1px solid var(--border-color); + white-space: pre-wrap; + word-break: break-word; + max-height: 500px; + overflow-y: auto; + margin: 0; + line-height: 1.5; +} + +/* Robot Status Indicator */ + +.status-label.executing { + color: #2e7d32; +} + +.status-label.executing::before { + background: #4caf50; + animation: pulse 1.5s ease-in-out infinite; +} + +.dark-theme .status-label.executing { + color: #66bb6a; +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.3; } +} + +/* Keyboard Shortcuts */ +.shortcuts-table { + width: 100%; + margin: 16px 0; +} +.shortcuts-table td { + padding: 8px 12px; + border-bottom: 1px solid var(--border); +} +.shortcuts-table td:first-child { + width: 100px; + text-align: center; +} +kbd { + display: inline-block; + padding: 3px 8px; + font-size: 12px; + font-family: monospace; + background: var(--surface); + border: 1px solid var(--border); + border-radius: 4px; + box-shadow: 0 1px 0 var(--border); +} + +/* --- Debug Footer & Inline Debug Entries --- */ +.debug-footer { + position: fixed; + bottom: 0; + left: 0; + right: 0; + height: 32px; + background: var(--bg-surface); + border-top: 1px solid var(--border-color); + display: flex; + align-items: center; + padding: 0 12px; + gap: 8px; + z-index: 100; + font-size: 12px; +} + +.debug-toggle-btn { + background: none; + border: none; + color: var(--text-secondary); + cursor: pointer; + font-size: 12px; + padding: 2px 8px; + border-radius: 3px; + transition: all 0.15s ease; + flex-shrink: 0; +} + +.debug-toggle-btn:hover { + background: var(--accent-bg); + color: var(--text-primary); +} + +.debug-toggle-btn.active { + background: var(--accent); + color: var(--bg-dark); +} + + +.debug-event-count { + color: var(--text-secondary); + font-size: 11px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + flex-shrink: 0; +} + +.debug-filters { + display: flex; + flex-wrap: wrap; + gap: 4px; + flex: 1; + min-width: 0; +} + +.debug-filter-chip { + padding: 2px 8px; + border: 1px solid var(--border-color); + border-radius: 10px; + background: transparent; + color: var(--text-secondary); + cursor: pointer; + font-size: 11px; + transition: all 0.15s ease; +} + +.debug-filter-chip.active { + background: var(--accent-bg); + color: var(--text-primary); + border-color: var(--text-secondary); +} + +.debug-filter-chip:hover { + border-color: var(--text-primary); +} + +/* Debug entries rendered in separate debug-log container */ +.debug-log { + flex: 1; + overflow-y: auto; + padding: 4px 8px; + background: var(--bg-dark); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + min-height: 0; +} + +.debug-log::-webkit-scrollbar { + width: 6px; +} + +.debug-log::-webkit-scrollbar-thumb { + background: var(--border-color); + border-radius: 3px; +} + +.debug-entry { + display: flex; + align-items: baseline; + gap: 6px; + padding: 2px 4px; + line-height: 1.5; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 11px; + border-radius: 3px; +} + +.debug-entry:hover { + opacity: 1; + background: var(--accent-bg); +} + +.debug-time { + color: var(--text-secondary); + flex-shrink: 0; +} + +.debug-icon { + flex-shrink: 0; + width: 16px; + text-align: center; +} + +.debug-badge { + display: inline-block; + padding: 0 5px; + border-radius: 3px; + font-size: 9px; + font-weight: 600; + text-transform: uppercase; + flex-shrink: 0; + min-width: 75px; + text-align: center; +} + +.debug-badge-tool_call { background: #1a3a5c; color: #64b5f6; } +.debug-badge-tool_response { background: #1b3a1b; color: #66bb6a; } +.debug-badge-gemini, .debug-badge-gemini_transcript { background: #3a2a1a; color: #ffb74d; } +.debug-badge-user, .debug-badge-user_transcript { background: #2a1a3a; color: #ce93d8; } +.debug-badge-turn_complete { background: #1a2a2a; color: #80cbc4; } +.debug-badge-interrupted { background: #3a1a1a; color: #ef9a9a; } +.debug-badge-draw_points { background: #1a3a2a; color: #a5d6a7; } +.debug-badge-error { background: #4a1a1a; color: #ff5252; } +.debug-badge-audio_response { background: #2a2a1a; color: #fff176; } + +.debug-data { + color: var(--text-primary); + word-break: break-all; + flex: 1; + min-width: 0; +} + + +/* ============================================================================= + Eval Mode + ============================================================================= */ +.eval-toggle-btn{background:none;border:none;color:var(--text-secondary);cursor:pointer;font-size:12px;padding:2px 8px;border-radius:3px;transition:all .15s ease;flex-shrink:0} +.eval-toggle-btn:hover{background:var(--accent-bg);color:var(--text-primary)} +.eval-toggle-btn.active{background:#1a73e8;color:#fff} +.eval-toggle-btn:disabled{opacity:.4;cursor:not-allowed} +.eval-panel{padding:16px;overflow-y:auto;flex:1} +.eval-section{animation:fadeIn .2s ease-in} +@keyframes fadeIn{from{opacity:0;transform:translateY(4px)}to{opacity:1;transform:translateY(0)}} +.eval-header{font-size:18px;font-weight:600;margin-bottom:16px;color:var(--text-primary)} +.eval-robot-status{font-size:13px;padding:8px 12px;border-radius:6px;margin-bottom:12px} +.eval-robot-status.healthy{background:rgba(52,168,83,.1);color:#34a853} +.eval-robot-status.unhealthy{background:rgba(234,67,53,.1);color:#ea4335} +.eval-robot-status.checking{background:rgba(251,188,4,.1);color:#fbbc04} +.eval-episode-header{display:flex;justify-content:space-between;align-items:center;margin-bottom:16px;font-size:16px;font-weight:600} +.eval-timer{font-family:monospace;font-size:14px;color:var(--text-secondary);background:var(--accent-bg);padding:2px 8px;border-radius:4px} +.eval-instruction{margin-bottom:12px} +.eval-label{font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.5px;color:var(--text-secondary);margin-bottom:4px} +.eval-content{font-size:14px;line-height:1.5;padding:8px 12px;background:var(--accent-bg);border-radius:6px;border-left:3px solid #1a73e8} +.eval-actions{display:flex;gap:8px;margin-top:16px} +.eval-actions .btn{flex:1;padding:10px;font-size:14px;border-radius:8px;cursor:pointer;border:none;font-weight:500} +.eval-actions .btn-primary{background:#1a73e8;color:#fff} +.eval-actions .btn-primary:hover{background:#1557b0} +.eval-actions .btn-primary:disabled{opacity:.5;cursor:not-allowed} +.eval-actions .btn-secondary{background:var(--accent-bg);color:var(--text-primary);border:1px solid var(--border-color)} +.eval-actions .btn-secondary:hover{background:var(--border-color)} +.eval-score-options{display:flex;flex-direction:column;gap:6px;margin:12px 0} +.eval-score-btn{padding:10px 14px;border:1px solid var(--border-color);border-radius:8px;background:var(--bg-surface);color:var(--text-primary);cursor:pointer;font-size:14px;text-align:left;transition:all .15s} +.eval-score-btn:hover{background:var(--accent-bg);border-color:#1a73e8} +.eval-score-btn.selected{background:rgba(26,115,232,.1);border-color:#1a73e8;color:#1a73e8;font-weight:500} +.eval-note{width:100%;padding:8px 12px;border:1px solid var(--border-color);border-radius:6px;font-size:13px;font-family:inherit;resize:vertical;background:var(--bg-surface);color:var(--text-primary);margin-bottom:12px;box-sizing:border-box} +.eval-stop-floating{position:fixed;bottom:44px;right:20px;padding:10px 20px;background:#ea4335;color:#fff;border:none;border-radius:24px;font-size:14px;font-weight:500;cursor:pointer;box-shadow:0 2px 8px rgba(0,0,0,.3);z-index:1000;transition:transform .15s} +.eval-stop-floating:hover{transform:scale(1.05);background:#d93025} +#eval-panel .btn-primary{background:#1a73e8;color:#fff;border:none;padding:10px 20px;border-radius:8px;cursor:pointer;font-size:14px;font-weight:500} +#eval-panel .btn-primary:hover{background:#1557b0} +#eval-panel .btn-primary:disabled{opacity:.5;cursor:not-allowed} +#eval-panel .btn{padding:10px 20px;border-radius:8px;cursor:pointer;font-size:14px;border:1px solid var(--border-color);background:var(--bg-surface);color:var(--text-primary)} +#eval-panel .btn:hover{background:var(--accent-bg)} +.eval-config-chips{display:flex;flex-wrap:wrap;gap:6px;border-left:3px solid #fbbc04} +.eval-config-chip{display:inline-block;padding:3px 10px;font-size:12px;background:rgba(251,188,4,.1);border:1px solid rgba(251,188,4,.3);border-radius:12px;color:var(--text-primary);font-weight:500} +.eval-scoring-header{display:flex;justify-content:space-between;align-items:center} +.eval-scoring-elapsed{font-family:monospace;font-size:13px;color:var(--text-secondary)} +.eval-scoring-hint{font-size:11px;color:var(--text-secondary);margin:4px 0 8px;letter-spacing:.3px} +.eval-summary{margin:12px 0;padding:12px;background:var(--accent-bg);border-radius:8px;font-size:13px;line-height:1.8} +.eval-summary-row{display:flex;justify-content:space-between} +.eval-summary-label{color:var(--text-secondary)} +.eval-summary-value{font-weight:600;color:var(--text-primary)} +.eval-score-btn{display:flex;justify-content:space-between;align-items:center} +.eval-score-def{flex:1} +.eval-score-value{font-size:12px;font-weight:600;color:var(--text-secondary);white-space:nowrap;margin-left:12px} +.eval-score-btn.selected .eval-score-value{color:#1a73e8} +.eval-actions .btn-danger{background:#fff;color:#d93025;border:1px solid #d93025} +.eval-actions .btn-danger:hover{background:#fce8e6} +.eval-actions .btn-danger:disabled{opacity:.5;cursor:not-allowed} +.eval-inline-card{margin:8px 0;padding:12px;background:var(--bg-surface);border:1px solid var(--border-color);border-radius:10px} +.eval-inline-header{font-size:16px;font-weight:600;margin-bottom:12px;color:var(--text-primary)} +.eval-inline-score-options{display:flex;flex-direction:column;gap:6px;margin:12px 0} + +/* Latency bar in debug footer */ +.latency-bar { + display: flex; + gap: 12px; + align-items: center; + margin-left: auto; +} + +.latency-bar-item { + font-size: 11px; + color: var(--text-secondary); + white-space: nowrap; +} + +.latency-bar .metric-value { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 11px; + font-weight: 600; + color: #2ecc71; +} + +.latency-bar .metric-count { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 10px; + color: var(--text-secondary); +} + +/* Latency detail panel (slide-up above footer) */ +.latency-detail { + position: fixed; + bottom: 32px; + right: 0; + background: var(--bg-surface); + border: 1px solid var(--border-color); + border-bottom: none; + border-radius: 8px 8px 0 0; + padding: 12px 16px; + z-index: 99; + min-width: 280px; + box-shadow: 0 -2px 8px rgba(0, 0, 0, 0.2); + display: flex; + gap: 24px; +} + +.latency-bar-divider { + color: var(--border-color); + font-size: 11px; + padding: 0 2px; +} + +.latency-detail.hidden { + display: none; +} + +.latency-detail-section h4 { + margin: 0 0 8px 0; + font-size: 11px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.latency-detail-section hr { + border: none; + border-top: 1px solid var(--border-color); + margin: 8px 0; +} + +.metric-row { + display: flex; + justify-content: space-between; + align-items: baseline; + padding: 3px 0; + gap: 8px; +} + +.metric-label { + font-size: 12px; + color: var(--text-secondary); +} + +.metric-value { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 12px; + font-weight: 600; + color: #2ecc71; +} + +.metric-count { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 10px; + color: var(--text-secondary); + min-width: 35px; + text-align: right; +} + +/* --- Model Selector Dropdown --- */ +.model-selector-wrapper { + position: relative; +} + +.model-selector-btn { + display: flex; + align-items: center; + gap: 6px; + font-size: 13px; + max-width: 280px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.model-selector-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.model-selector-caret { + font-size: 10px; + color: var(--text-secondary); + flex-shrink: 0; +} + +.model-dropdown { + position: absolute; + bottom: 100%; + left: 0; + margin-bottom: 4px; + min-width: 280px; + background: var(--bg-surface); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12); + z-index: 100; + padding: 4px; + max-height: 300px; + overflow-y: auto; +} + +.model-option { + padding: 8px 12px; + cursor: pointer; + border-radius: 4px; + font-size: 13px; + transition: background 0.1s ease; + white-space: nowrap; +} + +.model-option:hover { + background: var(--accent-bg); +} + +.model-option.active { + background: var(--accent); + color: var(--bg-dark); + font-weight: 500; +} + +.model-option-custom { + border-top: 1px solid var(--border-color); + margin-top: 4px; + padding-top: 10px; + display: flex; + justify-content: space-between; + align-items: center; +} + +.model-option-custom::after { + content: '+'; + font-size: 14px; + color: var(--text-secondary); +} + +/* --- Custom Model Modal --- */ +.modal-content-sm { + width: 420px; + max-width: 90%; + height: auto; +} + +.custom-model-form { + display: flex; + flex-direction: column; + gap: 16px; +} + +.form-group { + display: flex; + flex-direction: column; + gap: 6px; +} + +.form-group label { + font-size: 12px; + font-weight: 500; + color: var(--text-secondary); +} + +.form-select { + padding: 0.5rem 0.75rem; + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + background: var(--bg-panel); + color: var(--text-primary); + font-size: 13px; + cursor: pointer; +} + +.form-select:focus { + outline: none; + border-color: #666; +} + +.custom-model-actions { + display: flex; + justify-content: flex-end; + gap: 8px; + margin-top: 8px; +} + +.btn-primary { + background: var(--accent); + color: var(--bg-dark); + border-color: var(--accent); +} + +.btn-primary:hover { + opacity: 0.9; +} + +.btn.active { + background: var(--accent); + color: var(--bg-dark); + border-color: var(--accent); +} + +.btn.active:hover { + opacity: 0.9; + background: var(--accent); +} + +/* --- Agent Selector Dropdown in Header --- */ +.agent-selector-wrapper { + position: absolute; + left: 50%; + transform: translateX(-50%); + z-index: 90; +} + +.agent-selector-btn { + background: transparent; + border: none; + color: var(--text-primary); + font-weight: 600; + font-size: 16px; + letter-spacing: -0.02em; + cursor: pointer; + display: flex; + align-items: center; + gap: 8px; + padding: 4px 12px; + border-radius: var(--radius-md); + transition: background 0.15s ease; +} + +.agent-selector-btn:hover { + background: var(--accent-bg); +} + +.agent-selector-caret { + font-size: 10px; + color: var(--text-secondary); +} + +.agent-dropdown { + position: absolute; + top: 100%; + left: 50%; + transform: translateX(-50%); + margin-top: 6px; + min-width: 220px; + background: var(--bg-surface); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12); + z-index: 100; + padding: 4px; +} + +.agent-option { + padding: 8px 16px; + cursor: pointer; + border-radius: 4px; + font-size: 13px; + font-weight: 500; + transition: background 0.15s ease; + text-align: center; +} + +.agent-option:hover { + background: var(--accent-bg); +} + +.agent-option.active { + background: var(--accent); + color: var(--bg-dark); + font-weight: 600; +} + +.btn svg, .eval-toggle-btn svg, .debug-toggle-btn svg { + vertical-align: middle; + display: inline-block; + flex-shrink: 0; +} + +/* Collapsible Thinking Trace */ +.message.thought { + align-self: flex-start; + background: var(--accent-bg); + border: 1px dashed var(--border-color); + color: var(--text-secondary); + max-width: 85%; +} + +details.message.thought { + padding: 0.5rem 0.75rem; + border-radius: var(--radius-md); +} + +details.message.thought summary { + cursor: pointer; + font-weight: 600; + color: var(--text-secondary); + outline: none; + user-select: none; +} + +.thought-content { + margin-top: 0.5rem; + font-style: italic; + white-space: pre-wrap; + color: var(--text-secondary); + opacity: 0.9; +} + +/* --- Context Memory Log Styles --- */ +#memory-log { + overflow-y: auto; + flex: 1; + display: flex; + flex-direction: column; + gap: 12px; + padding: 16px; + background: var(--bg-dark); +} + +/* --- Shared Context Card Styles (used by both main page and logs page) --- */ +.context-card { + background: var(--bg-surface); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + padding: 12px 14px; + font-size: 12px; + line-height: 1.45; + display: flex; + flex-direction: column; + gap: 8px; +} + +.context-card.role-system { + border: 1px dashed #9b59b6; + background: rgba(155, 89, 182, 0.02); +} + +.context-card.role-user { + border-left: 3px solid #3498db; +} + +.context-card.role-model { + border-left: 3px solid #2ecc71; +} + +.context-card-header { + display: flex; + justify-content: space-between; + font-size: 10px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; +} + +.part-text-block { + white-space: pre-wrap; + word-break: break-word; +} + +.part-text-block mark { + background: rgba(255, 215, 0, 0.4); + border-radius: 2px; + color: inherit; +} + +.internal-monologue-box { + background: rgba(169, 169, 169, 0.08); + border: 1px solid var(--border-color); + border-radius: 4px; + margin-top: 4px; + overflow: hidden; +} + +.monologue-trigger { + padding: 6px 10px; + font-size: 11px; + font-weight: 600; + color: var(--text-secondary); + cursor: pointer; + background: var(--bg-dark); + display: flex; + align-items: center; + justify-content: space-between; + user-select: none; +} + +.monologue-trigger::after { + content: '▼'; + font-size: 8px; + transition: transform 0.2s ease; +} + +.internal-monologue-box.collapsed .monologue-trigger::after { + transform: rotate(-90deg); +} + +.monologue-content { + padding: 10px; + font-family: ui-monospace, monospace; + font-size: 11px; + white-space: pre-wrap; + color: var(--text-secondary); + border-top: 1px solid var(--border-color); +} + +.internal-monologue-box.collapsed .monologue-content { + display: none; +} + +.tool-block { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 11px; + background: var(--accent-bg); + color: var(--text-primary); + padding: 8px 12px; + border-radius: var(--radius-sm); + border: 1px solid var(--border-color); + margin-top: 6px; + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.01); +} + +.tool-name { color: var(--text-primary); font-weight: 600; } +.tool-args { color: var(--text-secondary); font-family: monospace; } + +.context-inline-image { + max-width: 180px; + max-height: 180px; + border-radius: 4px; + border: 1px solid var(--border-color); + object-fit: contain; + margin-top: 6px; +} + +.memory-row { + display: flex; + flex-direction: column; + padding: 10px 14px; + border-radius: var(--radius-md); + max-width: 90%; + font-size: 12px; + line-height: 1.4; + border: 1px solid var(--border-color); + box-shadow: 0 1px 3px rgba(0,0,0,0.05); +} + +.memory-user { + align-self: flex-end; + background: var(--bg-surface); + border-right: 4px solid #3498db; +} + +.memory-model { + align-self: flex-start; + background: var(--bg-surface); + border-left: 4px solid #2ecc71; +} + +.memory-system { + align-self: center; + background: var(--accent-bg); + border-left: 4px solid var(--text-secondary); + max-width: 95%; +} + +.memory-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 6px; + font-size: 10px; + font-weight: bold; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.memory-header .timestamp { + font-weight: normal; +} + +.memory-parts { + list-style: none; + padding: 0; + margin: 0; + display: flex; + flex-direction: column; + gap: 6px; +} + +.memory-parts li { + white-space: pre-wrap; + word-break: break-word; +} + +.memory-inline-img { + display: block; + max-width: 150px; + max-height: 100px; + margin-top: 4px; + border-radius: 4px; + border: 1px solid var(--border-color); + cursor: pointer; + transition: opacity 0.15s ease; +} + +.memory-inline-img:hover { + opacity: 0.85; +} + +.chip { + display: inline-block; + font-size: 9px; + font-weight: bold; + padding: 1px 5px; + border-radius: 3px; + background: var(--accent-bg); + color: var(--text-secondary); + margin-right: 4px; + text-transform: uppercase; +} + +.chip.code-chip { + font-family: monospace; + background: #2c3e50; + color: #ecf0f1; +} + +/* Tool call details and response styles */ +.tool_call_details { + align-self: flex-start; + max-width: 90%; + padding: 0 !important; /* Override message padding */ + border: 1px solid var(--border-color); + margin-top: 4px; + margin-bottom: 4px; + background: var(--bg-panel); +} + +.tool_call_summary { + padding: 0.5rem 0.75rem; + color: var(--text-secondary); + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 11px; + cursor: pointer; + outline: none; + list-style: none; /* Hide default details arrow */ +} + +.tool_call_summary::-webkit-details-marker { + display: none; /* Hide default details arrow in Chrome/Safari */ +} + +.tool_call_summary::before { + content: "▶ "; + font-size: 9px; + display: inline-block; + transition: transform 0.2s ease; + margin-right: 4px; +} + +.tool_call_details[open] .tool_call_summary::before { + transform: rotate(90deg); +} + +.tool_response_content { + padding: 0.5rem 0.75rem; + border-top: 1px solid var(--border-color); + background: var(--bg-panel); + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 11px; + color: var(--text-primary); + white-space: pre-wrap; +} diff --git a/live-api/agent/uv.lock b/live-api/agent/uv.lock new file mode 100644 index 0000000..d1cde67 --- /dev/null +++ b/live-api/agent/uv.lock @@ -0,0 +1,1395 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/annotated-doc/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/annotated-doc/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/annotated-types/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/annotated-types/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53" }, +] + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/anyio/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/anyio/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/certifi/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/certifi/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775" }, +] + +[[package]] +name = "cffi" +version = "2.1.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-win32.whl", hash = "sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-win32.whl", hash = "sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-win32.whl", hash = "sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-win32.whl", hash = "sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cffi/cffi-2.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/charset-normalizer/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/click/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/click/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/colorama/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/colorama/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" }, +] + +[[package]] +name = "cryptography" +version = "49.0.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/cryptography/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/distro/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/distro/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/exceptiongroup/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/exceptiongroup/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598" }, +] + +[[package]] +name = "fastapi" +version = "0.139.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/fastapi/fastapi-0.139.2.tar.gz", hash = "sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/fastapi/fastapi-0.139.2-py3-none-any.whl", hash = "sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c" }, +] + +[[package]] +name = "google-ai-generativelanguage" +version = "0.6.15" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-ai-generativelanguage/google_ai_generativelanguage-0.6.15.tar.gz", hash = "sha256:8f6d9dc4c12b065fe2d0289026171acea5183ebf2d0b11cefe12f3821e159ec3" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-ai-generativelanguage/google_ai_generativelanguage-0.6.15-py3-none-any.whl", hash = "sha256:5a03ef86377aa184ffef3662ca28f19eeee158733e45d7947982eb953c6ebb6c" }, +] + +[[package]] +name = "google-api-core" +version = "2.25.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-api-core/google_api_core-2.25.2.tar.gz", hash = "sha256:1c63aa6af0d0d5e37966f157a77f9396d820fba59f9e43e9415bc3dc5baff300" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-api-core/google_api_core-2.25.2-py3-none-any.whl", hash = "sha256:e9a8f62d363dc8424a8497f4c2a47d6bcda6c16514c935629c257ab5d10210e7" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + +[[package]] +name = "google-api-python-client" +version = "2.198.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "httplib2" }, + { name = "uritemplate" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-api-python-client/google_api_python_client-2.198.0.tar.gz", hash = "sha256:dfe3e16fb241af6e9c460a33f65085b3450e05cea09364f6b5d8997fb7e43e2a" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-api-python-client/google_api_python_client-2.198.0-py3-none-any.whl", hash = "sha256:fabac935474e817da5e662ff61bf7139439d6f92b32d332a7318a2d45931e03e" }, +] + +[[package]] +name = "google-auth" +version = "2.56.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-auth/google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-auth/google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + +[[package]] +name = "google-auth-httplib2" +version = "0.4.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "google-auth" }, + { name = "httplib2" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-auth-httplib2/google_auth_httplib2-0.4.0.tar.gz", hash = "sha256:d5b030a204b7a4b4d553ba9ca701b62481ee2b74419325580be70f7d85ffed35" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-auth-httplib2/google_auth_httplib2-0.4.0-py3-none-any.whl", hash = "sha256:8e55cfafa3358cba85f6cad4a886138e88e158d71e7e5c9ee5936a5c1507fb91" }, +] + +[[package]] +name = "google-genai" +version = "2.13.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "google-auth", extra = ["requests"] }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "sniffio" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-genai/google_genai-2.13.0.tar.gz", hash = "sha256:94f0763a023e686041376210d5a7cafc9d3bc40c2d623d8839be0dc6bb3cab19" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-genai/google_genai-2.13.0-py3-none-any.whl", hash = "sha256:e1fd408db6864ac12f23eb7634df670934020c918112f4f886e5609d26dd92d3" }, +] + +[[package]] +name = "google-generativeai" +version = "0.8.6" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "google-ai-generativelanguage" }, + { name = "google-api-core" }, + { name = "google-api-python-client" }, + { name = "google-auth" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/google-generativeai/google_generativeai-0.8.6-py3-none-any.whl", hash = "sha256:37a0eaaa95e5bbf888828e20a4a1b2c196cc9527d194706e58a68ff388aeb0fa" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.75.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/googleapis-common-protos/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/googleapis-common-protos/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed" }, +] + +[[package]] +name = "grpcio" +version = "1.82.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1.tar.gz", hash = "sha256:707b24abd90fcb1e45bcc080577da1dbf9971d107490589b9539af8e1e77b4b5" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:91859d1cac5f47caec5fc40e9f827500cdb54ce5b36450dc9a65616b5af49c17" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c80c9741dcef192f669876a81957cf7713b441c2f0c43631350d75fa49321d31" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b89cff456796d2f0581783726ad017a2c70aff2d27b0f05504c34e2e417f7560" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:d6e8a08f7038ba7a77f71e250804e4aba84fe91d22cfc54ff43c07b7529c4728" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:50fd2fe83426b1b1c6cdc4d72d555223b7dddf8ce07c5bac218b13fc6d684c6f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b758540a24d5394a9c578bf9f6126389f474b106ac3d9df1d53de56cb14c9fd9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c4ba4aac238f685743575d9d700003ac16537cce26e7c774993134f530652464" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed6fc621d6f366c88a60f0b971d5afd21d441d9aa561ee688de5b7acdb2cf901" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-win32.whl", hash = "sha256:bd2f45e46fff5b91c10997d0743a987517a7dde67c64c592835c2dcaac66f587" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp310-cp310-win_amd64.whl", hash = "sha256:5e171d5f0d6a0af78ea7512783f170a44f80c165259d8773e3a354a7f991f2b5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:0ddb18a9a9e1f46692b3567ae4abb3f8d117ce6afea48650f8eca06d8ab5d06f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cf855b1af246720f567b0ce5d0724d45dfa4188eecc3296a2a69257b11b9e94b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb30cb13e25bc13cea70ffc69d6d90c49d36ea6c1d4549e6912f70177834cac" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1e822b2774f719c017cbe700b6e47173b6ae290fb84906f52a5a3c2c60b62e1e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5dafb1ece8ed45dee7c738f166ec82e19673221ed5ab8967f72858a4685345b2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e06503106e7271e0a49fd5a1ac04747f1e47e87d900476db6fe45bc87ee411f4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ff99bc8cafb6a952201c37b995f425e641c93ffa6e072258525feab57290141d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:644ae1b94266ac785330f4590a69e52b6a7eb73029043a02209db81c81397d69" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-win32.whl", hash = "sha256:e203d2e19d471630084a16c815616f8211dff21c268ab3c5f5bf38417832e074" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp311-cp311-win_amd64.whl", hash = "sha256:0d8299c285fe6cc6a1f56badf8d3bc5078c8d20273ee64bafa3783b4bc29a769" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:c09bd5fa0d5b1fbd773ec349fe61441c3e4ebf168c229aa7538a820bdfad6a58" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1eae24810720734598e3e6a1a528d5de0f265fe3fc86575e9ecce424b9ec7379" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a6bd5daf5bde7b24d7ad2cbaf8bf9eac620d96222016bb5e7ddde930dec0673f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1ecfde669cb687ac020d31ff76debe5dc7a62213335f02262eb6625628da1c03" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:011c8badee95734dee8bf05ce3464756a0ac3ebb8d443afd20c0e2b5e4640ad9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b85f4564926fb23114d239392bdcae200db1e6179629edd7d7ab0ab89c96a197" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2c0c8270833395644c3fe6b6a806397955a2bc0538000a19a78b90c05a6c16e0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2ba199205ff46c7778290fe1673c91ac8e7e45678dd5c86e9e56fa33ec8788f6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-win32.whl", hash = "sha256:06127691866e295c14e84a1fb86356dd962254f6abd0da4ca4b001eea9e89438" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp312-cp312-win_amd64.whl", hash = "sha256:1fa3223a3a2e1db74f4c2b255189eb7ea875dfba56e221d252ee3fc7b204778e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:b454a2d97bfab7565683a02345f86bd182ab69fd7c2bdb7414171e7538f266b1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3dde70abfc80b3be11de53ba0d601c439e7fb2afd3583ad1788d1146bec92fdc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5523099c98c292ea1ae08e617249db760c56a78f8deae879027fe7d1ffbcbf6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5e5c4dc0a59b0f8490a6bdfd6fc8395b9d8ad8a8407c7d67ca7b5bba15c0877f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c40d94ba820329cc191981bc22fa6f6eed0799c6d921f3c6709521d59d4a2fd7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c816180e31e273caaec6f8bd86a8392499d5bbb26f41da44e3dce48bde69095" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e31fd780b261830720cb70b0fd8f0aa51d49e75a66d7464ad2e31d4b765f2580" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d76152d7c31d7210d4a106e5d8b64da5bba5d6abf11be30e2f7b0a0c59bbcbf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-win32.whl", hash = "sha256:38e9dcb5258226fb3282630b31b16a968df52c8c6ad514af540646e0a4578f8a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp313-cp313-win_amd64.whl", hash = "sha256:3dbfb52c36d9511ac2b8e6c94fdde837b393ae520cc321f52a333a2deedf5a90" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:35f990f7784c8fd2872644f07f96ebb4d9e48e145a190ab80d0280af91a1bfb2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:46536a4a1f4434df3c851b9254ff6fc7df5705b273681a15ca277d5921c178a0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d6650a7c1ebb7921c70e12a385439a8118efb99e669fa9ed31cf25db1843937c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b8e110c66df5204c0506d6c8787b35d48b8b699ef5aa366d6c4d67325c67fe9a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f853eae07235a51a27bb5d6a9a175a59ca55dc9b99edc6ce2f76f07332d333ae" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:60b0f2c95337694fc094b77d9f60f50566c84b5677393e342eb98daeee242d98" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:b064fc444812bdaa9825d33c26f8d732d63ee6a5d78557c1faf92c98687fed27" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d7ede11d747b4e1bd05e3bc0260e155b65a88735a895a10f6521f19b889511e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-win32.whl", hash = "sha256:3d21f19838dc255ecbb79321b15ae9b98fbddff4c3d4aedb0a81bdd7f4ab572a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5" }, +] + +[[package]] +name = "grpcio-status" +version = "1.71.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio-status/grpcio_status-1.71.2.tar.gz", hash = "sha256:c7a97e176df71cdc2c179cd1847d7fc86cca5832ad12e9798d7fed6b7a1aab50" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/grpcio-status/grpcio_status-1.71.2-py3-none-any.whl", hash = "sha256:803c98cb6a8b7dc6dbb785b1111aed739f241ab5e9da0bba96888aa74704cfd3" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/h11/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/h11/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/httpcore/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/httpcore/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55" }, +] + +[[package]] +name = "httplib2" +version = "0.32.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/httplib2/httplib2-0.32.0.tar.gz", hash = "sha256:48a0ef30a42db65d8f3399045e1d09ab0ba66e3b9efc360d07f80ea55d286025" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/httplib2/httplib2-0.32.0-py3-none-any.whl", hash = "sha256:dc6705cacdf3fb0a2aba7629fa33c90fd93e30035db0c157325826be177e4816" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/httpx/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/httpx/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/idna/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/idna/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +resolution-markers = [ + "python_full_version == '3.11.*'", +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/numpy/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb" }, +] + +[[package]] +name = "physical-agent" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "fastapi" }, + { name = "google-genai" }, + { name = "google-generativeai" }, + { name = "grpcio" }, + { name = "httpx" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" }, marker = "python_full_version >= '3.12'" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "uvicorn" }, + { name = "websocket-client" }, + { name = "websockets" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "google-genai", specifier = ">=1.0.0" }, + { name = "google-generativeai", specifier = ">=0.8.0" }, + { name = "grpcio", specifier = ">=1.60.0" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "numpy", specifier = ">=1.26.0" }, + { name = "pillow", specifier = ">=10.0.0" }, + { name = "protobuf", specifier = ">=4.25.0" }, + { name = "uvicorn", specifier = ">=0.30.0" }, + { name = "websocket-client", specifier = ">=1.8.0" }, + { name = "websockets", specifier = ">=12.0" }, +] + +[[package]] +name = "pillow" +version = "12.3.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pillow/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a" }, +] + +[[package]] +name = "proto-plus" +version = "1.28.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/proto-plus/proto_plus-1.28.1.tar.gz", hash = "sha256:832e68e7fe064cf90ab153b6e5eb935b27891bb89aaeb68b115e9b702f6cb168" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/proto-plus/proto_plus-1.28.1-py3-none-any.whl", hash = "sha256:6660f5f1970874bdcfc3088b435188a36a37bd3596668f7d726417c4ae8cfbed" }, +] + +[[package]] +name = "protobuf" +version = "5.29.6" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/protobuf/protobuf-5.29.6.tar.gz", hash = "sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/protobuf/protobuf-5.29.6-cp310-abi3-win32.whl", hash = "sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/protobuf/protobuf-5.29.6-cp310-abi3-win_amd64.whl", hash = "sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/protobuf/protobuf-5.29.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/protobuf/protobuf-5.29.6-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/protobuf/protobuf-5.29.6-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/protobuf/protobuf-5.29.6-py3-none-any.whl", hash = "sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pyasn1/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pyasn1/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pyasn1-modules/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pyasn1-modules/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pycparser/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pycparser/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pydantic-core/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pyparsing/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/pyparsing/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/requests/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/requests/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/sniffio/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/sniffio/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2" }, +] + +[[package]] +name = "starlette" +version = "1.3.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/starlette/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/starlette/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/tenacity/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/tenacity/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55" }, +] + +[[package]] +name = "tqdm" +version = "4.69.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/tqdm/tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/tqdm/tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/typing-extensions/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/typing-extensions/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/typing-inspection/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/typing-inspection/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7" }, +] + +[[package]] +name = "uritemplate" +version = "4.2.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/uritemplate/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/uritemplate/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/urllib3/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/urllib3/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" }, +] + +[[package]] +name = "uvicorn" +version = "0.51.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +dependencies = [ + { name = "click" }, + { name = "h11" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/uvicorn/uvicorn-0.51.0.tar.gz", hash = "sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/uvicorn/uvicorn-0.51.0-py3-none-any.whl", hash = "sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websocket-client/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websocket-client/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef" }, +] + +[[package]] +name = "websockets" +version = "16.1.1" +source = { registry = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/simple/" } +sdist = { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57" } +wheels = [ + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-win32.whl", hash = "sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-win32.whl", hash = "sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-win32.whl", hash = "sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ab59169ace05dcb49a1d4118f0bde139557adf45091bd85747e36bf5de984dd1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5e3b7d601f6f84156b08cc4a5e541c2b50ad7b36cfc302b657a12477c904a5df" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cd2ca96a082a36964aca83e992f72abeb61b7306c1a6cba4c7d06a7b93750cac" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bae954c382e013d5ea5b190d2830526bfa45ad121c326da0049b8c769f185db6" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e09f753a169951eb4f28c2c774f71069304f66e7277e0f5a2892423599cfa854" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:024193f8551a2b0eafbdd160911012c4e6c228c28430c84433253299a9e42d6a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:aabe464bfd13bd25f4821faf111da6fefdc389f870265a53105580e45b0a2e49" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a28fcbc9b6baf54a2e23f8655f308e4ccc6afdd7266f8fe7954f320dcda0f785" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:79eace538c6a97e96d0d03d4f9d314f9677f5ed85a8a984992ffd90b13cb8a56" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:496af849a472b531f758dbd4d61338f5000538cb1a7b3d20d9d32a264517f509" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5283810d2646741a0d8da2aa733d6aefa0545809afccb2a5d105a26bc45125f1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:4e3b680b1e0a27457e727a0d572fd81dffa87b6dbf8b228ab57da64f7d85aead" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:69159730a823dde3ea8d08783e8d47ef135a6d7e8d44eb127e32b321c9db8e3e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-win32.whl", hash = "sha256:cfb70b4eb56cac4da0a83588f3ad50d46beb0690391082f3d4e2d488c70b68ea" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:d9531d9cbeac99af6f038fb1bc351403531f7d634a2c2e10e2f7c854c6ed5b68" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-win32.whl", hash = "sha256:195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-win32.whl", hash = "sha256:8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5" }, + { url = "https://us-python.pkg.dev/artifact-foundry-prod/ah-3p-staging-python/websockets/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3" }, +] diff --git a/live-api/spot/.config.example b/live-api/spot/.config.example new file mode 100644 index 0000000..c7ee22f --- /dev/null +++ b/live-api/spot/.config.example @@ -0,0 +1,13 @@ +# Copy this file to .config and fill in real values. +# The file is ignored by git. + +SPOT_HOSTNAME=192.168.80.3 +BOSDYN_CLIENT_USERNAME=user +BOSDYN_CLIENT_PASSWORD=replace-with-spot-password +SPOT_TAKE_LEASE=true + +GEMINI_API_KEY=replace-with-gemini-api-key + +FASTAPI_BASE_URL=http://127.0.0.1:8000 +HYDRATION_PORT=3000 +HYDRATION_AUTO_CONNECT=false diff --git a/live-api/spot/.gitignore b/live-api/spot/.gitignore new file mode 100644 index 0000000..2b9baaa --- /dev/null +++ b/live-api/spot/.gitignore @@ -0,0 +1,46 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Virtual environments +.venv +.venv/ +venv/ +env/ + +# uv local cache +.uv-cache/ + +# Local credentials/config +.config +.env +.env.* +apps/hydration/.config + +# Node apps +apps/hydration/node_modules/ +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime captures and generated local visualizations +apps/api/captures/ +apps/navigation/waypoints_map.html + +# OS/editor local files +.DS_Store +Thumbs.db +*.swp +*.swo +.idea/ +.vscode/ diff --git a/live-api/spot/README.md b/live-api/spot/README.md new file mode 100644 index 0000000..919230f --- /dev/null +++ b/live-api/spot/README.md @@ -0,0 +1,137 @@ +# spot-agent + +Python project managed with `uv` for Boston Dynamics Spot SDK and Gemini API work. + +## Environment + +Use the project-local cache so `uv` does not write outside this workspace: + +```bash +UV_CACHE_DIR=.uv-cache uv sync +``` + +Run Python commands through the locked environment: + +```bash +UV_CACHE_DIR=.uv-cache uv run python main.py +``` + +Run the named-place navigation app: + +```bash +UV_CACHE_DIR=.uv-cache uv run python -m apps.navigation.cli --help +``` + +## Dependencies + +Primary SDK packages: + +- `bosdyn-client` +- `bosdyn-mission` +- `bosdyn-choreography-client` +- `google-genai` + +Add future packages with: + +```bash +UV_CACHE_DIR=.uv-cache uv add +``` + +## Navigation App + +The navigation app lives in `apps/navigation`. It stores named GraphNav keypoints in +`apps/navigation/keypoints.json` and can command Spot to navigate to a saved place +by name. + +Set credentials with environment variables to avoid putting passwords in shell +history: + +```bash +export BOSDYN_CLIENT_USERNAME=user +export BOSDYN_CLIENT_PASSWORD=password +``` + +List saved places: + +```bash +UV_CACHE_DIR=.uv-cache uv run python -m apps.navigation.cli list +``` + +Register a place manually with a GraphNav waypoint ID: + +```bash +UV_CACHE_DIR=.uv-cache uv run python -m apps.navigation.cli register kitchen waypoint-id-123 +``` + +Sync named waypoints from the map currently loaded on Spot: + +```bash +UV_CACHE_DIR=.uv-cache uv run python -m apps.navigation.cli sync --hostname 192.168.80.3 +``` + +Navigate to a saved place: + +```bash +UV_CACHE_DIR=.uv-cache uv run python -m apps.navigation.cli go kitchen --hostname 192.168.80.3 --power-on --stand +``` + +## Manipulation App + +The manipulation APIs live in `apps/manipulation`. They cover arm deployment, +Gemini-based object detection, 2D-to-3D projection with Spot hand depth, native +Spot image-pixel picking, force-change detection, gripper opening, and arm stow. + +```bash +UV_CACHE_DIR=.uv-cache uv run python -m apps.manipulation.cli --help +``` + +## FastAPI Server + +The HTTP API server lives in `apps/api` and exposes navigation, waypoint, +visualization, arm, gripper, detection, pick, force, and lease endpoints. + +```bash +UV_CACHE_DIR=.uv-cache uv run uvicorn apps.api.main:app --host 127.0.0.1 --port 8000 +``` + +Open docs: + +```text +http://127.0.0.1:8000/docs +``` + +The server can hold Spot's lease across calls. Use `/lease/take` or call +`/connect` with `take_lease: true`. It does not command sit or power-off when +the server stops. + +The server also reads local credentials from `.config` if present. Copy +`.config.example` to `.config` and fill in the Spot password and Gemini API key: + +```text +BOSDYN_CLIENT_PASSWORD=... +GEMINI_API_KEY=... +``` + +## Hydration Service + +The drink delivery web app lives in `apps/hydration`. It is a separate Node app +serving a React UI and an order worker. The worker only controls Spot by calling +the FastAPI server. + +Run FastAPI first, then start the hydration app with Node 18 or newer: + +```bash +cd apps/hydration +npm start +``` + +Open: + +```text +http://127.0.0.1:3000 +``` + +Orders go through this sequence: navigate to `snack1`, detect the selected +drink with Gemini, open and rotate the gripper, approach and grasp the drink, +carry it to the selected waypoint, wait for delivery, then either serve the next +order or return to `home`. diff --git a/live-api/spot/apps/__init__.py b/live-api/spot/apps/__init__.py new file mode 100644 index 0000000..329438f --- /dev/null +++ b/live-api/spot/apps/__init__.py @@ -0,0 +1 @@ +"""Application packages for spot-agent.""" diff --git a/live-api/spot/apps/api/README.md b/live-api/spot/apps/api/README.md new file mode 100644 index 0000000..6d74a67 --- /dev/null +++ b/live-api/spot/apps/api/README.md @@ -0,0 +1,42 @@ +# Spot FastAPI Server + +Start the API server: + +```bash +export SPOT_HOSTNAME=192.168.80.3 +export BOSDYN_CLIENT_USERNAME=user +export BOSDYN_CLIENT_PASSWORD=password +export GEMINI_API_KEY=... + +UV_CACHE_DIR=.uv-cache uv run uvicorn apps.api.main:app --host 127.0.0.1 --port 8000 +``` + +The server keeps one Spot connection in process. If you call `/lease/take` or start +with `SPOT_TAKE_LEASE=true`, it keeps the lease alive until `/lease/release` or +server shutdown. It does not command sit or power-off on shutdown. + +Common calls: + +```bash +curl http://127.0.0.1:8000/health +curl -X POST http://127.0.0.1:8000/lease/take +curl http://127.0.0.1:8000/waypoints +curl -X POST http://127.0.0.1:8000/navigate \ + -H 'content-type: application/json' \ + -d '{"name":"peng-desk","take_lease":true}' +curl -X POST http://127.0.0.1:8000/arm/deploy \ + -H 'content-type: application/json' \ + -d '{"take_lease":true}' +curl -X POST http://127.0.0.1:8000/gripper/open \ + -H 'content-type: application/json' \ + -d '{"take_lease":true}' +curl -X POST http://127.0.0.1:8000/detect \ + -H 'content-type: application/json' \ + -d '{"instruction":"the red cup"}' +curl -X POST http://127.0.0.1:8000/pick \ + -H 'content-type: application/json' \ + -d '{"instruction":"the red cup","take_lease":true}' +curl -X POST http://127.0.0.1:8000/arm/stow \ + -H 'content-type: application/json' \ + -d '{"take_lease":true}' +``` diff --git a/live-api/spot/apps/api/__init__.py b/live-api/spot/apps/api/__init__.py new file mode 100644 index 0000000..c97e9d4 --- /dev/null +++ b/live-api/spot/apps/api/__init__.py @@ -0,0 +1,2 @@ +"""FastAPI server for Spot navigation and manipulation actions.""" + diff --git a/live-api/spot/apps/api/config.py b/live-api/spot/apps/api/config.py new file mode 100644 index 0000000..501dcc1 --- /dev/null +++ b/live-api/spot/apps/api/config.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import os +from pathlib import Path + + +CONFIG_PATHS = ( + Path(".config"), + Path(__file__).resolve().parents[2] / ".config", +) + + +def load_dot_config() -> dict[str, str]: + """Load simple KEY=VALUE config files without overriding explicit env vars.""" + values: dict[str, str] = {} + for path in CONFIG_PATHS: + if not path.is_file(): + continue + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + if not key: + continue + values[key] = value + os.environ.setdefault(key, value) + return values diff --git a/live-api/spot/apps/api/main.py b/live-api/spot/apps/api/main.py new file mode 100644 index 0000000..1743a49 --- /dev/null +++ b/live-api/spot/apps/api/main.py @@ -0,0 +1,768 @@ +from __future__ import annotations + +import os +import base64 +import logging +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from fastapi import FastAPI, HTTPException, Response +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field + +from apps.api.config import load_dot_config +from apps.api.session import SpotSession +from apps.manipulation.gemini_detector import DEFAULT_MODEL +from apps.manipulation.models import Pose3D +from apps.manipulation.spot_client import DEFAULT_COLOR_SOURCE, DEFAULT_DEPTH_SOURCE +from apps.navigation.visualizer import DEFAULT_MAP_PATH + + +app = FastAPI(title="Spot Agent API", version="0.1.0") +logger = logging.getLogger(__name__) +session = SpotSession() +TELEOP_HTML = Path(__file__).with_name("static") / "teleop.html" +DETECTION_HTML = Path(__file__).with_name("static") / "detection.html" +CAMERAS_HTML = Path(__file__).with_name("static") / "cameras.html" + + +class ConnectRequest(BaseModel): + hostname: str = Field(default="192.168.80.3") + username: str = Field(default="user") + password: str + take_lease: bool = False + + +class NavigateRequest(BaseModel): + name: str + command_duration: float = 30.0 + timeout: float = 180.0 + feedback_interval: float = 1.0 + power_on: bool = True + stand: bool = True + take_lease: bool = False + + +class LocalizeRequest(BaseModel): + waypoint_id: str | None = None + waypoint_name: str | None = None + fiducial_init: str = "nearest" + use_fiducial_id: int | None = None + refine_fiducial_result_with_icp: bool = True + do_ambiguity_check: bool = False + refine_with_visual_features: bool = False + verify_visual_features_quality: bool = False + max_distance: float | None = None + max_yaw: float | None = None + + +class LoadMapRequest(BaseModel): + path: str + replace_graph: bool = True + generate_new_anchoring: bool = False + take_lease: bool = False + + +class StandRequest(BaseModel): + power_on: bool = True + take_lease: bool = False + timeout: float = 10.0 + + +class SitRequest(BaseModel): + take_lease: bool = False + timeout: float = 10.0 + + +class TeleopVelocityRequest(BaseModel): + v_x: float = 0.0 + v_y: float = 0.0 + v_rot: float = 0.0 + duration: float = 0.6 + take_lease: bool = False + power_on: bool = False + stand: bool = False + body_follow_arm: bool = False + + +class TeleopStopRequest(BaseModel): + take_lease: bool = False + + +class StopActionsRequest(BaseModel): + take_lease: bool = False + freeze_arm: bool = True + + +class ArmFreezeRequest(BaseModel): + take_lease: bool = False + + +class ArmOscillationMonitorRequest(BaseModel): + enabled: bool + take_lease: bool = True + sample_interval: float = 0.1 + window_sec: float = 1.2 + min_peak_to_peak_m: float = 0.012 + min_direction_changes: int = 4 + min_speed_mps: float = 0.025 + freeze_cooldown_sec: float = 2.0 + + +class SyncWaypointsRequest(BaseModel): + overwrite: bool = False + prune_stale: bool = False + + +class VisualizeRequest(BaseModel): + include_point_clouds: bool = True + + +class LeaseRequest(BaseModel): + take: bool = False + + +class ArmRequest(BaseModel): + take_lease: bool = False + timeout: float = 10.0 + + +class DeployArmRequest(ArmRequest): + power_on: bool = True + + +class ArmJogRequest(BaseModel): + dx: float = 0.0 + dy: float = 0.0 + dz: float = 0.0 + droll: float = 0.0 + dpitch: float = 0.0 + dyaw: float = 0.0 + seconds: float = 0.8 + take_lease: bool = False + timeout: float = 3.0 + + +class ArmCameraRollRequest(BaseModel): + direction: str + angle_rad: float = 0.105 + seconds: float = 0.7 + take_lease: bool = True + timeout: float = 3.0 + + +class WaitForPickUpRequest(BaseModel): + monitor_sec: float = 30.0 + upward_threshold_m: float = 0.02 + sample_interval: float = 0.1 + open_duration_sec: float = 3.0 + take_lease: bool = True + gripper_timeout: float = 5.0 + stow_timeout: float = 10.0 + + +class PoseRequest(BaseModel): + frame_name: str = "vision" + x: float + y: float + z: float + qw: float = 1.0 + qx: float = 0.0 + qy: float = 0.0 + qz: float = 0.0 + + +class ArmApproachRequest(BaseModel): + pose: PoseRequest + standoff_m: float = 0.0 + seconds: float = 1.2 + take_lease: bool = False + timeout: float = 5.0 + max_step_m: float = 0.45 + + +class ArmReachDistanceRequest(BaseModel): + pose: PoseRequest + + +class OpenGripperRequest(BaseModel): + take_lease: bool = False + timeout: float = 5.0 + open_fraction: float = Field(default=1.0, ge=0.0, le=1.0) + max_vel: float | None = Field(default=None, gt=0.0) + max_acc: float | None = Field(default=None, gt=0.0) + + +class DetectRequest(BaseModel): + instruction: str + model: str = DEFAULT_MODEL + api_key: str | None = None + color_source: str = DEFAULT_COLOR_SOURCE + depth_source: str = DEFAULT_DEPTH_SOURCE + include_point_cloud: bool = True + point_cloud_stride: int = Field(default=4, ge=1, le=32) + max_point_cloud_points: int = Field(default=6000, ge=0, le=50000) + + +class DetectPickTargetRequest(BaseModel): + instruction: str + model: str = DEFAULT_MODEL + api_key: str | None = None + + +class PickRequest(BaseModel): + instruction: str + model: str = DEFAULT_MODEL + api_key: str | None = None + take_lease: bool = False + timeout: float = 30.0 + grip_max_torque_nm: float = Field(default=2.0, ge=0.5, le=5.5) + + +class ForceChangeRequest(BaseModel): + threshold_newtons: float = 5.0 + sample_window_sec: float = 3.0 + interval_sec: float = 0.1 + + +class GraspPixelRequest(BaseModel): + x: int = Field(ge=0, le=1000) + y: int = Field(ge=0, le=1000) + take_lease: bool = True + max_depth_m: float = Field(default=2.0, ge=0.2, le=3.0) + grip_max_torque_nm: float = Field(default=2.0, ge=0.5, le=5.5) + + +class Scan360Request(BaseModel): + duration: float = 8.0 + camera_source: str = "hand_color_image" + + +class PlacePixelRequest(BaseModel): + x: int = Field(ge=0, le=1000) + y: int = Field(ge=0, le=1000) + take_lease: bool = True + standoff_m: float = Field(default=0.05, ge=0.03, le=0.15) + settle_time_sec: float = Field(default=0.5, ge=0.0, le=3.0) + + +class StowSmartRequest(BaseModel): + take_lease: bool = True + timeout: float = 20.0 + + +@app.on_event("startup") +def startup_connect() -> None: + load_dot_config() + hostname = os.getenv("SPOT_HOSTNAME") or os.getenv("BOSDYN_CLIENT_HOSTNAME") + username = os.getenv("BOSDYN_CLIENT_USERNAME") + password = os.getenv("BOSDYN_CLIENT_PASSWORD") + if hostname and username and password: + try: + session.connect( + hostname=hostname, + username=username, + password=password, + take_lease=os.getenv("SPOT_TAKE_LEASE", "").lower() in {"1", "true", "yes"}, + ) + except Exception as exc: # pylint: disable=broad-except + logger.warning( + "Spot startup connection failed; API will remain available: %s: %s", + type(exc).__name__, + exc, + ) + + +@app.on_event("shutdown") +def shutdown() -> None: + session.stop_arm_oscillation_monitor() + session.shutdown_lease() + + +def _run(action): + try: + return action() + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=500, detail=f"{type(exc).__name__}: {exc}") from exc + + +@app.get("/health") +def health() -> dict[str, Any]: + return {"ok": True, **session.status()} + + +@app.post("/connect") +def connect(request: ConnectRequest) -> dict[str, Any]: + return _run(lambda: session.connect( + hostname=request.hostname, + username=request.username, + password=request.password, + take_lease=request.take_lease, + )) + + +@app.get("/lease") +def lease_status() -> dict[str, Any]: + return session.status() + + +@app.get("/localization") +def localization_status() -> dict[str, Any]: + return _run(session.localization_status) + + +@app.post("/localize") +def localize(request: LocalizeRequest) -> dict[str, Any]: + return _run(lambda: session.localize( + waypoint_id=request.waypoint_id, + waypoint_name=request.waypoint_name, + fiducial_init=request.fiducial_init, + use_fiducial_id=request.use_fiducial_id, + refine_fiducial_result_with_icp=request.refine_fiducial_result_with_icp, + do_ambiguity_check=request.do_ambiguity_check, + refine_with_visual_features=request.refine_with_visual_features, + verify_visual_features_quality=request.verify_visual_features_quality, + max_distance=request.max_distance, + max_yaw=request.max_yaw, + )) + + +@app.post("/map/load") +def load_map(request: LoadMapRequest) -> dict[str, Any]: + return _run(lambda: session.load_map( + request.path, + replace_graph=request.replace_graph, + generate_new_anchoring=request.generate_new_anchoring, + take_lease=request.take_lease, + )) + + +@app.get("/lease/details") +def lease_details() -> dict[str, str]: + return _run(lambda: {"leases": session.list_leases()}) + + +@app.get("/battery") +def battery() -> dict[str, Any]: + return _run(session.battery_status) + + +@app.post("/faults/behavior/clear") +def clear_behavior_faults() -> dict[str, Any]: + return _run(session.clear_behavior_faults) + + +@app.post("/lease/acquire") +def acquire_lease() -> dict[str, Any]: + return _run(session.acquire_lease) + + +@app.post("/lease/take") +def take_lease() -> dict[str, Any]: + return _run(session.take_lease) + + +@app.post("/lease/release") +def release_lease() -> dict[str, Any]: + session.shutdown_lease() + return session.status() + + +@app.get("/waypoints") +def waypoints() -> list[dict[str, Any]]: + return session.list_waypoints() + + +@app.post("/waypoints/sync") +def sync_waypoints(request: SyncWaypointsRequest) -> dict[str, Any]: + return _run(lambda: session.sync_waypoints( + overwrite=request.overwrite, + prune_stale=request.prune_stale, + )) + + +@app.post("/navigate") +def navigate(request: NavigateRequest) -> dict[str, Any]: + return _run(lambda: session.navigate( + request.name, + command_duration=request.command_duration, + timeout=request.timeout, + feedback_interval=request.feedback_interval, + power_on=request.power_on, + stand=request.stand, + take_lease=request.take_lease, + )) + + +@app.post("/stand") +def stand(request: StandRequest) -> dict[str, Any]: + return _run(lambda: session.stand( + power_on=request.power_on, + take_lease=request.take_lease, + timeout=request.timeout, + )) + + +@app.post("/sit") +def sit(request: SitRequest) -> dict[str, Any]: + return _run(lambda: session.sit( + take_lease=request.take_lease, + timeout=request.timeout, + )) + + +@app.get("/teleop") +def teleop_page(): + return FileResponse(TELEOP_HTML) + + +@app.get("/detection") +def detection_page(): + return FileResponse(DETECTION_HTML) + + +@app.get("/cameras") +def cameras_page(): + return FileResponse(CAMERAS_HTML) + + +@app.post("/teleop/velocity") +def teleop_velocity(request: TeleopVelocityRequest) -> dict[str, Any]: + return _run(lambda: session.velocity( + v_x=request.v_x, + v_y=request.v_y, + v_rot=request.v_rot, + duration=request.duration, + take_lease=request.take_lease, + power_on=request.power_on, + stand=request.stand, + body_follow_arm=request.body_follow_arm, + )) + + +@app.post("/teleop/stop") +def teleop_stop(request: TeleopStopRequest) -> dict[str, Any]: + return _run(lambda: session.stop(take_lease=request.take_lease)) + + +@app.post("/actions/stop") +def stop_actions(request: StopActionsRequest) -> dict[str, Any]: + return _run(lambda: session.stop_all_actions( + take_lease=request.take_lease, + freeze_arm=request.freeze_arm, + )) + + +@app.post("/visualize") +def visualize(request: VisualizeRequest) -> dict[str, Any]: + return _run(lambda: session.visualize(include_point_clouds=request.include_point_clouds)) + + +@app.get("/visualize/map") +def visualization_map(): + if not DEFAULT_MAP_PATH.exists(): + raise HTTPException(status_code=404, detail="No visualization has been generated yet.") + return FileResponse(DEFAULT_MAP_PATH) + + +@app.get("/images/sources") +def image_sources() -> list[dict[str, Any]]: + return _run(session.image_sources) + + +def _camera_image_payload(source: str, quality_percent: int) -> tuple[bytes, str]: + image_response = _run(lambda: session.capture_image(source, quality_percent=quality_percent)) + image = image_response.shot.image + media_type = "application/octet-stream" + if image.format == image.FORMAT_JPEG: + media_type = "image/jpeg" + return image.data, media_type + + +@app.get("/images/{source}") +def camera_image(source: str, quality_percent: int = 85): + data, media_type = _camera_image_payload(source, quality_percent) + return Response( + content=data, + media_type=media_type, + headers={"Cache-Control": "no-store"}, + ) + + +@app.get("/camera_image/") +def eden_camera_image(camera_id: str, quality_percent: int = 85): + """Return the HTML data-URI snapshot expected by Eden RobotClient.""" + data, media_type = _camera_image_payload(camera_id, quality_percent) + encoded = base64.b64encode(data).decode("ascii") + html = f'camera' + return Response( + content=html, + media_type="text/html", + headers={"Cache-Control": "no-store"}, + ) + + +@app.post("/arm/deploy") +def deploy_arm(request: DeployArmRequest) -> dict[str, Any]: + return _run(lambda: session.deploy_arm( + power_on=request.power_on, + take_lease=request.take_lease, + timeout=request.timeout, + )) + + +@app.post("/arm/carry") +def carry_arm(request: ArmRequest) -> dict[str, Any]: + return _run(lambda: session.carry_arm(take_lease=request.take_lease, timeout=request.timeout)) + + +@app.post("/arm/freeze") +def freeze_arm(request: ArmFreezeRequest) -> dict[str, Any]: + return _run(lambda: session.freeze_arm(take_lease=request.take_lease)) + + +@app.get("/arm/oscillation-monitor") +def arm_oscillation_monitor_status() -> dict[str, Any]: + return session.arm_oscillation_monitor_status() + + +@app.post("/arm/oscillation-monitor") +def configure_arm_oscillation_monitor(request: ArmOscillationMonitorRequest) -> dict[str, Any]: + return _run(lambda: session.configure_arm_oscillation_monitor( + enabled=request.enabled, + take_lease=request.take_lease, + sample_interval=request.sample_interval, + window_sec=request.window_sec, + min_peak_to_peak_m=request.min_peak_to_peak_m, + min_direction_changes=request.min_direction_changes, + min_speed_mps=request.min_speed_mps, + freeze_cooldown_sec=request.freeze_cooldown_sec, + )) + + +@app.post("/arm/jog") +def jog_arm(request: ArmJogRequest) -> dict[str, Any]: + return _run(lambda: session.jog_arm( + dx=request.dx, + dy=request.dy, + dz=request.dz, + droll=request.droll, + dpitch=request.dpitch, + dyaw=request.dyaw, + seconds=request.seconds, + take_lease=request.take_lease, + timeout=request.timeout, + )) + + +@app.post("/arm/camera-roll") +def camera_roll_arm(request: ArmCameraRollRequest) -> dict[str, Any]: + return _run(lambda: session.roll_camera_view( + direction=request.direction, + angle_rad=request.angle_rad, + seconds=request.seconds, + take_lease=request.take_lease, + timeout=request.timeout, + )) + + +@app.post("/arm/reach-distance") +def reach_distance(request: ArmReachDistanceRequest) -> dict[str, Any]: + return _run(lambda: session.reach_distance_to_pose(Pose3D( + frame_name=request.pose.frame_name, + x=request.pose.x, + y=request.pose.y, + z=request.pose.z, + qw=request.pose.qw, + qx=request.pose.qx, + qy=request.pose.qy, + qz=request.pose.qz, + ))) + + +@app.post("/arm/approach") +def approach_arm(request: ArmApproachRequest) -> dict[str, Any]: + return _run(lambda: session.approach_pose( + Pose3D( + frame_name=request.pose.frame_name, + x=request.pose.x, + y=request.pose.y, + z=request.pose.z, + qw=request.pose.qw, + qx=request.pose.qx, + qy=request.pose.qy, + qz=request.pose.qz, + ), + standoff_m=request.standoff_m, + seconds=request.seconds, + take_lease=request.take_lease, + timeout=request.timeout, + max_step_m=request.max_step_m, + )) + + +@app.post("/arm/approach-whole-body") +def approach_arm_whole_body(request: ArmApproachRequest) -> dict[str, Any]: + return _run(lambda: session.approach_pose_whole_body( + Pose3D( + frame_name=request.pose.frame_name, + x=request.pose.x, + y=request.pose.y, + z=request.pose.z, + qw=request.pose.qw, + qx=request.pose.qx, + qy=request.pose.qy, + qz=request.pose.qz, + ), + standoff_m=request.standoff_m, + seconds=request.seconds, + take_lease=request.take_lease, + timeout=request.timeout, + max_step_m=request.max_step_m, + )) + + +@app.post("/arm/stow") +def stow_arm(request: ArmRequest) -> dict[str, Any]: + return _run(lambda: session.stow_arm(take_lease=request.take_lease, timeout=request.timeout)) + + +@app.post("/pickup/wait") +def wait_for_pick_up(request: WaitForPickUpRequest) -> dict[str, Any]: + return _run(lambda: session.wait_for_pick_up( + monitor_sec=request.monitor_sec, + upward_threshold_m=request.upward_threshold_m, + sample_interval=request.sample_interval, + open_duration_sec=request.open_duration_sec, + take_lease=request.take_lease, + gripper_timeout=request.gripper_timeout, + stow_timeout=request.stow_timeout, + )) + + +@app.post("/delivery/wait", include_in_schema=False) +def wait_for_delivery_compat(request: WaitForPickUpRequest) -> dict[str, Any]: + """Compatibility alias for clients using the former route.""" + return wait_for_pick_up(request) + + +@app.post("/gripper/open") +def open_gripper(request: OpenGripperRequest) -> dict[str, Any]: + return _run(lambda: session.open_gripper( + take_lease=request.take_lease, + timeout=request.timeout, + open_fraction=request.open_fraction, + max_vel=request.max_vel, + max_acc=request.max_acc, + )) + + +@app.post("/gripper/close") +def close_gripper(request: OpenGripperRequest) -> dict[str, Any]: + return _run(lambda: session.close_gripper( + take_lease=request.take_lease, + timeout=request.timeout, + max_vel=request.max_vel, + max_acc=request.max_acc, + )) + + +@app.post("/detect") +def detect(request: DetectRequest) -> dict[str, Any]: + def action(): + scene = session.detect_scene( + request.instruction, + model=request.model, + api_key=request.api_key, + color_source=request.color_source, + depth_source=request.depth_source, + point_cloud_stride=request.point_cloud_stride, + max_point_cloud_points=request.max_point_cloud_points if request.include_point_cloud else 0, + ) + image = scene["image"] + image_bytes = image.pop("data") + image["data_url"] = ( + f"data:{image['mime_type']};base64," + f"{base64.b64encode(image_bytes).decode('ascii')}" + ) + if not request.include_point_cloud: + scene["point_cloud"]["points"] = [] + return scene + + return _run(action) + + +@app.post("/detect/pick-target") +def detect_pick_target(request: DetectPickTargetRequest) -> dict[str, Any]: + return _run(lambda: session.detect_pick_target( + request.instruction, + model=request.model, + api_key=request.api_key, + )) + + +@app.post("/pick") +def pick(request: PickRequest) -> dict[str, Any]: + return _run(lambda: session.pick( + request.instruction, + model=request.model, + api_key=request.api_key, + take_lease=request.take_lease, + timeout=request.timeout, + grip_max_torque_nm=request.grip_max_torque_nm, + )) + + +@app.post("/force/change") +def force_change(request: ForceChangeRequest) -> dict[str, Any]: + result = _run(lambda: session.detect_external_force_change( + threshold_newtons=request.threshold_newtons, + sample_window_sec=request.sample_window_sec, + interval_sec=request.interval_sec, + )) + return asdict(result) + + +@app.post("/manipulation/grasp-pixel") +def grasp_pixel(request: GraspPixelRequest) -> dict[str, Any]: + return _run(lambda: session.grasp_at_pixel( + x_pct=request.x / 1000.0, + y_pct=request.y / 1000.0, + take_lease=request.take_lease, + max_depth_m=request.max_depth_m, + grip_max_torque_nm=request.grip_max_torque_nm, + )) + + +@app.post("/teleop/scan-360") +def scan_360(request: Scan360Request) -> dict[str, Any]: + return _run(lambda: session.execute_360_scan( + duration=request.duration, + camera_source=request.camera_source, + )) + + +@app.post("/manipulation/place-pixel") +def place_pixel(request: PlacePixelRequest) -> dict[str, Any]: + return _run(lambda: session.place_at_pixel( + x_pct=request.x / 1000.0, + y_pct=request.y / 1000.0, + take_lease=request.take_lease, + standoff_m=request.standoff_m, + settle_time_sec=request.settle_time_sec, + )) + + +@app.post("/arm/stow-smart") +def stow_smart(request: StowSmartRequest) -> dict[str, Any]: + return _run(lambda: session.stow_smart( + take_lease=request.take_lease, + timeout=request.timeout, + )) diff --git a/live-api/spot/apps/api/main_test.py b/live-api/spot/apps/api/main_test.py new file mode 100644 index 0000000..113f6fd --- /dev/null +++ b/live-api/spot/apps/api/main_test.py @@ -0,0 +1,114 @@ +"""Tests for Spot API static debug pages.""" + +import unittest +from unittest.mock import patch + +from apps.api import main + + +class CameraDebugPageTest(unittest.TestCase): + + def test_camera_debug_page_is_served(self): + response = main.cameras_page() + + self.assertEqual(main.CAMERAS_HTML, response.path) + self.assertTrue(main.CAMERAS_HTML.is_file()) + + def test_camera_debug_page_discovers_and_renders_depth_sources(self): + html = main.CAMERAS_HTML.read_text(encoding="utf-8") + + self.assertIn("/images/sources", html) + self.assertIn("renderDepth", html) + self.assertIn("runLimited(sources, 3", html) + + +class StartupConnectionTest(unittest.TestCase): + + @patch.object(main, "load_dot_config") + @patch.object(main.session, "connect", side_effect=RuntimeError("offline")) + @patch.dict( + main.os.environ, + { + "SPOT_HOSTNAME": "192.0.2.1", + "BOSDYN_CLIENT_USERNAME": "user", + "BOSDYN_CLIENT_PASSWORD": "password", + }, + ) + def test_robot_connection_failure_does_not_abort_api_startup( + self, connect, _load_dot_config + ): + main.startup_connect() + + connect.assert_called_once() + + +class LightweightDetectionEndpointTest(unittest.TestCase): + + @patch.object(main.session, "detect_pick_target") + def test_dispatches_compact_pick_target_detection(self, detect_pick_target): + detect_pick_target.return_value = { + "detected": True, + "target": {"normalized_x": 500, "normalized_y": 400}, + } + + result = main.detect_pick_target( + main.DetectPickTargetRequest(instruction="red cube") + ) + + self.assertTrue(result["detected"]) + detect_pick_target.assert_called_once_with( + "red cube", + model=main.DEFAULT_MODEL, + api_key=None, + ) + + +class CommandEndpointDispatchTest(unittest.TestCase): + + @patch.object(main.session, "stand") + def test_stand_dispatches_only_stand_parameters(self, stand): + stand.return_value = {"standing": True} + + result = main.stand(main.StandRequest(take_lease=True)) + + self.assertTrue(result["standing"]) + stand.assert_called_once_with( + power_on=True, + take_lease=True, + timeout=10.0, + ) + + @patch.object(main.session, "pick") + def test_pick_dispatches_light_grip_torque(self, pick): + pick.return_value = {"state": "MANIP_STATE_GRASP_SUCCEEDED"} + + main.pick(main.PickRequest(instruction="red cube")) + + pick.assert_called_once_with( + "red cube", + model=main.DEFAULT_MODEL, + api_key=None, + take_lease=False, + timeout=30.0, + grip_max_torque_nm=2.0, + ) + + @patch.object(main.session, "wait_for_pick_up") + def test_wait_for_pick_up_dispatches_monitor_parameters(self, wait_for_pick_up): + wait_for_pick_up.return_value = {"triggered": False, "reason": "timeout"} + + main.wait_for_pick_up(main.WaitForPickUpRequest()) + + wait_for_pick_up.assert_called_once_with( + monitor_sec=30.0, + upward_threshold_m=0.02, + sample_interval=0.1, + open_duration_sec=3.0, + take_lease=True, + gripper_timeout=5.0, + stow_timeout=10.0, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/spot/apps/api/session.py b/live-api/spot/apps/api/session.py new file mode 100644 index 0000000..5c25d30 --- /dev/null +++ b/live-api/spot/apps/api/session.py @@ -0,0 +1,1954 @@ +from __future__ import annotations + +import math +import threading +import time +from collections import deque +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import bosdyn.client +import numpy as np +from bosdyn.api import geometry_pb2, gripper_command_pb2, image_pb2, manipulation_api_pb2, robot_state_pb2 +from bosdyn.api.graph_nav import graph_nav_pb2, map_pb2, nav_pb2 +from bosdyn.client import frame_helpers +from bosdyn.client.graph_nav import GraphNavClient +from bosdyn.client.image import ImageClient, build_image_request, pixel_to_camera_space +from bosdyn.client.lease import ( + LeaseClient, + LeaseKeepAlive, + LeaseNotOwnedByWallet, + LeaseResponseError, +) +from bosdyn.client.manipulation_api_client import ManipulationApiClient +from bosdyn.client.math_helpers import Quat +from bosdyn.client.power import PowerClient, power_on_motors +from bosdyn.client.robot_command import ( + RobotCommandBuilder, + RobotCommandClient, + blocking_sit, + blocking_stand, + block_until_arm_arrives, +) +from bosdyn.client.robot_state import RobotStateClient + +from apps.manipulation.gemini_detector import GeminiObjectDetector +from apps.manipulation.models import Detection2D, ForceChange, ImageObservation, Pose3D +from apps.manipulation.spot_client import ( + DEFAULT_COLOR_SOURCE, + DEFAULT_DEPTH_SOURCE, + _depth_at_pixel, + constrain_grasp_to_top_down, +) +from apps.navigation.registry import KeypointRegistry +from apps.navigation.visualizer import DEFAULT_MAP_PATH, write_graph_html + + +MAX_LINEAR_SPEED_MPS = 0.8 +MAX_SIDEWAYS_SPEED_MPS = 0.5 +MAX_ANGULAR_SPEED_RADPS = 1.0 +SUCCESS_STATUS = graph_nav_pb2.NavigationFeedbackResponse.STATUS_REACHED_GOAL +ACTIVE_STATUSES = { + graph_nav_pb2.NavigationFeedbackResponse.STATUS_UNKNOWN, + graph_nav_pb2.NavigationFeedbackResponse.STATUS_FOLLOWING_ROUTE, +} +TERMINAL_MANIP_STATES = { + manipulation_api_pb2.MANIP_STATE_DONE, + manipulation_api_pb2.MANIP_STATE_GRASP_SUCCEEDED, + manipulation_api_pb2.MANIP_STATE_GRASP_FAILED, + manipulation_api_pb2.MANIP_STATE_GRASP_PLANNING_NO_SOLUTION, + manipulation_api_pb2.MANIP_STATE_PLACE_SUCCEEDED, + manipulation_api_pb2.MANIP_STATE_PLACE_FAILED, +} + + +class SpotSession: + """Persistent Spot connection and lease holder used by the HTTP API.""" + + def __init__(self, registry: KeypointRegistry | None = None): + self.registry = registry or KeypointRegistry() + self.lock = threading.RLock() + self.hostname: str | None = None + self.robot = None + self.graph_nav = None + self.image = None + self.lease = None + self.lease_keepalive: LeaseKeepAlive | None = None + self.manipulation = None + self.power = None + self.command = None + self.state = None + self.cancel_actions_event = threading.Event() + self.arm_oscillation_monitor_stop = threading.Event() + self.arm_oscillation_monitor_thread: threading.Thread | None = None + self.arm_oscillation_monitor_config: dict[str, Any] = {} + self.arm_oscillation_monitor_state: dict[str, Any] = { + "enabled": False, + "samples": 0, + "last_detection": None, + "last_freeze": None, + "last_error": None, + } + + @property + def connected(self) -> bool: + return self.robot is not None + + @property + def holding_lease(self) -> bool: + return self.lease_keepalive is not None and self.lease_keepalive.is_alive() + + def connect(self, hostname: str, username: str, password: str, *, take_lease: bool = False) -> dict[str, Any]: + self.stop_arm_oscillation_monitor() + with self.lock: + self.shutdown_lease() + sdk = bosdyn.client.create_standard_sdk("spot-fastapi-server") + robot = sdk.create_robot(hostname) + robot.authenticate(username, password) + robot.time_sync.wait_for_sync() + + self.hostname = hostname + self.robot = robot + self.graph_nav = robot.ensure_client(GraphNavClient.default_service_name) + self.image = robot.ensure_client(ImageClient.default_service_name) + self.lease = robot.ensure_client(LeaseClient.default_service_name) + self.manipulation = robot.ensure_client(ManipulationApiClient.default_service_name) + self.power = robot.ensure_client(PowerClient.default_service_name) + self.command = robot.ensure_client(RobotCommandClient.default_service_name) + self.state = robot.ensure_client(RobotStateClient.default_service_name) + + if take_lease: + self.take_lease() + return self.status() + + def status(self) -> dict[str, Any]: + return { + "connected": self.connected, + "hostname": self.hostname, + "holding_lease": self.holding_lease, + "arm_oscillation_monitor": self.arm_oscillation_monitor_status(), + } + + def battery_status(self) -> dict[str, Any]: + self.require_connected() + robot_state = self.state.get_robot_state() + power_state = robot_state.power_state + if hasattr(power_state, "locomotion_charge_percentage"): + runtime = None + if power_state.HasField("locomotion_estimated_runtime"): + runtime = ( + power_state.locomotion_estimated_runtime.seconds + + power_state.locomotion_estimated_runtime.nanos / 1e9 + ) + charge = ( + power_state.locomotion_charge_percentage.value + if power_state.HasField("locomotion_charge_percentage") + else None + ) + return { + "charge_percentage": charge, + "estimated_runtime_sec": runtime, + "motor_power_state": robot_state_pb2.PowerState.MotorPowerState.Name( + power_state.motor_power_state + ), + "shore_power_state": robot_state_pb2.PowerState.ShorePowerState.Name( + power_state.shore_power_state + ), + "robot_power_state": robot_state_pb2.PowerState.RobotPowerState.Name( + power_state.robot_power_state + ), + } + + batteries = [] + for battery in power_state.battery_states: + runtime = None + if battery.HasField("estimated_runtime"): + runtime = battery.estimated_runtime.seconds + battery.estimated_runtime.nanos / 1e9 + timestamp = None + if battery.HasField("timestamp"): + timestamp = battery.timestamp.seconds + battery.timestamp.nanos / 1e9 + batteries.append({ + "identifier": battery.identifier, + "charge_percentage": battery.charge_percentage.value, + "estimated_runtime_sec": runtime, + "current": battery.current.value if battery.HasField("current") else None, + "voltage": battery.voltage.value if battery.HasField("voltage") else None, + "temperatures": list(battery.temperatures), + "communications_loss_percent": battery.communications_loss_percent.value, + "status": robot_state_pb2.BatteryState.Status.Name(battery.status), + "timestamp": timestamp, + }) + charge_values = [ + item["charge_percentage"] + for item in batteries + if item["charge_percentage"] is not None + ] + return { + "batteries": batteries, + "charge_percentage": min(charge_values) if charge_values else None, + } + + def clear_behavior_faults(self) -> dict[str, Any]: + self.require_connected() + with self.lock: + robot_state = self.state.get_robot_state() + faults = list(robot_state.behavior_fault_state.faults) + cleared: list[dict[str, Any]] = [] + errors: list[dict[str, Any]] = [] + for fault in faults: + fault_id = fault.behavior_fault_id + fault_info = { + "behavior_fault_id": fault_id, + "cause": robot_state_pb2.BehaviorFault.Cause.Name(fault.cause), + "status": robot_state_pb2.BehaviorFault.Status.Name(fault.status), + } + try: + is_cleared = self.command.clear_behavior_fault(fault_id) + cleared.append({**fault_info, "cleared": bool(is_cleared)}) + except Exception as exc: + errors.append({**fault_info, "error": f"{type(exc).__name__}: {exc}"}) + + remaining_state = self.state.get_robot_state() + remaining = [ + { + "behavior_fault_id": fault.behavior_fault_id, + "cause": robot_state_pb2.BehaviorFault.Cause.Name(fault.cause), + "status": robot_state_pb2.BehaviorFault.Status.Name(fault.status), + } + for fault in remaining_state.behavior_fault_state.faults + ] + return { + "attempted": len(faults), + "cleared": cleared, + "errors": errors, + "remaining": remaining, + } + + def require_connected(self) -> None: + if not self.connected: + raise RuntimeError("Spot is not connected. Call POST /connect first or set startup env vars.") + + def take_lease(self) -> dict[str, Any]: + self.require_connected() + self.shutdown_lease() + self.lease.take() + self.lease_keepalive = LeaseKeepAlive( + self.lease, + must_acquire=False, + return_at_exit=False, + ) + return self.status() + + def acquire_lease(self) -> dict[str, Any]: + self.require_connected() + self.shutdown_lease() + self.lease_keepalive = LeaseKeepAlive( + self.lease, + must_acquire=True, + return_at_exit=False, + ) + return self.status() + + def shutdown_lease(self) -> None: + if self.lease_keepalive is not None: + self.lease_keepalive.shutdown() + self.lease_keepalive = None + + def list_leases(self) -> str: + self.require_connected() + return str(self.lease.list_leases(include_full_lease_info=True)) + + def ensure_lease(self, *, take_if_needed: bool = False) -> None: + self.require_connected() + if self.holding_lease: + try: + lease = self.lease.lease_wallet.get_lease() + self.lease.retain_lease(lease) + return + except (LeaseNotOwnedByWallet, LeaseResponseError): + self.shutdown_lease() + if take_if_needed: + self.take_lease() + else: + self.acquire_lease() + + def sync_waypoints( + self, + *, + overwrite: bool = False, + prune_stale: bool = False, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + graph = self.graph_nav.download_graph() + named_waypoints = { + waypoint.annotations.name.strip(): waypoint.id + for waypoint in graph.waypoints + if waypoint.annotations.name.strip() + } + synced = self.registry.sync_from_waypoints( + named_waypoints, overwrite=overwrite + ) + removed = ( + self.registry.prune_stale_waypoint_ids( + {waypoint.id for waypoint in graph.waypoints} + ) + if prune_stale + else [] + ) + return { + "count": len(synced), + "removed_count": len(removed), + "synced": [asdict(keypoint) for keypoint in synced], + "removed": [asdict(keypoint) for keypoint in removed], + } + + def list_waypoints(self) -> list[dict[str, Any]]: + return [asdict(keypoint) for keypoint in self.registry.all()] + + def named_waypoints(self) -> dict[str, str]: + graph = self.graph_nav.download_graph() + waypoints: dict[str, str] = {} + for waypoint in graph.waypoints: + name = waypoint.annotations.name.strip() + if name: + waypoints[name] = waypoint.id + return waypoints + + def load_map( + self, + path: str | Path, + *, + replace_graph: bool = True, + generate_new_anchoring: bool = False, + take_lease: bool = False, + ) -> dict[str, Any]: + self.require_connected() + map_dir = Path(path).expanduser().resolve() + graph_path = map_dir / "graph" + waypoint_dir = map_dir / "waypoint_snapshots" + edge_dir = map_dir / "edge_snapshots" + if not graph_path.is_file(): + raise ValueError(f"Map directory must contain a graph file: {graph_path}") + + graph = map_pb2.Graph() + graph.ParseFromString(graph_path.read_bytes()) + + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + response = self.graph_nav.upload_graph( + graph=graph, + replace_graph=replace_graph, + generate_new_anchoring=generate_new_anchoring, + ) + + uploaded_waypoint_snapshots: list[str] = [] + missing_waypoint_snapshots: list[str] = [] + for snapshot_id in response.unknown_waypoint_snapshot_ids: + snapshot_path = waypoint_dir / snapshot_id + if not snapshot_path.is_file(): + missing_waypoint_snapshots.append(snapshot_id) + continue + snapshot = map_pb2.WaypointSnapshot() + snapshot.ParseFromString(snapshot_path.read_bytes()) + self.graph_nav.upload_waypoint_snapshot(snapshot) + uploaded_waypoint_snapshots.append(snapshot_id) + + uploaded_edge_snapshots: list[str] = [] + missing_edge_snapshots: list[str] = [] + for snapshot_id in response.unknown_edge_snapshot_ids: + snapshot_path = edge_dir / snapshot_id + if not snapshot_path.is_file(): + missing_edge_snapshots.append(snapshot_id) + continue + snapshot = map_pb2.EdgeSnapshot() + snapshot.ParseFromString(snapshot_path.read_bytes()) + self.graph_nav.upload_edge_snapshot(snapshot) + uploaded_edge_snapshots.append(snapshot_id) + + if missing_waypoint_snapshots or missing_edge_snapshots: + raise ValueError( + "Map graph loaded but some required snapshots were missing: " + f"waypoint={missing_waypoint_snapshots}, edge={missing_edge_snapshots}" + ) + + synced = self.registry.sync_from_waypoints(self.named_waypoints(), overwrite=False) + return { + "path": str(map_dir), + "status": graph_nav_pb2.UploadGraphResponse.Status.Name(response.status), + "replaced_graph": response.replaced_graph, + "waypoints": len(graph.waypoints), + "edges": len(graph.edges), + "uploaded_waypoint_snapshots": uploaded_waypoint_snapshots, + "uploaded_edge_snapshots": uploaded_edge_snapshots, + "loaded_waypoint_snapshot_ids": list(response.loaded_waypoint_snapshot_ids), + "loaded_edge_snapshot_ids": list(response.loaded_edge_snapshot_ids), + "synced_keypoints": [asdict(item) for item in synced], + } + + def localization_status(self) -> dict[str, Any]: + self.require_connected() + response = self.graph_nav.get_localization_state() + return { + "localization": _localization_to_dict(response.localization), + "localized": bool(response.localization.waypoint_id), + } + + def localize( + self, + *, + waypoint_id: str | None = None, + waypoint_name: str | None = None, + fiducial_init: str = "nearest", + use_fiducial_id: int | None = None, + refine_fiducial_result_with_icp: bool = True, + do_ambiguity_check: bool = False, + refine_with_visual_features: bool = False, + verify_visual_features_quality: bool = False, + max_distance: float | None = None, + max_yaw: float | None = None, + ) -> dict[str, Any]: + self.require_connected() + self._require_graph_loaded() + if waypoint_name: + waypoint_id = self.registry.get(waypoint_name).waypoint_id + + initial_guess = nav_pb2.Localization() + if waypoint_id: + initial_guess.waypoint_id = waypoint_id + initial_guess.waypoint_tform_body.rotation.w = 1.0 + + fiducial_init_value = _fiducial_init_value(fiducial_init) + response = self.graph_nav.set_localization_full_response( + initial_guess, + max_distance=max_distance, + max_yaw=max_yaw, + fiducial_init=fiducial_init_value, + use_fiducial_id=use_fiducial_id, + refine_fiducial_result_with_icp=refine_fiducial_result_with_icp, + do_ambiguity_check=do_ambiguity_check, + refine_with_visual_features=refine_with_visual_features, + verify_visual_features_quality=verify_visual_features_quality, + ) + return { + "status": graph_nav_pb2.SetLocalizationResponse.Status.Name(response.status), + "localized": response.status == graph_nav_pb2.SetLocalizationResponse.STATUS_OK, + "localization": _localization_to_dict(response.localization), + "waypoint_id": response.localization.waypoint_id, + "error_report": response.error_report, + "quality_check_result": graph_nav_pb2.SetLocalizationResponse.QualityCheckResult.Name( + response.quality_check_result + ), + } + + def navigate( + self, + name: str, + *, + command_duration: float, + timeout: float, + feedback_interval: float, + power_on: bool, + stand: bool, + take_lease: bool, + ) -> dict[str, Any]: + self.require_connected() + self.cancel_actions_event.clear() + keypoint = self.registry.get(name) + with self.lock: + self._require_navigation_ready() + self.ensure_lease(take_if_needed=take_lease) + if power_on: + power_on_motors(self.power) + if stand: + blocking_stand(self.command, timeout_sec=10) + + command_id = self.graph_nav.navigate_to(keypoint.waypoint_id, command_duration) + deadline = time.monotonic() + timeout + next_refresh = time.monotonic() + max(command_duration / 2, feedback_interval) + last_status = graph_nav_pb2.NavigationFeedbackResponse.STATUS_UNKNOWN + + while time.monotonic() < deadline: + if self.cancel_actions_event.is_set(): + command_id = self.command.robot_command(RobotCommandBuilder.stop_command()) + return { + "name": name, + "waypoint_id": keypoint.waypoint_id, + "command_id": command_id, + "status": "CANCELLED_BY_STOP_ENDPOINT", + "reached_goal": False, + } + feedback = self.graph_nav.navigation_feedback(command_id) + last_status = feedback.status + if last_status == SUCCESS_STATUS or last_status not in ACTIVE_STATUSES: + return { + "name": name, + "waypoint_id": keypoint.waypoint_id, + "command_id": command_id, + "status": graph_nav_pb2.NavigationFeedbackResponse.Status.Name(last_status), + "reached_goal": last_status == SUCCESS_STATUS, + } + if time.monotonic() >= next_refresh: + command_id = self.graph_nav.navigate_to( + keypoint.waypoint_id, + command_duration, + command_id=command_id, + ) + next_refresh = time.monotonic() + max(command_duration / 2, feedback_interval) + time.sleep(feedback_interval) + + return { + "name": name, + "waypoint_id": keypoint.waypoint_id, + "command_id": command_id, + "status": f"TIMED_OUT_WAITING_FOR_{graph_nav_pb2.NavigationFeedbackResponse.Status.Name(last_status)}", + "reached_goal": False, + } + + def _require_graph_loaded(self) -> None: + graph = self.graph_nav.download_graph() + if not graph.waypoints: + raise RuntimeError( + "No GraphNav map is loaded. Load a map with POST /map/load or" + " select and load the map from the Spot tablet, then localize." + ) + + def _require_navigation_ready(self) -> None: + self._require_graph_loaded() + localization = self.graph_nav.get_localization_state().localization + if not localization.waypoint_id: + raise RuntimeError( + "GraphNav is not localized. Call POST /localize near a visible" + " fiducial or provide the robot's current waypoint before" + " navigating." + ) + + def stand(self, *, power_on: bool = True, take_lease: bool = False, timeout: float = 10.0) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + if power_on: + power_on_motors(self.power) + blocking_stand(self.command, timeout_sec=timeout) + return {"standing": True} + + def sit(self, *, take_lease: bool = False, timeout: float = 10.0) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + blocking_sit(self.command, timeout_sec=timeout) + return {"sitting": True} + + def velocity( + self, + *, + v_x: float, + v_y: float, + v_rot: float, + duration: float = 0.6, + take_lease: bool = False, + power_on: bool = False, + stand: bool = False, + body_follow_arm: bool = False, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + if power_on: + power_on_motors(self.power) + if stand: + blocking_stand(self.command, timeout_sec=10) + + safe_duration = max(0.1, min(float(duration), 2.0)) + safe_v_x = _clamp(float(v_x), -MAX_LINEAR_SPEED_MPS, MAX_LINEAR_SPEED_MPS) + safe_v_y = _clamp(float(v_y), -MAX_SIDEWAYS_SPEED_MPS, MAX_SIDEWAYS_SPEED_MPS) + safe_v_rot = _clamp(float(v_rot), -MAX_ANGULAR_SPEED_RADPS, MAX_ANGULAR_SPEED_RADPS) + mobility_command = RobotCommandBuilder.synchro_velocity_command( + safe_v_x, safe_v_y, safe_v_rot + ) + command = ( + RobotCommandBuilder.build_synchro_command( + mobility_command, + RobotCommandBuilder.arm_joint_freeze_command(), + ) + if body_follow_arm + else mobility_command + ) + command_id = self.command.robot_command( + command, + end_time_secs=time.time() + safe_duration, + timesync_endpoint=self.robot.time_sync.endpoint, + ) + return { + "command_id": command_id, + "v_x": safe_v_x, + "v_y": safe_v_y, + "v_rot": safe_v_rot, + "duration": safe_duration, + "body_follow_arm": bool(body_follow_arm), + } + + def stop(self, *, take_lease: bool = False) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + command_id = self.command.robot_command(RobotCommandBuilder.stop_command()) + return {"command_id": command_id, "stopped": True} + + def stop_all_actions(self, *, take_lease: bool = False, freeze_arm: bool = True) -> dict[str, Any]: + self.require_connected() + self.cancel_actions_event.set() + command_results: list[dict[str, Any]] = [] + errors: list[str] = [] + + if take_lease and not self.holding_lease: + try: + self.take_lease() + except Exception as exc: + errors.append(f"take_lease: {type(exc).__name__}: {exc}") + + try: + command_id = self.command.robot_command(RobotCommandBuilder.stop_command()) + command_results.append({"command": "body_stop", "command_id": command_id}) + except Exception as exc: + errors.append(f"body_stop: {type(exc).__name__}: {exc}") + + if freeze_arm: + try: + command_id = self.command.robot_command(RobotCommandBuilder.arm_joint_freeze_command()) + command_results.append({"command": "arm_freeze", "command_id": command_id}) + except Exception as exc: + errors.append(f"arm_freeze: {type(exc).__name__}: {exc}") + + return { + "cancel_requested": True, + "commands": command_results, + "errors": errors, + "holding_lease": self.holding_lease, + } + + def visualize(self, *, output: Path = DEFAULT_MAP_PATH, include_point_clouds: bool = True) -> dict[str, Any]: + self.require_connected() + with self.lock: + graph = self.graph_nav.download_graph() + snapshots = {} + if include_point_clouds: + for snapshot_id in sorted({waypoint.snapshot_id for waypoint in graph.waypoints if waypoint.snapshot_id}): + snapshots[snapshot_id] = self.graph_nav.download_waypoint_snapshot( + snapshot_id, + download_images=False, + do_not_download_point_cloud=False, + ) + path = write_graph_html(graph, self.registry, output, snapshots=snapshots) + return {"path": str(path), "waypoints": len(graph.waypoints), "edges": len(graph.edges)} + + def deploy_arm(self, *, power_on: bool = True, take_lease: bool = False, timeout: float = 10.0) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + if power_on: + power_on_motors(self.power) + command_id = self.command.robot_command(RobotCommandBuilder.arm_ready_command()) + arrived = block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return {"command_id": command_id, "arrived": arrived} + + def carry_arm(self, *, take_lease: bool = False, timeout: float = 10.0) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + command_id = self.command.robot_command(RobotCommandBuilder.arm_carry_command()) + arrived = block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return {"command_id": command_id, "arrived": arrived} + + def freeze_arm(self, *, take_lease: bool = False) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + command_id = self.command.robot_command(RobotCommandBuilder.arm_joint_freeze_command()) + return {"command_id": command_id, "frozen": True} + + def reach_distance_to_pose(self, pose: Pose3D) -> dict[str, Any]: + self.require_connected() + if pose.frame_name != frame_helpers.VISION_FRAME_NAME: + raise ValueError(f"Pose must be in {frame_helpers.VISION_FRAME_NAME!r}; got {pose.frame_name!r}.") + + with self.lock: + robot_state = self.state.get_robot_state() + transforms_snapshot = robot_state.kinematic_state.transforms_snapshot + vision_tform_hand = frame_helpers.get_a_tform_b( + transforms_snapshot, + frame_helpers.VISION_FRAME_NAME, + frame_helpers.HAND_FRAME_NAME, + validate=False, + ) + vision_tform_body = frame_helpers.get_a_tform_b( + transforms_snapshot, + frame_helpers.VISION_FRAME_NAME, + frame_helpers.BODY_FRAME_NAME, + validate=False, + ) + if vision_tform_hand is None: + raise ValueError("Could not get current hand pose in vision frame.") + if vision_tform_body is None: + raise ValueError("Could not get current body pose in vision frame.") + + hand = (vision_tform_hand.x, vision_tform_hand.y, vision_tform_hand.z) + vector = np.array([ + float(pose.x) - hand[0], + float(pose.y) - hand[1], + float(pose.z) - hand[2], + ], dtype=np.float64) + distance = float(np.linalg.norm(vector)) + body_tform_vision = vision_tform_body.inverse() + object_body = body_tform_vision.transform_point(float(pose.x), float(pose.y), float(pose.z)) + hand_body = body_tform_vision.transform_point(hand[0], hand[1], hand[2]) + body_vector = np.array([ + float(object_body[0] - hand_body[0]), + float(object_body[1] - hand_body[1]), + float(object_body[2] - hand_body[2]), + ], dtype=np.float64) + planar_body_distance = float(np.linalg.norm(body_vector[:2])) + return { + "frame_name": frame_helpers.VISION_FRAME_NAME, + "distance_m": distance, + "hand_position": _xyz_dict(hand), + "object_position": {"x": pose.x, "y": pose.y, "z": pose.z}, + "delta": { + "x": float(vector[0]), + "y": float(vector[1]), + "z": float(vector[2]), + }, + "body_frame_name": frame_helpers.BODY_FRAME_NAME, + "delta_body": { + "x": float(body_vector[0]), + "y": float(body_vector[1]), + "z": float(body_vector[2]), + }, + "planar_distance_body_m": planar_body_distance, + } + + def approach_pose( + self, + pose: Pose3D, + *, + standoff_m: float = 0.0, + take_lease: bool = False, + seconds: float = 1.2, + timeout: float = 5.0, + max_step_m: float = 0.45, + ) -> dict[str, Any]: + self.require_connected() + if pose.frame_name != frame_helpers.VISION_FRAME_NAME: + raise ValueError(f"Approach pose must be in {frame_helpers.VISION_FRAME_NAME!r}; got {pose.frame_name!r}.") + + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + robot_state = self.state.get_robot_state() + vision_tform_hand = frame_helpers.get_a_tform_b( + robot_state.kinematic_state.transforms_snapshot, + frame_helpers.VISION_FRAME_NAME, + frame_helpers.HAND_FRAME_NAME, + validate=False, + ) + if vision_tform_hand is None: + raise ValueError("Could not get current hand pose in vision frame.") + + vector = np.array([ + float(pose.x) - vision_tform_hand.x, + float(pose.y) - vision_tform_hand.y, + float(pose.z) - vision_tform_hand.z, + ], dtype=np.float64) + distance = float(np.linalg.norm(vector)) + if distance <= 1e-6: + raise ValueError("Detected object pose is too close to current hand pose to compute an approach vector.") + + safe_standoff = _clamp(float(standoff_m), -0.10, 0.10) + safe_seconds = _clamp(float(seconds), 0.25, 3.0) + safe_max_step = _clamp(float(max_step_m), 0.05, 0.8) + direction = vector / distance + target_distance = max(0.0, distance - safe_standoff) + step_distance = min(target_distance, safe_max_step) + target = np.array([vision_tform_hand.x, vision_tform_hand.y, vision_tform_hand.z]) + direction * step_distance + + command = RobotCommandBuilder.arm_pose_command( + float(target[0]), + float(target[1]), + float(target[2]), + vision_tform_hand.rot.w, + vision_tform_hand.rot.x, + vision_tform_hand.rot.y, + vision_tform_hand.rot.z, + frame_helpers.VISION_FRAME_NAME, + seconds=safe_seconds, + ) + command_id = self.command.robot_command(command) + arrived = block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return { + "command_id": command_id, + "arrived": arrived, + "target": { + "frame_name": frame_helpers.VISION_FRAME_NAME, + "x": float(target[0]), + "y": float(target[1]), + "z": float(target[2]), + "qw": vision_tform_hand.rot.w, + "qx": vision_tform_hand.rot.x, + "qy": vision_tform_hand.rot.y, + "qz": vision_tform_hand.rot.z, + }, + "object_pose": asdict(pose), + "distance_to_object_m": distance, + "standoff_m": safe_standoff, + "step_distance_m": step_distance, + "seconds": safe_seconds, + } + + def approach_pose_whole_body( + self, + pose: Pose3D, + *, + standoff_m: float = 0.0, + take_lease: bool = False, + seconds: float = 2.0, + timeout: float = 8.0, + max_step_m: float = 0.8, + ) -> dict[str, Any]: + self.require_connected() + if pose.frame_name != frame_helpers.VISION_FRAME_NAME: + raise ValueError(f"Approach pose must be in {frame_helpers.VISION_FRAME_NAME!r}; got {pose.frame_name!r}.") + + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + robot_state = self.state.get_robot_state() + vision_tform_hand = frame_helpers.get_a_tform_b( + robot_state.kinematic_state.transforms_snapshot, + frame_helpers.VISION_FRAME_NAME, + frame_helpers.HAND_FRAME_NAME, + validate=False, + ) + if vision_tform_hand is None: + raise ValueError("Could not get current hand pose in vision frame.") + + vector = np.array([ + float(pose.x) - vision_tform_hand.x, + float(pose.y) - vision_tform_hand.y, + float(pose.z) - vision_tform_hand.z, + ], dtype=np.float64) + distance = float(np.linalg.norm(vector)) + if distance <= 1e-6: + raise ValueError("Detected object pose is too close to current hand pose to compute an approach vector.") + + safe_standoff = _clamp(float(standoff_m), -0.10, 0.10) + safe_seconds = _clamp(float(seconds), 0.5, 5.0) + safe_max_step = _clamp(float(max_step_m), 0.05, 1.2) + direction = vector / distance + target_distance = max(0.0, distance - safe_standoff) + step_distance = min(target_distance, safe_max_step) + target = np.array([vision_tform_hand.x, vision_tform_hand.y, vision_tform_hand.z]) + direction * step_distance + + arm_command = RobotCommandBuilder.arm_pose_command( + float(target[0]), + float(target[1]), + float(target[2]), + vision_tform_hand.rot.w, + vision_tform_hand.rot.x, + vision_tform_hand.rot.y, + vision_tform_hand.rot.z, + frame_helpers.VISION_FRAME_NAME, + seconds=safe_seconds, + ) + command = RobotCommandBuilder.build_synchro_command( + RobotCommandBuilder.follow_arm_command(), + arm_command, + ) + command_id = self.command.robot_command(command) + arrived = block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return { + "command_id": command_id, + "arrived": arrived, + "whole_body": True, + "target": { + "frame_name": frame_helpers.VISION_FRAME_NAME, + "x": float(target[0]), + "y": float(target[1]), + "z": float(target[2]), + "qw": vision_tform_hand.rot.w, + "qx": vision_tform_hand.rot.x, + "qy": vision_tform_hand.rot.y, + "qz": vision_tform_hand.rot.z, + }, + "object_pose": asdict(pose), + "distance_to_object_m": distance, + "standoff_m": safe_standoff, + "step_distance_m": step_distance, + "seconds": safe_seconds, + } + + def arm_oscillation_monitor_status(self) -> dict[str, Any]: + thread = self.arm_oscillation_monitor_thread + status = dict(self.arm_oscillation_monitor_state) + status["enabled"] = bool(thread and thread.is_alive() and not self.arm_oscillation_monitor_stop.is_set()) + status["config"] = dict(self.arm_oscillation_monitor_config) + return status + + def configure_arm_oscillation_monitor( + self, + *, + enabled: bool, + take_lease: bool = True, + sample_interval: float = 0.1, + window_sec: float = 1.2, + min_peak_to_peak_m: float = 0.012, + min_direction_changes: int = 4, + min_speed_mps: float = 0.025, + freeze_cooldown_sec: float = 2.0, + ) -> dict[str, Any]: + self.require_connected() + if not enabled: + self.stop_arm_oscillation_monitor() + return self.arm_oscillation_monitor_status() + + self.arm_oscillation_monitor_config = { + "take_lease": bool(take_lease), + "sample_interval": _clamp(float(sample_interval), 0.05, 0.5), + "window_sec": _clamp(float(window_sec), 0.5, 3.0), + "min_peak_to_peak_m": _clamp(float(min_peak_to_peak_m), 0.003, 0.08), + "min_direction_changes": max(2, min(12, int(min_direction_changes))), + "min_speed_mps": _clamp(float(min_speed_mps), 0.005, 0.25), + "freeze_cooldown_sec": _clamp(float(freeze_cooldown_sec), 0.5, 10.0), + } + thread = self.arm_oscillation_monitor_thread + if thread and thread.is_alive(): + self.arm_oscillation_monitor_state["enabled"] = True + return self.arm_oscillation_monitor_status() + + self.arm_oscillation_monitor_stop.clear() + self.arm_oscillation_monitor_state.update({ + "enabled": True, + "samples": 0, + "last_detection": None, + "last_freeze": None, + "last_error": None, + }) + self.arm_oscillation_monitor_thread = threading.Thread( + target=self._arm_oscillation_monitor_loop, + name="arm-oscillation-monitor", + daemon=True, + ) + self.arm_oscillation_monitor_thread.start() + return self.arm_oscillation_monitor_status() + + def stop_arm_oscillation_monitor(self) -> dict[str, Any]: + self.arm_oscillation_monitor_stop.set() + thread = self.arm_oscillation_monitor_thread + if thread and thread.is_alive(): + thread.join(timeout=1.0) + self.arm_oscillation_monitor_thread = None + self.arm_oscillation_monitor_state["enabled"] = False + return self.arm_oscillation_monitor_status() + + def _arm_oscillation_monitor_loop(self) -> None: + samples: deque[tuple[float, tuple[float, float, float]]] = deque() + last_freeze_time = 0.0 + while not self.arm_oscillation_monitor_stop.is_set(): + config = dict(self.arm_oscillation_monitor_config) + interval = config.get("sample_interval", 0.1) + try: + pose = self._current_hand_position() + now = time.monotonic() + samples.append((now, pose)) + while samples and now - samples[0][0] > config.get("window_sec", 1.2): + samples.popleft() + self.arm_oscillation_monitor_state["samples"] = len(samples) + detection = self._detect_arm_oscillation(samples, config) + if detection and now - last_freeze_time >= config.get("freeze_cooldown_sec", 2.0): + with self.lock: + self.ensure_lease(take_if_needed=config.get("take_lease", True)) + command_id = self.command.robot_command(RobotCommandBuilder.arm_joint_freeze_command()) + last_freeze_time = now + detection["command_id"] = command_id + detection["monotonic_time"] = now + self.arm_oscillation_monitor_state["last_detection"] = detection + self.arm_oscillation_monitor_state["last_freeze"] = { + "command_id": command_id, + "monotonic_time": now, + } + samples.clear() + except Exception as exc: + self.arm_oscillation_monitor_state["last_error"] = f"{type(exc).__name__}: {exc}" + self.arm_oscillation_monitor_stop.wait(interval) + + def _current_hand_position(self) -> tuple[float, float, float]: + with self.lock: + robot_state = self.state.get_robot_state() + vision_tform_hand = frame_helpers.get_a_tform_b( + robot_state.kinematic_state.transforms_snapshot, + frame_helpers.VISION_FRAME_NAME, + frame_helpers.HAND_FRAME_NAME, + validate=False, + ) + if vision_tform_hand is None: + raise ValueError("Could not get current hand pose in vision frame.") + return (vision_tform_hand.x, vision_tform_hand.y, vision_tform_hand.z) + + def _detect_arm_oscillation( + self, + samples: deque[tuple[float, tuple[float, float, float]]], + config: dict[str, Any], + ) -> dict[str, Any] | None: + if len(samples) < 6: + return None + times = [item[0] for item in samples] + positions = [item[1] for item in samples] + duration = times[-1] - times[0] + if duration <= 0: + return None + + axis_names = ("x", "y", "z") + for axis_index, axis_name in enumerate(axis_names): + values = [position[axis_index] for position in positions] + peak_to_peak = max(values) - min(values) + if peak_to_peak < config.get("min_peak_to_peak_m", 0.012): + continue + + signs: list[int] = [] + for index in range(1, len(values)): + dt = times[index] - times[index - 1] + if dt <= 0: + continue + speed = (values[index] - values[index - 1]) / dt + if abs(speed) < config.get("min_speed_mps", 0.025): + continue + sign = 1 if speed > 0 else -1 + if not signs or signs[-1] != sign: + signs.append(sign) + + direction_changes = max(0, len(signs) - 1) + if direction_changes >= config.get("min_direction_changes", 4): + return { + "axis": axis_name, + "duration_sec": duration, + "peak_to_peak_m": peak_to_peak, + "direction_changes": direction_changes, + } + return None + + def jog_arm( + self, + *, + dx: float = 0.0, + dy: float = 0.0, + dz: float = 0.0, + droll: float = 0.0, + dpitch: float = 0.0, + dyaw: float = 0.0, + seconds: float = 0.8, + take_lease: bool = False, + timeout: float = 3.0, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + robot_state = self.state.get_robot_state() + vision_tform_hand = frame_helpers.get_a_tform_b( + robot_state.kinematic_state.transforms_snapshot, + frame_helpers.VISION_FRAME_NAME, + frame_helpers.HAND_FRAME_NAME, + validate=False, + ) + if vision_tform_hand is None: + raise ValueError("Could not get current hand pose in vision frame.") + + safe_dx = _clamp(float(dx), -0.08, 0.08) + safe_dy = _clamp(float(dy), -0.08, 0.08) + safe_dz = _clamp(float(dz), -0.08, 0.08) + requested_droll = float(droll) + requested_dpitch = float(dpitch) + requested_dyaw = float(dyaw) + safe_seconds = _clamp(float(seconds), 0.25, 2.0) + + delta_x, delta_y, delta_z = vision_tform_hand.rot.transform_point(safe_dx, safe_dy, safe_dz) + delta_rot = ( + Quat.from_roll(requested_droll) + .mult(Quat.from_pitch(requested_dpitch)) + .mult(Quat.from_yaw(requested_dyaw)) + ) + new_rot = vision_tform_hand.rot.mult(delta_rot) + + command = RobotCommandBuilder.arm_pose_command( + vision_tform_hand.x + delta_x, + vision_tform_hand.y + delta_y, + vision_tform_hand.z + delta_z, + new_rot.w, + new_rot.x, + new_rot.y, + new_rot.z, + frame_helpers.VISION_FRAME_NAME, + seconds=safe_seconds, + ) + command_id = self.command.robot_command(command) + arrived = block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return { + "command_id": command_id, + "arrived": arrived, + "dx": safe_dx, + "dy": safe_dy, + "dz": safe_dz, + "droll": requested_droll, + "dpitch": requested_dpitch, + "dyaw": requested_dyaw, + "seconds": safe_seconds, + } + + def roll_camera_view( + self, + *, + direction: str, + angle_rad: float = 0.105, + seconds: float = 0.7, + take_lease: bool = True, + timeout: float = 3.0, + ) -> dict[str, Any]: + normalized = direction.strip().lower().replace("_", "-") + safe_angle = _clamp(abs(float(angle_rad)), 0.01, math.pi) + if normalized in {"cw", "clockwise", "right"}: + sign = -1.0 + direction_name = "clockwise" + elif normalized in {"ccw", "counterclockwise", "counter-clockwise", "left"}: + sign = 1.0 + direction_name = "counterclockwise" + else: + raise ValueError("direction must be one of: clockwise, counterclockwise") + + remaining = safe_angle + step_limit = 0.25 + commands: list[dict[str, Any]] = [] + while remaining > 1e-6: + step = min(step_limit, remaining) + commands.append(self.jog_arm( + droll=sign * step, + seconds=seconds, + take_lease=take_lease, + timeout=timeout, + )) + remaining -= step + + return { + "direction": direction_name, + "angle_rad": safe_angle, + "steps": len(commands), + "commands": commands, + } + + def stow_arm(self, *, take_lease: bool = False, timeout: float = 10.0) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + command_id = self.command.robot_command(RobotCommandBuilder.arm_stow_command()) + arrived = block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return {"command_id": command_id, "arrived": arrived} + + def open_gripper( + self, + *, + take_lease: bool = False, + timeout: float = 5.0, + open_fraction: float = 1.0, + max_vel: float | None = None, + max_acc: float | None = None, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + safe_fraction = _clamp(float(open_fraction), 0.0, 1.0) + command_id = self.command.robot_command(RobotCommandBuilder.claw_gripper_open_fraction_command( + safe_fraction, + max_vel=max_vel, + max_acc=max_acc, + )) + at_goal = self._block_until_gripper_at_goal(command_id, timeout_sec=timeout) + return {"command_id": command_id, "at_goal": at_goal, "open_fraction": safe_fraction} + + def close_gripper( + self, + *, + take_lease: bool = False, + timeout: float = 5.0, + max_vel: float | None = None, + max_acc: float | None = None, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + command_id = self.command.robot_command(RobotCommandBuilder.claw_gripper_close_command( + max_vel=max_vel, + max_acc=max_acc, + )) + at_goal = self._block_until_gripper_at_goal(command_id, timeout_sec=timeout) + return {"command_id": command_id, "at_goal": at_goal, "max_vel": max_vel, "max_acc": max_acc} + + def wait_for_pick_up( + self, + *, + monitor_sec: float = 30.0, + upward_threshold_m: float = 0.02, + sample_interval: float = 0.1, + open_duration_sec: float = 3.0, + take_lease: bool = True, + gripper_timeout: float = 5.0, + stow_timeout: float = 10.0, + ) -> dict[str, Any]: + self.require_connected() + self.cancel_actions_event.clear() + self.ensure_lease(take_if_needed=take_lease) + + safe_monitor = _clamp(float(monitor_sec), 1.0, 120.0) + safe_threshold = _clamp(float(upward_threshold_m), 0.005, 0.20) + safe_interval = _clamp(float(sample_interval), 0.05, 1.0) + safe_open_duration = _clamp(float(open_duration_sec), 0.1, 10.0) + safe_gripper_timeout = _clamp(float(gripper_timeout), 1.0, 15.0) + safe_stow_timeout = _clamp(float(stow_timeout), 1.0, 30.0) + + start_time = time.monotonic() + initial_position = self._current_hand_position() + max_position = initial_position + max_upward_delta = 0.0 + samples = 1 + + while time.monotonic() - start_time < safe_monitor: + if self.cancel_actions_event.is_set(): + return { + "triggered": False, + "reason": "cancelled", + "samples": samples, + "elapsed_sec": time.monotonic() - start_time, + "max_upward_delta_m": max_upward_delta, + "initial_position": _xyz_dict(initial_position), + "max_position": _xyz_dict(max_position), + "monitor_sec": safe_monitor, + "upward_threshold_m": safe_threshold, + } + time.sleep(safe_interval) + position = self._current_hand_position() + samples += 1 + upward_delta = position[2] - initial_position[2] + if upward_delta > max_upward_delta: + max_upward_delta = upward_delta + max_position = position + if upward_delta >= safe_threshold: + open_result = self.open_gripper(take_lease=take_lease, timeout=safe_gripper_timeout) + time.sleep(safe_open_duration) + close_result = self.close_gripper(take_lease=take_lease, timeout=safe_gripper_timeout) + stow_result = self.stow_arm(take_lease=take_lease, timeout=safe_stow_timeout) + return { + "triggered": True, + "reason": "upward_motion", + "samples": samples, + "elapsed_sec": time.monotonic() - start_time, + "upward_delta_m": upward_delta, + "max_upward_delta_m": max_upward_delta, + "initial_position": _xyz_dict(initial_position), + "trigger_position": _xyz_dict(position), + "max_position": _xyz_dict(max_position), + "monitor_sec": safe_monitor, + "upward_threshold_m": safe_threshold, + "open_duration_sec": safe_open_duration, + "open": open_result, + "close": close_result, + "stow": stow_result, + } + + return { + "triggered": False, + "reason": "timeout", + "samples": samples, + "elapsed_sec": time.monotonic() - start_time, + "max_upward_delta_m": max_upward_delta, + "initial_position": _xyz_dict(initial_position), + "max_position": _xyz_dict(max_position), + "monitor_sec": safe_monitor, + "upward_threshold_m": safe_threshold, + } + + def capture_rgbd( + self, + *, + color_source: str = DEFAULT_COLOR_SOURCE, + depth_source: str = DEFAULT_DEPTH_SOURCE, + quality_percent: int = 85, + ) -> ImageObservation: + self.require_connected() + requests = [ + build_image_request(color_source, quality_percent=quality_percent), + build_image_request( + depth_source, + image_format=image_pb2.Image.FORMAT_RAW, + pixel_format=image_pb2.Image.PIXEL_FORMAT_DEPTH_U16, + ), + ] + color_response, depth_response = self.image.get_image(requests) + return ImageObservation(color_response=color_response, depth_response=depth_response) + + def image_sources(self) -> list[dict[str, Any]]: + self.require_connected() + sources = self.image.list_image_sources() + return [ + { + "name": source.name, + "cols": source.cols, + "rows": source.rows, + "depth_scale": source.depth_scale, + "image_type": source.image_type, + } + for source in sources + ] + + def capture_image(self, source: str, *, quality_percent: int = 85): + self.require_connected() + request = build_image_request(source, quality_percent=quality_percent) + return self.image.get_image([request])[0] + + def detect_object_2d( + self, + instruction: str, + *, + model: str, + api_key: str | None, + color_source: str = DEFAULT_COLOR_SOURCE, + depth_source: str = DEFAULT_DEPTH_SOURCE, + ) -> tuple[Detection2D, Pose3D, ImageObservation]: + self.require_connected() + with self.lock: + observation = self.capture_rgbd(color_source=color_source, depth_source=depth_source) + detector = GeminiObjectDetector(model=model, api_key=api_key) + detection = detector.detect( + instruction=instruction, + image_bytes=observation.color_bytes, + mime_type=observation.color_mime_type, + width=observation.width, + height=observation.height, + ) + pose = self.detection_to_3d_pose(detection, observation) + return detection, pose, observation + + def detect_pick_target( + self, + instruction: str, + *, + model: str, + api_key: str | None, + ) -> dict[str, Any]: + """Return only the language-conditioned 2D target needed by pick.""" + self.require_connected() + with self.lock: + color_response = self.capture_image(DEFAULT_COLOR_SOURCE) + observation = ImageObservation(color_response=color_response) + detector = GeminiObjectDetector(model=model, api_key=api_key) + detection = detector.detect( + instruction=instruction, + image_bytes=observation.color_bytes, + mime_type=observation.color_mime_type, + width=observation.width, + height=observation.height, + ) + if observation.width <= 1 or observation.height <= 1: + raise ValueError("Detection image dimensions must be greater than one.") + pixel_x, pixel_y = detection.grasp_px + normalized_x = round(1000.0 * pixel_x / (observation.width - 1)) + normalized_y = round(1000.0 * pixel_y / (observation.height - 1)) + return { + "detected": True, + "instruction": instruction, + "label": detection.label, + "confidence": detection.confidence, + "target": { + "normalized_x": max(0, min(1000, normalized_x)), + "normalized_y": max(0, min(1000, normalized_y)), + "pixel_x": pixel_x, + "pixel_y": pixel_y, + "image_width": observation.width, + "image_height": observation.height, + }, + } + + def detect_scene( + self, + instruction: str, + *, + model: str, + api_key: str | None, + color_source: str = DEFAULT_COLOR_SOURCE, + depth_source: str = DEFAULT_DEPTH_SOURCE, + point_cloud_stride: int = 4, + max_point_cloud_points: int = 6000, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + observation = self.capture_rgbd(color_source=color_source, depth_source=depth_source) + detector = GeminiObjectDetector(model=model, api_key=api_key) + detection_result = detector.detect_with_result( + instruction=instruction, + image_bytes=observation.color_bytes, + mime_type=observation.color_mime_type, + width=observation.width, + height=observation.height, + ) + detection = detection_result.detection + errors: dict[str, str] = {} + + pose_vision = None + try: + pose_vision = asdict(self.detection_to_3d_pose(detection, observation)) + except ValueError as exc: + errors["pose"] = str(exc) + + grasp_camera = None + try: + grasp_camera = self.detection_to_camera_point(detection, observation) + except ValueError as exc: + errors["grasp_camera"] = str(exc) + + point_cloud = { + "frame_name": observation.depth_response.shot.frame_name_image_sensor, + "stride": max(1, int(point_cloud_stride)), + "fields": ["x", "y", "z", "u", "v"], + "points": [], + } + if max_point_cloud_points: + try: + point_cloud = self.depth_point_cloud( + observation, + stride=point_cloud_stride, + max_points=max_point_cloud_points, + ) + except ValueError as exc: + errors["point_cloud"] = str(exc) + + return { + "detection": asdict(detection), + "pose": pose_vision, + "grasp_camera": grasp_camera, + "model": { + "name": model, + "prompt": detection_result.prompt, + "raw_response": detection_result.raw_response, + "parsed_json": detection_result.parsed_json, + }, + "errors": errors, + "image": { + "source": color_source, + "width": observation.width, + "height": observation.height, + "mime_type": observation.color_mime_type, + "data": observation.color_bytes, + }, + "depth": { + "source": depth_source, + "width": observation.depth_response.shot.image.cols, + "height": observation.depth_response.shot.image.rows, + "frame_name": observation.depth_response.shot.frame_name_image_sensor, + "camera": self.camera_intrinsics(observation.depth_response), + }, + "point_cloud": point_cloud, + } + + def detection_to_3d_pose( + self, + detection: Detection2D, + observation: ImageObservation, + *, + frame_name: str = frame_helpers.VISION_FRAME_NAME, + depth_window_px: int = 5, + ) -> Pose3D: + if observation.depth_response is None: + raise ValueError("A depth image is required to project a 2D detection into 3D.") + + depth_response = observation.depth_response + depth_m = _depth_at_pixel(depth_response, detection.grasp_px, window_px=depth_window_px) + point_sensor = pixel_to_camera_space( + depth_response.source, + int(round(detection.grasp_px[0])), + int(round(detection.grasp_px[1])), + depth=depth_m, + ) + frame_tform_sensor = frame_helpers.get_a_tform_b( + depth_response.shot.transforms_snapshot, + frame_name, + depth_response.shot.frame_name_image_sensor, + validate=False, + ) + if frame_tform_sensor is None: + raise ValueError( + f"Could not transform {depth_response.shot.frame_name_image_sensor} into {frame_name}." + ) + x, y, z = frame_tform_sensor.transform_point(*point_sensor) + return Pose3D(frame_name=frame_name, x=x, y=y, z=z) + + def detection_to_camera_point( + self, + detection: Detection2D, + observation: ImageObservation, + *, + depth_window_px: int = 5, + ) -> dict[str, float | str]: + if observation.depth_response is None: + raise ValueError("A depth image is required to project a 2D detection into 3D.") + + depth_response = observation.depth_response + depth_m = _depth_at_pixel(depth_response, detection.grasp_px, window_px=depth_window_px) + x, y, z = pixel_to_camera_space( + depth_response.source, + int(round(detection.grasp_px[0])), + int(round(detection.grasp_px[1])), + depth=depth_m, + ) + return { + "frame_name": depth_response.shot.frame_name_image_sensor, + "x": x, + "y": y, + "z": z, + } + + def camera_intrinsics(self, image_response) -> dict[str, float]: + source = image_response.source + if not source.HasField("pinhole"): + raise ValueError(f"Image source {source.name!r} does not have pinhole intrinsics.") + intrinsics = source.pinhole.intrinsics + return { + "fx": intrinsics.focal_length.x, + "fy": intrinsics.focal_length.y, + "cx": intrinsics.principal_point.x, + "cy": intrinsics.principal_point.y, + "depth_scale": source.depth_scale or 1000.0, + } + + def depth_point_cloud( + self, + observation: ImageObservation, + *, + stride: int = 4, + max_points: int = 6000, + ) -> dict[str, Any]: + if observation.depth_response is None: + raise ValueError("A depth image is required to create a point cloud.") + depth_response = observation.depth_response + image = depth_response.shot.image + depth = np.frombuffer(image.data, dtype=np.uint16).reshape((image.rows, image.cols)) + camera = self.camera_intrinsics(depth_response) + step = max(1, int(stride)) + ys, xs = np.mgrid[0:image.rows:step, 0:image.cols:step] + sampled_depth = depth[ys, xs].astype(np.float64) + valid = (sampled_depth > 0) & (sampled_depth < np.iinfo(np.uint16).max) + xs = xs[valid].astype(np.float64) + ys = ys[valid].astype(np.float64) + zs = sampled_depth[valid] / camera["depth_scale"] + + if max_points > 0 and zs.size > max_points: + indices = np.linspace(0, zs.size - 1, int(max_points), dtype=np.int64) + xs = xs[indices] + ys = ys[indices] + zs = zs[indices] + + us = xs.astype(np.int64) + vs = ys.astype(np.int64) + points_x = zs * (xs - camera["cx"]) / camera["fx"] + points_y = zs * (ys - camera["cy"]) / camera["fy"] + points = np.column_stack((points_x.round(4), points_y.round(4), zs.round(4), us, vs)) + return { + "frame_name": depth_response.shot.frame_name_image_sensor, + "stride": step, + "fields": ["x", "y", "z", "u", "v"], + "points": points.tolist(), + } + + def pick( + self, + instruction: str, + *, + model: str, + api_key: str | None, + take_lease: bool, + timeout: float, + grip_max_torque_nm: float = 2.0, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + observation = self.capture_rgbd() + detector = GeminiObjectDetector(model=model, api_key=api_key) + detection = detector.detect( + instruction=instruction, + image_bytes=observation.color_bytes, + mime_type=observation.color_mime_type, + width=observation.width, + height=observation.height, + ) + pose = self.detection_to_3d_pose(detection, observation) + pick = manipulation_api_pb2.PickObjectInImage( + pixel_xy=geometry_pb2.Vec2(x=detection.grasp_px[0], y=detection.grasp_px[1]), + transforms_snapshot_for_camera=observation.color_response.shot.transforms_snapshot, + frame_name_image_sensor=observation.color_response.shot.frame_name_image_sensor, + camera_model=observation.color_response.source.pinhole, + ) + constrain_grasp_to_top_down(pick) + request = manipulation_api_pb2.ManipulationApiRequest(pick_object_in_image=pick) + response = self.manipulation.manipulation_api_command(request) + feedback = self.wait_for_manipulation(response.manipulation_cmd_id, timeout=timeout) + light_grip = self._apply_light_grip_after_pick( + feedback.current_state, + max_torque_nm=grip_max_torque_nm, + ) + return { + "command_id": response.manipulation_cmd_id, + "state": manipulation_api_pb2.ManipulationFeedbackState.Name(feedback.current_state), + "detection": asdict(detection), + "pose": asdict(pose), + "light_grip": light_grip, + } + + def grasp_at_pixel( + self, + x_pct: float, + y_pct: float, + *, + take_lease: bool = True, + timeout: float = 120.0, + max_depth_m: float = 2.0, + grip_max_torque_nm: float = 2.0, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + observation = self.capture_rgbd() + + pixel_x = _normalized_coordinate_to_pixel(x_pct, observation.width) + pixel_y = _normalized_coordinate_to_pixel(y_pct, observation.height) + depth_response = observation.depth_response + if depth_response is None: + raise ValueError("Aligned hand depth is required for a safe pick.") + depth_image = depth_response.shot.image + if (depth_image.cols, depth_image.rows) != (observation.width, observation.height): + raise ValueError( + "Pick RGB/depth dimensions are not aligned: " + f"RGB={observation.width}x{observation.height}, " + f"depth={depth_image.cols}x{depth_image.rows}." + ) + depth_m = _depth_at_pixel(depth_response, (pixel_x, pixel_y), window_px=3) + if depth_m < 0.08 or depth_m > max_depth_m: + raise ValueError( + f"Unsafe pick target depth {depth_m:.3f}m at ({pixel_x}, {pixel_y}); " + f"expected 0.08-{max_depth_m:.2f}m." + ) + point_x, point_y, point_z = pixel_to_camera_space( + depth_response.source, + pixel_x, + pixel_y, + depth=depth_m, + ) + + pick = manipulation_api_pb2.PickObjectInImage( + pixel_xy=geometry_pb2.Vec2(x=pixel_x, y=pixel_y), + transforms_snapshot_for_camera=observation.color_response.shot.transforms_snapshot, + frame_name_image_sensor=observation.color_response.shot.frame_name_image_sensor, + camera_model=observation.color_response.source.pinhole, + ) + constrain_grasp_to_top_down(pick) + request = manipulation_api_pb2.ManipulationApiRequest(pick_object_in_image=pick) + response = self.manipulation.manipulation_api_command(request) + feedback = self.wait_for_manipulation(response.manipulation_cmd_id, timeout=timeout) + state_name = manipulation_api_pb2.ManipulationFeedbackState.Name(feedback.current_state) + light_grip = self._apply_light_grip_after_pick( + feedback.current_state, + max_torque_nm=grip_max_torque_nm, + ) + time.sleep(0.25) + manipulator_state = self.state.get_robot_state().manipulator_state + holding_item = bool(manipulator_state.is_gripper_holding_item) + return { + "command_id": response.manipulation_cmd_id, + "state": state_name, + "success": ( + feedback.current_state == manipulation_api_pb2.MANIP_STATE_GRASP_SUCCEEDED + and holding_item + ), + "holding_item": holding_item, + "gripper_open_percentage": manipulator_state.gripper_open_percentage, + "light_grip": light_grip, + "target": { + "normalized_x": round(float(x_pct) * 1000), + "normalized_y": round(float(y_pct) * 1000), + "pixel_x": pixel_x, + "pixel_y": pixel_y, + "image_width": observation.width, + "image_height": observation.height, + "depth_m": depth_m, + "camera_point": { + "frame_name": depth_response.shot.frame_name_image_sensor, + "x": point_x, + "y": point_y, + "z": point_z, + }, + }, + } + + def _apply_light_grip_after_pick( + self, + manipulation_state: int, + *, + max_torque_nm: float, + ) -> dict[str, Any] | None: + """Replace the native post-pick hold with a lower-torque gripper hold.""" + if manipulation_state != manipulation_api_pb2.MANIP_STATE_GRASP_SUCCEEDED: + return None + command_id = self.command.robot_command( + RobotCommandBuilder.claw_gripper_close_command( + max_torque=max_torque_nm, + ) + ) + applying_force = self._block_until_gripper_at_goal(command_id, timeout_sec=5.0) + return { + "command_id": command_id, + "max_torque_nm": max_torque_nm, + "holding": applying_force, + } + + def execute_360_scan( + self, + duration: float = 8.0, + camera_source: str = "hand_color_image", + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=True) + blocking_stand(self.command, timeout_sec=10) + + v_rot = 0.785398 # pi / 4 + command = RobotCommandBuilder.synchro_velocity_command(0.0, 0.0, v_rot) + + command_id = self.command.robot_command( + command, + end_time_secs=time.time() + duration, + timesync_endpoint=self.robot.time_sync.endpoint, + ) + + time.sleep(duration + 0.5) + + return { + "status": "SUCCESS", + "message": f"Completed 360 scan rotation. Hand camera feed '{camera_source}' streamed context.", + "command_id": command_id, + } + + def _project_pixel_to_3d_pose( + self, + observation: ImageObservation, + pixel_x: int, + pixel_y: int, + *, + frame_name: str = frame_helpers.VISION_FRAME_NAME, + depth_window_px: int = 5, + ) -> Pose3D: + if observation.depth_response is None: + raise ValueError("A depth image is required to project a pixel into 3D.") + + depth_response = observation.depth_response + depth_m = _depth_at_pixel(depth_response, [pixel_x, pixel_y], window_px=depth_window_px) + point_sensor = pixel_to_camera_space( + depth_response.source, + pixel_x, + pixel_y, + depth=depth_m, + ) + frame_tform_sensor = frame_helpers.get_a_tform_b( + depth_response.shot.transforms_snapshot, + frame_name, + depth_response.shot.frame_name_image_sensor, + validate=False, + ) + if frame_tform_sensor is None: + raise ValueError( + f"Could not transform {depth_response.shot.frame_name_image_sensor} into {frame_name}." + ) + x, y, z = frame_tform_sensor.transform_point(*point_sensor) + return Pose3D(frame_name=frame_name, x=x, y=y, z=z) + + def place_at_pixel( + self, + x_pct: float, + y_pct: float, + *, + take_lease: bool = True, + timeout: float = 120.0, + standoff_m: float = 0.05, + settle_time_sec: float = 0.5, + ) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + observation = self.capture_rgbd() + + pixel_x = _normalized_coordinate_to_pixel(x_pct, observation.width) + pixel_y = _normalized_coordinate_to_pixel(y_pct, observation.height) + pose_3d = self._project_pixel_to_3d_pose(observation, pixel_x, pixel_y) + + approach_result = self.approach_pose_whole_body( + pose_3d, + standoff_m=standoff_m, + seconds=2.5, + timeout=timeout, + ) + if not approach_result.get("arrived", False): + return { + "status": "FAILED", + "released": False, + "reason": "Arm did not reach the requested placement pose; gripper remains closed.", + "placed_at": {"x": pixel_x, "y": pixel_y}, + "pose_3d": asdict(pose_3d), + "approach_result": approach_result, + } + + time.sleep(_clamp(float(settle_time_sec), 0.0, 3.0)) + release_result = self.open_gripper(open_fraction=1.0, timeout=5.0) + if not release_result.get("at_goal", False): + return { + "status": "FAILED", + "released": False, + "reason": "Gripper did not confirm that it opened; arm was not stowed.", + "placed_at": {"x": pixel_x, "y": pixel_y}, + "pose_3d": asdict(pose_3d), + "approach_result": approach_result, + "release_result": release_result, + } + + stow_result = self.stow_smart(timeout=timeout) + return { + "status": "SUCCESS", + "released": True, + "placed_at": {"x": pixel_x, "y": pixel_y}, + "pose_3d": asdict(pose_3d), + "approach_result": approach_result, + "release_result": release_result, + "stow_result": stow_result, + } + + def stow_smart(self, *, take_lease: bool = True, timeout: float = 20.0) -> dict[str, Any]: + self.require_connected() + with self.lock: + self.ensure_lease(take_if_needed=take_lease) + robot_state = self.state.get_robot_state() + manipulator_state = robot_state.manipulator_state + + is_holding = manipulator_state.is_gripper_holding_item + + if is_holding: + command_id = self.command.robot_command(RobotCommandBuilder.arm_carry_command()) + block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return {"stowed": False, "carrying": True, "message": "Arm moved to carry pose since gripper is holding an item."} + else: + command_id = self.command.robot_command(RobotCommandBuilder.arm_stow_command()) + block_until_arm_arrives(self.command, command_id, timeout_sec=timeout) + return {"stowed": True, "carrying": False, "message": "Arm stowed completely."} + + def wait_for_manipulation(self, command_id: int, *, timeout: float = 30.0, interval: float = 0.5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + request = manipulation_api_pb2.ManipulationApiFeedbackRequest(manipulation_cmd_id=command_id) + feedback = self.manipulation.manipulation_api_feedback_command(request) + if feedback.current_state in TERMINAL_MANIP_STATES: + return feedback + time.sleep(interval) + raise TimeoutError(f"Manipulation command {command_id} did not finish within {timeout} seconds.") + + def detect_external_force_change( + self, + *, + threshold_newtons: float, + sample_window_sec: float, + interval_sec: float, + ) -> ForceChange: + self.require_connected() + baseline = self._end_effector_force_norm() + deadline = time.monotonic() + sample_window_sec + current = baseline + max_delta = 0.0 + while time.monotonic() < deadline: + current = self._end_effector_force_norm() + max_delta = max(max_delta, abs(current - baseline)) + if max_delta >= threshold_newtons: + return ForceChange(baseline, current, max_delta, True) + time.sleep(interval_sec) + return ForceChange(baseline, current, max_delta, False) + + def _end_effector_force_norm(self) -> float: + force = self.state.get_robot_state().manipulator_state.estimated_end_effector_force_in_hand + return math.sqrt(force.x * force.x + force.y * force.y + force.z * force.z) + + def _block_until_gripper_at_goal(self, command_id: int, *, timeout_sec: float) -> bool: + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + feedback = self.command.robot_command_feedback(command_id) + gripper_feedback = feedback.feedback.synchronized_feedback.gripper_command_feedback + if gripper_feedback.HasField("claw_gripper_feedback"): + status = gripper_feedback.claw_gripper_feedback.status + if status in { + gripper_command_pb2.ClawGripperCommand.Feedback.STATUS_AT_GOAL, + gripper_command_pb2.ClawGripperCommand.Feedback.STATUS_APPLYING_FORCE, + }: + return True + time.sleep(0.1) + return False + + +def _clamp(value: float, low: float, high: float) -> float: + return max(low, min(high, value)) + + +def _normalized_coordinate_to_pixel(value: float, size: int) -> int: + if size <= 0: + raise ValueError(f"Image dimension must be positive, got {size}.") + normalized = float(value) + if not 0.0 <= normalized <= 1.0: + raise ValueError(f"Normalized coordinate must be in [0, 1], got {normalized}.") + return int(round(normalized * (size - 1))) + + +def _xyz_dict(position: tuple[float, float, float]) -> dict[str, float]: + return {"x": position[0], "y": position[1], "z": position[2]} + + +def _fiducial_init_value(value: str) -> int: + normalized = value.strip().lower().replace("-", "_") + mapping = { + "none": graph_nav_pb2.SetLocalizationRequest.FIDUCIAL_INIT_NO_FIDUCIAL, + "no_fiducial": graph_nav_pb2.SetLocalizationRequest.FIDUCIAL_INIT_NO_FIDUCIAL, + "nearest": graph_nav_pb2.SetLocalizationRequest.FIDUCIAL_INIT_NEAREST, + "nearest_at_target": graph_nav_pb2.SetLocalizationRequest.FIDUCIAL_INIT_NEAREST_AT_TARGET, + "specific": graph_nav_pb2.SetLocalizationRequest.FIDUCIAL_INIT_SPECIFIC, + } + if normalized not in mapping: + raise ValueError(f"Unsupported fiducial_init {value!r}. Use nearest, nearest_at_target, specific, or none.") + return mapping[normalized] + + +def _localization_to_dict(localization) -> dict[str, Any]: + result: dict[str, Any] = {"waypoint_id": localization.waypoint_id} + if localization.HasField("waypoint_tform_body"): + result["waypoint_tform_body"] = _se3_pose_to_dict(localization.waypoint_tform_body) + if localization.HasField("seed_tform_body"): + result["seed_tform_body"] = _se3_pose_to_dict(localization.seed_tform_body) + if localization.HasField("timestamp"): + result["timestamp_sec"] = localization.timestamp.seconds + localization.timestamp.nanos / 1e9 + return result + + +def _se3_pose_to_dict(pose) -> dict[str, float]: + return { + "x": pose.position.x, + "y": pose.position.y, + "z": pose.position.z, + "qw": pose.rotation.w, + "qx": pose.rotation.x, + "qy": pose.rotation.y, + "qz": pose.rotation.z, + } diff --git a/live-api/spot/apps/api/session_test.py b/live-api/spot/apps/api/session_test.py new file mode 100644 index 0000000..7b92a5f --- /dev/null +++ b/live-api/spot/apps/api/session_test.py @@ -0,0 +1,204 @@ +"""Tests for Spot API session state recovery.""" + +import unittest +from unittest.mock import Mock, patch + +from bosdyn.api import image_pb2, manipulation_api_pb2 +from bosdyn.client.lease import LeaseNotOwnedByWallet + +from apps.api.session import SpotSession, _normalized_coordinate_to_pixel +from apps.manipulation.models import Detection2D, Pose3D +from apps.manipulation.spot_client import constrain_grasp_to_top_down + + +class LeaseRecoveryTest(unittest.TestCase): + + def setUp(self): + self.session = SpotSession() + self.session.robot = Mock() + self.session.lease = Mock() + self.session.lease_keepalive = Mock() + self.session.lease_keepalive.is_alive.return_value = True + + def test_ensure_lease_retains_current_lease(self): + lease = Mock() + self.session.lease.lease_wallet.get_lease.return_value = lease + + self.session.ensure_lease(take_if_needed=True) + + self.session.lease.retain_lease.assert_called_once_with(lease) + + def test_ensure_lease_takes_over_displaced_lease_when_requested(self): + self.session.lease.lease_wallet.get_lease.side_effect = LeaseNotOwnedByWallet( + "body", Mock(lease_status="other owner") + ) + self.session.shutdown_lease = Mock() + self.session.take_lease = Mock() + + self.session.ensure_lease(take_if_needed=True) + + self.session.shutdown_lease.assert_called_once_with() + self.session.take_lease.assert_called_once_with() + + +class PickCoordinateTest(unittest.TestCase): + + def test_maps_normalized_endpoints_inside_image(self): + self.assertEqual(0, _normalized_coordinate_to_pixel(0.0, 640)) + self.assertEqual(639, _normalized_coordinate_to_pixel(1.0, 640)) + + def test_rejects_out_of_range_coordinate(self): + with self.assertRaises(ValueError): + _normalized_coordinate_to_pixel(1.01, 640) + + def test_top_down_constraint_aligns_gripper_x_with_negative_vision_z(self): + pick = manipulation_api_pb2.PickObjectInImage() + + constrain_grasp_to_top_down(pick) + + self.assertEqual("vision", pick.grasp_params.grasp_params_frame_name) + self.assertEqual(1, len(pick.grasp_params.allowable_orientation)) + alignment = pick.grasp_params.allowable_orientation[ + 0 + ].vector_alignment_with_tolerance + self.assertEqual((1.0, 0.0, 0.0), ( + alignment.axis_on_gripper_ewrt_gripper.x, + alignment.axis_on_gripper_ewrt_gripper.y, + alignment.axis_on_gripper_ewrt_gripper.z, + )) + self.assertEqual((0.0, 0.0, -1.0), ( + alignment.axis_to_align_with_ewrt_frame.x, + alignment.axis_to_align_with_ewrt_frame.y, + alignment.axis_to_align_with_ewrt_frame.z, + )) + self.assertAlmostEqual(0.25, alignment.threshold_radians) + + def test_successful_native_pick_switches_to_light_gripper_hold(self): + session = SpotSession() + session.command = Mock() + session.command.robot_command.return_value = 73 + session._block_until_gripper_at_goal = Mock(return_value=True) + + result = session._apply_light_grip_after_pick( + manipulation_api_pb2.MANIP_STATE_GRASP_SUCCEEDED, + max_torque_nm=2.0, + ) + + command = session.command.robot_command.call_args.args[0] + gripper = command.synchronized_command.gripper_command.claw_gripper_command + self.assertEqual(2.0, gripper.maximum_torque.value) + self.assertEqual(2.0, result["max_torque_nm"]) + + +class PlaceSequenceTest(unittest.TestCase): + + def setUp(self): + self.session = SpotSession() + self.session.robot = Mock() + self.session.ensure_lease = Mock() + self.observation = Mock(width=640, height=480) + self.session.capture_rgbd = Mock(return_value=self.observation) + self.pose = Pose3D(frame_name="vision", x=1.0, y=2.0, z=0.5) + self.session._project_pixel_to_3d_pose = Mock(return_value=self.pose) + self.session.open_gripper = Mock(return_value={"at_goal": True}) + self.session.stow_smart = Mock(return_value={"arrived": True}) + + def test_failed_approach_does_not_release_object(self): + self.session.approach_pose_whole_body = Mock( + return_value={"arrived": False, "command_id": 10} + ) + + result = self.session.place_at_pixel(0.5, 0.5, settle_time_sec=0.0) + + self.assertEqual("FAILED", result["status"]) + self.assertFalse(result["released"]) + self.session.open_gripper.assert_not_called() + self.session.stow_smart.assert_not_called() + + @patch("apps.api.session.time.sleep") + def test_releases_only_after_confirmed_arrival(self, sleep): + events = [] + self.session.approach_pose_whole_body = Mock( + side_effect=lambda *args, **kwargs: events.append("arrive") or {"arrived": True} + ) + self.session.open_gripper = Mock( + side_effect=lambda *args, **kwargs: events.append("release") or {"at_goal": True} + ) + self.session.stow_smart = Mock( + side_effect=lambda *args, **kwargs: events.append("stow") or {"arrived": True} + ) + + result = self.session.place_at_pixel(0.5, 0.5) + + self.assertEqual("SUCCESS", result["status"]) + self.assertTrue(result["released"]) + self.assertEqual(["arrive", "release", "stow"], events) + sleep.assert_called_once_with(0.5) + + +class VelocityCommandTest(unittest.TestCase): + + def setUp(self): + self.session = SpotSession() + self.session.robot = Mock() + self.session.robot.time_sync.endpoint = Mock() + self.session.command = Mock() + self.session.command.robot_command.return_value = 42 + self.session.ensure_lease = Mock() + + def test_body_follow_arm_combines_joint_hold_with_mobility(self): + result = self.session.velocity( + v_x=0.2, + v_y=0.0, + v_rot=0.1, + duration=0.5, + body_follow_arm=True, + ) + + command = self.session.command.robot_command.call_args.args[0] + synchronized = command.synchronized_command + self.assertTrue(synchronized.HasField("mobility_command")) + self.assertTrue(synchronized.HasField("arm_command")) + self.assertTrue( + synchronized.arm_command.HasField("arm_joint_move_command") + ) + self.assertTrue(result["body_follow_arm"]) + + +class LightweightDetectionTest(unittest.TestCase): + + @patch("apps.api.session.GeminiObjectDetector") + def test_returns_only_pick_target_metadata(self, detector_class): + session = SpotSession() + session.robot = Mock() + image = Mock() + image.data = b"jpeg" + image.cols = 640 + image.rows = 480 + image.format = image_pb2.Image.FORMAT_JPEG + image.FORMAT_JPEG = image_pb2.Image.FORMAT_JPEG + color_response = Mock() + color_response.shot.image = image + session.capture_image = Mock(return_value=color_response) + detector_class.return_value.detect.return_value = Detection2D( + label="red cube", + confidence=0.93, + bbox_xyxy=(319.5, 191.6, 319.5, 191.6), + grasp_px=(319.5, 191.6), + ) + + result = session.detect_pick_target( + "red cube", model="model", api_key=None + ) + + self.assertEqual("red cube", result["label"]) + self.assertEqual(500, result["target"]["normalized_x"]) + self.assertEqual(400, result["target"]["normalized_y"]) + self.assertNotIn("image", result) + self.assertNotIn("point_cloud", result) + self.assertNotIn("model", result) + self.assertNotIn("pose", result) + + +if __name__ == "__main__": + unittest.main() diff --git a/live-api/spot/apps/api/static/cameras.html b/live-api/spot/apps/api/static/cameras.html new file mode 100644 index 0000000..1681730 --- /dev/null +++ b/live-api/spot/apps/api/static/cameras.html @@ -0,0 +1,467 @@ + + + + + + Spot Camera Debug + + + +
+
+

Spot Camera Debug

+ loading sources +
+
+ + + +
+
+ +
+
+
+ + + + diff --git a/live-api/spot/apps/api/static/detection.html b/live-api/spot/apps/api/static/detection.html new file mode 100644 index 0000000..e2aae32 --- /dev/null +++ b/live-api/spot/apps/api/static/detection.html @@ -0,0 +1,1123 @@ + + + + + + Spot Detection + + + +
+

Spot Detection

+
idle
+
+
+ + +
+
+
+ Spot camera + +
+
+ + + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+
+
+
+ +
+
+
+ + + + diff --git a/live-api/spot/apps/api/static/teleop.html b/live-api/spot/apps/api/static/teleop.html new file mode 100644 index 0000000..b1ecb07 --- /dev/null +++ b/live-api/spot/apps/api/static/teleop.html @@ -0,0 +1,324 @@ + + + + + + Spot Teleop + + + +
+

Spot Teleop

+
idle
+
+
+
+ Spot camera +
+ +
+ + + + diff --git a/live-api/spot/apps/hydration-agent/README.md b/live-api/spot/apps/hydration-agent/README.md new file mode 100644 index 0000000..530bb51 --- /dev/null +++ b/live-api/spot/apps/hydration-agent/README.md @@ -0,0 +1,38 @@ +# Gemini Live Hydration Agent + +This is the agent-driven counterpart to `apps/hydration`. Gemini Live chooses and calls guarded tools; the Node server executes those tools through the Spot FastAPI service. + +## Run + +From this directory: + +```bash +npm install +npm run build +npm start +``` + +Open `http://127.0.0.1:3001`. + +The app reads the repository-level `.config`. Supported settings: + +```ini +GEMINI_API_KEY=... +GEMINI_LIVE_MODEL=gemini-3.1-flash-live-preview +FASTAPI_BASE_URL=http://127.0.0.1:8000 +HYDRATION_AGENT_PORT=3001 +HYDRATION_AGENT_AUTO_START=false +``` + +Spot credentials remain server-side. The browser never receives the API key or robot password. + +## Behavior + +- Structured orders enter a queue and are dispatched to the Live agent one at a time. +- Free-form chat can create orders, inspect state, or call robot tools for debugging. +- While Spot is connected, the server sends the gripper color camera to Gemini Live at 1 FPS for general visual questions. +- Each model function call maps to a validated FastAPI operation. +- Tool calls are serialized. A failed tool skips any remaining calls in that batch, marks the active order failed, and pauses service. +- Pause and stop call `/actions/stop` to interrupt tracked FastAPI actions. +- `GET /api/prompt` returns the active generated system prompt without credentials. +- Context compression and Live session resumption keep the audio/video session available across periodic WebSocket reconnects. diff --git a/live-api/spot/apps/hydration-agent/index.html b/live-api/spot/apps/hydration-agent/index.html new file mode 100644 index 0000000..26e39bb --- /dev/null +++ b/live-api/spot/apps/hydration-agent/index.html @@ -0,0 +1,13 @@ + + + + + + + Gemini Robotics Hydration Agent + + +
+ + + diff --git a/live-api/spot/apps/hydration-agent/package-lock.json b/live-api/spot/apps/hydration-agent/package-lock.json new file mode 100644 index 0000000..be830ed --- /dev/null +++ b/live-api/spot/apps/hydration-agent/package-lock.json @@ -0,0 +1,3095 @@ +{ + "name": "gemini-live-hydration-agent", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "gemini-live-hydration-agent", + "version": "0.1.0", + "dependencies": { + "@google/genai": "^1.31.0", + "express": "^5.1.0", + "lucide-react": "^0.468.0", + "react": "^19.2.1", + "react-dom": "^19.2.1" + }, + "devDependencies": { + "@vitejs/plugin-react": "^5.0.0", + "vite": "^6.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.3.tgz", + "integrity": "sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", + "integrity": "sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.2.tgz", + "integrity": "sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.2.tgz", + "integrity": "sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.2.tgz", + "integrity": "sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.2.tgz", + "integrity": "sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.2.tgz", + "integrity": "sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.2.tgz", + "integrity": "sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.2.tgz", + "integrity": "sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.2.tgz", + "integrity": "sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.2.tgz", + "integrity": "sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.2.tgz", + "integrity": "sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.2.tgz", + "integrity": "sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.2.tgz", + "integrity": "sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.2.tgz", + "integrity": "sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.2.tgz", + "integrity": "sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.2.tgz", + "integrity": "sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.2.tgz", + "integrity": "sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.2.tgz", + "integrity": "sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.2.tgz", + "integrity": "sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.2.tgz", + "integrity": "sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.2.tgz", + "integrity": "sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.2.tgz", + "integrity": "sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.2.tgz", + "integrity": "sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.2.tgz", + "integrity": "sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.2.tgz", + "integrity": "sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "26.1.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", + "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@vitejs/plugin-react": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.2.0.tgz", + "integrity": "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.29.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-rc.3", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.18.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.43", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.43.tgz", + "integrity": "sha512-AjYpR78kDWAY3Efj+cDTFH9t9SCoL7OoTp1BOb0mQV7S+6CiLwnWM3FyxhJtdPufDFKzmCSFoUncKjWgJEZTCQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/browserslist": { + "version": "4.28.6", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.6.tgz", + "integrity": "sha512-FQBYNK15VMslhLHpA7+n+n1GOlF1kId2xcCg7/j95f24AOF6VDYMNH4mFxF7KuaTdv627faazpOAjFzMrfJOUw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.10.42", + "caniuse-lite": "^1.0.30001803", + "electron-to-chromium": "^1.5.389", + "node-releases": "^2.0.51", + "update-browserslist-db": "^1.2.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001805", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001805.tgz", + "integrity": "sha512-52noaS3DubycKSXaU30TwPGIp+POyQSUVa5jBEq3vkRkY0kjyb3LQgvhU6WGyCcyXqVLWO0Cw0Q6BSdD0kUfVA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.392", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.392.tgz", + "integrity": "sha512-1yQq3VQCZRwsnYc67Oc+1fge6Lwtn0hzi6zmEVkB61Zx21kTbwJAW4dFLadl5Rc1tKhG/kSpYXnfiAhu0f0a1g==", + "dev": true, + "license": "ISC" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.2.0.tgz", + "integrity": "sha512-CUVb4wcYe+771XevyH6HtGmXFAGGKkIC3kswAP8Z1JCe0j80JMaTPZH930DWFrvo0atjh18Arc0pEyUCWa5bfg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/google-auth-library": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.0.tgz", + "integrity": "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lucide-react": { + "version": "0.468.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.468.0.tgz", + "integrity": "sha512-6koYRhnM2N0GGZIdXzSeiNwguv1gt/FAjZOiPl76roBi3xKEXa4WmfpxgQwTTL4KipXjefrnf3oV4IsYhi4JFA==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/node-releases": { + "version": "2.0.51", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz", + "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.19", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", + "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/react": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", + "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.7" + } + }, + "node_modules/react-refresh": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", + "integrity": "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rollup": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz", + "integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.62.2", + "@rollup/rollup-android-arm64": "4.62.2", + "@rollup/rollup-darwin-arm64": "4.62.2", + "@rollup/rollup-darwin-x64": "4.62.2", + "@rollup/rollup-freebsd-arm64": "4.62.2", + "@rollup/rollup-freebsd-x64": "4.62.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.2", + "@rollup/rollup-linux-arm-musleabihf": "4.62.2", + "@rollup/rollup-linux-arm64-gnu": "4.62.2", + "@rollup/rollup-linux-arm64-musl": "4.62.2", + "@rollup/rollup-linux-loong64-gnu": "4.62.2", + "@rollup/rollup-linux-loong64-musl": "4.62.2", + "@rollup/rollup-linux-ppc64-gnu": "4.62.2", + "@rollup/rollup-linux-ppc64-musl": "4.62.2", + "@rollup/rollup-linux-riscv64-gnu": "4.62.2", + "@rollup/rollup-linux-riscv64-musl": "4.62.2", + "@rollup/rollup-linux-s390x-gnu": "4.62.2", + "@rollup/rollup-linux-x64-gnu": "4.62.2", + "@rollup/rollup-linux-x64-musl": "4.62.2", + "@rollup/rollup-openbsd-x64": "4.62.2", + "@rollup/rollup-openharmony-arm64": "4.62.2", + "@rollup/rollup-win32-arm64-msvc": "4.62.2", + "@rollup/rollup-win32-ia32-msvc": "4.62.2", + "@rollup/rollup-win32-x64-gnu": "4.62.2", + "@rollup/rollup-win32-x64-msvc": "4.62.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/live-api/spot/apps/hydration-agent/package.json b/live-api/spot/apps/hydration-agent/package.json new file mode 100644 index 0000000..949913c --- /dev/null +++ b/live-api/spot/apps/hydration-agent/package.json @@ -0,0 +1,26 @@ +{ + "name": "gemini-live-hydration-agent", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "start": "node server/index.mjs", + "test": "node --test server/*.test.mjs" + }, + "engines": { + "node": ">=20" + }, + "dependencies": { + "@google/genai": "^1.31.0", + "express": "^5.1.0", + "lucide-react": "^0.468.0", + "react": "^19.2.1", + "react-dom": "^19.2.1" + }, + "devDependencies": { + "@vitejs/plugin-react": "^5.0.0", + "vite": "^6.2.0" + } +} diff --git a/live-api/spot/apps/hydration-agent/server/agent.mjs b/live-api/spot/apps/hydration-agent/server/agent.mjs new file mode 100644 index 0000000..6f03b9b --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/agent.mjs @@ -0,0 +1,570 @@ +import { GoogleGenAI, Modality } from "@google/genai"; +import { DRINKS } from "./catalog.mjs"; +import { buildSystemPrompt } from "./prompt.mjs"; +import { TOOL_DECLARATIONS } from "./tools.mjs"; + +export class GeminiHydrationAgent { + constructor({ state, tools, config }) { + this.state = state; + this.tools = tools; + this.config = config; + this.ai = null; + this.session = null; + this.connectPromise = null; + this.pendingInputs = []; + this.currentInputKind = null; + this.currentAssistantMessageId = null; + this.messageChain = Promise.resolve(); + this.continuationCounts = new Map(); + this.pumpScheduled = false; + this.sessionGeneration = 0; + this.sessionResumptionHandle = null; + this.reconnectTimer = null; + this.cameraInterval = null; + this.cameraFrameInFlight = false; + this.cameraIntervalMs = Math.max(1_000, Number(config.HYDRATION_AGENT_CAMERA_INTERVAL_MS || 1_000)); + this.lastCameraHealthCheck = 0; + this.toolExecutionActive = 0; + this.visualWatchTickQueued = false; + this.lastVisualWatchTick = 0; + this.visualWatchTurnTimer = null; + this.visualWatchIntervalMs = Math.max( + 1_000, + Number(config.HYDRATION_AGENT_VISUAL_WATCH_INTERVAL_MS || 2_000), + ); + this.visualWatchTurnTimeoutMs = Math.max( + 5_000, + Number(config.HYDRATION_AGENT_VISUAL_WATCH_TURN_TIMEOUT_MS || 10_000), + ); + } + + systemPrompt() { + return buildSystemPrompt({ + drinks: DRINKS, + waypoints: this.state.waypoints, + toolDescriptions: this.tools.descriptions(), + }); + } + + async connect() { + if (this.session) return this.session; + if (this.connectPromise) return this.connectPromise; + if (!this.config.GEMINI_API_KEY) throw new Error("GEMINI_API_KEY is missing from .config."); + + this.connectPromise = this.openSession(); + try { + return await this.connectPromise; + } finally { + this.connectPromise = null; + } + } + + async openSession() { + const generation = ++this.sessionGeneration; + try { + try { + await this.tools.refreshWaypoints(); + } catch (error) { + this.state.log("warning", `Could not refresh waypoints for prompt: ${errorMessage(error)}`); + } + try { + await this.tools.getRobotStatus(); + } catch (error) { + this.state.log("warning", `Could not read Spot status for Live vision: ${errorMessage(error)}`); + } + + this.ai = new GoogleGenAI({ apiKey: this.config.GEMINI_API_KEY }); + const session = await this.ai.live.connect({ + model: this.state.model, + config: { + responseModalities: [Modality.AUDIO], + outputAudioTranscription: {}, + systemInstruction: this.systemPrompt(), + tools: [{ functionDeclarations: TOOL_DECLARATIONS }], + contextWindowCompression: { slidingWindow: {} }, + sessionResumption: this.sessionResumptionHandle + ? { handle: this.sessionResumptionHandle } + : {}, + thinkingConfig: { thinkingLevel: "low" }, + }, + callbacks: { + onopen: () => { + if (generation !== this.sessionGeneration) return; + this.state.agentConnected = true; + this.state.lastError = null; + this.state.log("agent", `Gemini Live connected: ${this.state.model}`); + this.startCameraStream(); + }, + onmessage: (message) => { + if (generation !== this.sessionGeneration) return; + this.messageChain = this.messageChain + .then(() => this.handleMessage(message, generation)) + .catch((error) => this.handleAgentError(error)); + }, + onerror: (error) => { + if (generation === this.sessionGeneration) this.handleAgentError(error); + }, + onclose: (event) => { + if (generation !== this.sessionGeneration) return; + this.stopCameraStream(); + this.clearVisualWatchTurnTimer(); + if (this.currentInputKind === "visual-watch") this.visualWatchTickQueued = false; + this.currentInputKind = null; + this.session = null; + this.state.agentConnected = false; + if (!this.toolExecutionActive) this.state.agentBusy = false; + this.state.log("agent", `Gemini Live closed${event?.reason ? `: ${event.reason}` : ""}`); + this.scheduleReconnect(); + }, + }, + }); + this.session = session; + this.state.agentConnected = true; + this.state.changed(); + return session; + } catch (error) { + this.ai = null; + this.session = null; + this.state.agentConnected = false; + this.state.lastError = errorMessage(error); + this.state.changed(); + throw error; + } + } + + async start() { + this.state.serviceRunning = true; + this.state.paused = false; + await this.connect(); + this.state.log("service", "Agent service started"); + this.schedulePump(); + return this.state.snapshot(); + } + + async resume() { + this.state.serviceRunning = true; + this.state.paused = false; + await this.connect(); + this.state.log("service", "Agent service resumed"); + this.schedulePump(); + return this.state.snapshot(); + } + + async pause() { + this.state.paused = true; + this.state.log("service", "Agent service paused; stopping active robot actions"); + try { + await this.tools.fastApi.stopRobot(); + } catch (error) { + this.state.log("warning", `Robot stop while pausing: ${errorMessage(error)}`); + } + return this.state.snapshot(); + } + + async stop() { + this.state.serviceRunning = false; + this.state.paused = true; + this.state.log("service", "Agent service stopped; stopping active robot actions"); + try { + await this.tools.fastApi.stopRobot(); + } catch (error) { + this.state.log("warning", `Robot stop: ${errorMessage(error)}`); + } + return this.state.snapshot(); + } + + async resetSession() { + this.stopCameraStream(); + this.clearVisualWatchTurnTimer(); + this.clearReconnectTimer(); + this.sessionGeneration += 1; + this.sessionResumptionHandle = null; + if (this.session) { + try { + this.session.close(); + } catch { + // A closed Live socket needs no further cleanup. + } + } + this.session = null; + this.ai = null; + this.state.agentConnected = false; + this.state.agentBusy = false; + this.currentAssistantMessageId = null; + this.currentInputKind = null; + this.visualWatchTickQueued = false; + this.state.log("agent", "Gemini Live session reset"); + if (this.state.serviceRunning) await this.connect(); + this.schedulePump(); + return this.state.snapshot(); + } + + queueOrder(order) { + this.state.log("agent", `Order #${order.id} is ready for agent dispatch`); + this.schedulePump(); + } + + queueChat(text) { + this.pendingInputs.unshift({ kind: "chat", text }); + this.schedulePump(); + } + + schedulePump() { + if (this.pumpScheduled) return; + this.pumpScheduled = true; + setTimeout(() => { + this.pumpScheduled = false; + this.pump().catch((error) => this.handleAgentError(error)); + }, 0); + } + + async pump() { + if (this.state.agentBusy || this.toolExecutionActive) return; + if (!this.session) await this.connect(); + + const input = this.pendingInputs.shift(); + if (input) { + this.sendText(input.text, input.kind); + return; + } + + if (!this.state.serviceRunning || this.state.paused) return; + if (this.state.activeOrderId) { + const activeOrder = this.state.orders.find((candidate) => candidate.id === this.state.activeOrderId); + if (activeOrder && !["finished", "failed", "cancelled"].includes(activeOrder.status)) { + this.sendText(this.activeOrderRecoveryPrompt(activeOrder), "order-recovery"); + return; + } + } + const order = this.state.orders.find((candidate) => candidate.status === "pending"); + if (!order) return; + + this.state.activeOrderId = order.id; + this.state.addMessage( + "system", + `Dispatching order #${order.id}: ${order.drinkName} to ${order.destination}.`, + { orderId: order.id }, + ); + this.sendText( + `Execute hydration order #${order.id}. Drink ID: ${order.drinkId}. Drink: ${order.drinkName}. ` + + `Use detection instruction exactly: "${order.detectInstruction}". Delivery waypoint: ${order.destination}. ` + + "Follow the normal order workflow completely, update the tracked order status, and stop immediately on any error.", + "order", + ); + } + + sendText(text, kind = "chat") { + if (!this.session) throw new Error("Gemini Live session is not connected."); + this.state.agentBusy = true; + this.currentInputKind = kind; + this.currentAssistantMessageId = null; + this.state.changed(); + this.session.sendRealtimeInput({ text }); + if (kind === "visual-watch") { + this.clearVisualWatchTurnTimer(); + this.visualWatchTurnTimer = setTimeout( + () => this.recoverStuckVisualWatchTurn(), + this.visualWatchTurnTimeoutMs, + ); + } + } + + async handleMessage(message, generation) { + const resumption = message.sessionResumptionUpdate; + if (resumption?.resumable && resumption.newHandle) { + this.sessionResumptionHandle = resumption.newHandle; + } + if (message.goAway?.timeLeft) { + this.state.log("agent", `Gemini Live connection renewal in ${message.goAway.timeLeft}`); + } + + const transcription = message.serverContent?.outputTranscription?.text; + const modelText = extractModelText(message); + const text = transcription || modelText; + if (text && this.currentInputKind !== "visual-watch") { + if (!this.currentAssistantMessageId) { + this.currentAssistantMessageId = this.state.addMessage("assistant", "").id; + } + this.state.appendMessage(this.currentAssistantMessageId, text); + } + + if (message.toolCall?.functionCalls?.length) { + await this.handleToolCalls(message.toolCall.functionCalls, generation, this.currentInputKind); + } + + if (message.serverContent?.turnComplete) { + this.clearVisualWatchTurnTimer(); + if (this.currentInputKind === "visual-watch") this.visualWatchTickQueued = false; + this.currentInputKind = null; + this.currentAssistantMessageId = null; + this.state.agentBusy = false; + this.finishActiveTurn(); + this.state.changed(); + this.schedulePump(); + } + + if (message.serverContent?.interrupted) { + this.state.log("agent", "Gemini response interrupted"); + } + } + + async handleToolCalls(functionCalls, generation, inputKind) { + this.toolExecutionActive += 1; + const responses = []; + let failure = null; + const evaluatedWatches = new Set(); + try { + for (const functionCall of functionCalls) { + const args = functionCall.args || {}; + const trace = this.state.addToolCall(functionCall.name, args); + let result; + const watchId = Number(args.watch_id); + const visualTickViolation = inputKind === "visual-watch" && ( + functionCall.name !== "evaluate_visual_watch" || evaluatedWatches.has(watchId) + ); + const externalEvaluation = inputKind !== "visual-watch" && functionCall.name === "evaluate_visual_watch"; + if (visualTickViolation || externalEvaluation) { + const reason = visualTickViolation + ? "Visual-watch ticks may only evaluate each listed watch once." + : "evaluate_visual_watch is only available during a private visual-watch tick."; + result = { ok: false, error: reason }; + this.state.finishToolCall(trace, null, reason); + } else if (failure) { + result = { ok: false, error: `Skipped because ${failure.tool} failed: ${failure.error}` }; + this.state.finishToolCall(trace, null, result.error); + } else { + try { + if (functionCall.name === "evaluate_visual_watch") evaluatedWatches.add(watchId); + result = await this.tools.execute(functionCall.name, args); + this.state.finishToolCall(trace, result); + } catch (error) { + const message = errorMessage(error); + failure = { tool: functionCall.name, error: message }; + result = { ok: false, error: message, instruction: "Stop this workflow; do not call another action." }; + this.state.finishToolCall(trace, null, message); + if (inputKind === "visual-watch") { + this.state.log("warning", `Visual watch evaluation failed: ${message}`); + } else { + this.failActiveOrder(message); + } + } + } + responses.push({ + id: functionCall.id, + name: functionCall.name, + response: { result }, + }); + } + } finally { + this.toolExecutionActive -= 1; + } + + if (this.session && generation === this.sessionGeneration) { + this.session.sendToolResponse({ functionResponses: responses }); + return; + } + + this.sessionResumptionHandle = null; + this.state.agentBusy = false; + if (!failure) { + this.pendingInputs.unshift({ + kind: "recovery", + text: `The Live connection closed after these tools completed: ${compactJson(responses)}. ` + + "Do not repeat successful tools. Report their results and continue from the tracked service state.", + }); + } + this.scheduleReconnect(0); + this.schedulePump(); + } + + startCameraStream() { + this.stopCameraStream(); + this.state.cameraStreaming = true; + this.state.cameraError = null; + this.state.changed(); + this.sendCameraFrame(); + this.cameraInterval = setInterval(() => this.sendCameraFrame(), this.cameraIntervalMs); + } + + stopCameraStream() { + if (this.cameraInterval) clearInterval(this.cameraInterval); + this.cameraInterval = null; + this.cameraFrameInFlight = false; + if (this.state.cameraStreaming) { + this.state.cameraStreaming = false; + this.state.changed(); + } + } + + async sendCameraFrame() { + if (this.cameraFrameInFlight || !this.session) return; + this.cameraFrameInFlight = true; + const generation = this.sessionGeneration; + const session = this.session; + try { + if (!this.state.robotConnected && Date.now() - this.lastCameraHealthCheck >= 5_000) { + this.lastCameraHealthCheck = Date.now(); + const health = await this.tools.fastApi.get("/health", { timeoutMs: 3_000, tracked: false }); + this.state.robotConnected = health.connected === true; + } + if (!this.state.robotConnected) return; + + const frame = await this.tools.fastApi.getBinary( + "/images/hand_color_image?quality_percent=65", + { timeoutMs: 4_000 }, + ); + if (generation !== this.sessionGeneration || session !== this.session) return; + session.sendRealtimeInput({ + video: { + data: frame.data.toString("base64"), + mimeType: frame.mimeType, + }, + }); + this.state.cameraFrameAt = new Date().toISOString(); + this.state.cameraFramesSent += 1; + this.state.cameraError = null; + if (this.state.cameraFramesSent === 1 || this.state.cameraFramesSent % 5 === 0) this.state.changed(); + this.scheduleVisualWatchTick(); + } catch (error) { + const message = errorMessage(error); + if (this.state.cameraError !== message) { + this.state.cameraError = message; + this.state.log("warning", `Gemini camera stream: ${message}`); + } + } finally { + this.cameraFrameInFlight = false; + } + } + + scheduleVisualWatchTick() { + const activeWatches = this.state.visualWatches.filter((watch) => watch.status === "active"); + if (!activeWatches.length || this.visualWatchTickQueued || this.currentInputKind === "visual-watch") return; + if (Date.now() - this.lastVisualWatchTick < this.visualWatchIntervalMs) return; + + this.lastVisualWatchTick = Date.now(); + this.visualWatchTickQueued = true; + this.pendingInputs.push({ + kind: "visual-watch", + text: "PRIVATE VISUAL WATCH TICK. Inspect only the latest gripper-camera frame. " + + "Call evaluate_visual_watch exactly once for each active watch below. " + + "Set subject_visible and condition_met conservatively from visible evidence. " + + "Do not call any other tool. After all evaluations, respond with exactly DONE; this response is hidden. " + + "Active watches: " + + compactJson(activeWatches.map((watch) => ({ + watch_id: watch.id, + subject: watch.subject, + condition: watch.condition, + requires_prior_presence: watch.requiresPriorPresence, + armed: watch.armed, + }))), + }); + this.schedulePump(); + } + + scheduleReconnect(delayMs = 1_000) { + if (!this.state.serviceRunning || this.session || this.connectPromise || this.reconnectTimer) return; + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + if (this.toolExecutionActive) { + this.scheduleReconnect(500); + return; + } + this.connect() + .then(() => this.schedulePump()) + .catch((error) => { + this.state.log("error", `Gemini Live reconnect failed: ${errorMessage(error)}`); + this.scheduleReconnect(2_000); + }); + }, delayMs); + } + + clearReconnectTimer() { + if (this.reconnectTimer) clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + + clearVisualWatchTurnTimer() { + if (this.visualWatchTurnTimer) clearTimeout(this.visualWatchTurnTimer); + this.visualWatchTurnTimer = null; + } + + recoverStuckVisualWatchTurn() { + this.visualWatchTurnTimer = null; + if (this.currentInputKind !== "visual-watch") return; + this.state.log("warning", "Visual watch turn timed out; renewing the Gemini Live session"); + this.visualWatchTickQueued = false; + this.resetSession().catch((error) => this.handleAgentError(error)); + } + + activeOrderRecoveryPrompt(order) { + const recentTools = this.state.toolCalls.slice(-12).map((call) => ({ + name: call.name, + status: call.status, + result: call.result, + error: call.error, + })); + return `Resume active hydration order #${order.id} (${order.drinkName} to ${order.destination}). ` + + `Current order status: ${order.status}. Recent completed tool history: ${compactJson(recentTools)}. ` + + "Do not repeat successful physical actions. Continue from the next required workflow step."; + } + + failActiveOrder(message) { + this.state.lastError = message; + if (this.state.activeOrderId) { + const order = this.state.orders.find((candidate) => candidate.id === this.state.activeOrderId); + if (order && !["finished", "failed", "cancelled"].includes(order.status)) { + this.state.updateOrder(order.id, "failed", message); + } + this.state.paused = true; + this.state.log("service", "Service paused after tool failure"); + } + this.state.changed(); + } + + finishActiveTurn() { + if (!this.state.activeOrderId) return; + const order = this.state.orders.find((candidate) => candidate.id === this.state.activeOrderId); + if (!order || ["finished", "failed", "cancelled"].includes(order.status)) { + this.continuationCounts.delete(this.state.activeOrderId); + this.state.activeOrderId = null; + return; + } + if (this.state.paused || !this.state.serviceRunning) return; + + const attempts = (this.continuationCounts.get(order.id) || 0) + 1; + this.continuationCounts.set(order.id, attempts); + if (attempts <= 2) { + this.pendingInputs.unshift({ + kind: "continuation", + text: `Order #${order.id} is still ${order.status}. Continue the workflow now. If it cannot continue, mark it failed and explain why.`, + }); + return; + } + this.state.updateOrder(order.id, "failed", "Agent ended without completing the order workflow."); + this.state.paused = true; + this.state.lastError = `Order #${order.id} did not reach a terminal state.`; + this.state.log("service", "Service paused because the agent stopped before completing its order"); + } + + handleAgentError(error) { + const message = errorMessage(error); + this.state.lastError = message; + this.state.agentBusy = false; + this.state.log("error", message); + this.failActiveOrder(message); + } +} + +function extractModelText(message) { + const parts = message.serverContent?.modelTurn?.parts || []; + return parts.map((part) => part.text || "").join(""); +} + +function compactJson(value, maxLength = 4_000) { + const text = JSON.stringify(value); + return text.length > maxLength ? `${text.slice(0, maxLength - 3)}...` : text; +} + +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} diff --git a/live-api/spot/apps/hydration-agent/server/agent.test.mjs b/live-api/spot/apps/hydration-agent/server/agent.test.mjs new file mode 100644 index 0000000..afbc551 --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/agent.test.mjs @@ -0,0 +1,131 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { DRINKS, isDeliveryWaypoint } from "./catalog.mjs"; +import { validateRobotResult } from "./fastapi.mjs"; +import { buildSystemPrompt } from "./prompt.mjs"; +import { HydrationState } from "./state.mjs"; +import { HydrationTools } from "./tools.mjs"; + +test("delivery waypoints exclude service-only positions", () => { + const waypoints = ["peng-desk", "caden-desk"]; + assert.equal(isDeliveryWaypoint("peng-desk", waypoints), true); + assert.equal(isDeliveryWaypoint("home", waypoints), false); + assert.equal(isDeliveryWaypoint("snack1", waypoints), false); + assert.equal(isDeliveryWaypoint("made-up", waypoints), false); +}); + +test("prompt contains dynamic catalog, waypoints, and fail-stop rules", () => { + const prompt = buildSystemPrompt({ + drinks: DRINKS, + waypoints: ["peng-desk"], + toolDescriptions: "- navigate_to: calls /navigate", + }); + assert.match(prompt, /blue-hint-water/); + assert.match(prompt, /middle of red drink can/); + assert.match(prompt, /peng-desk/); + assert.match(prompt, /continuously receive the gripper color camera as video at 1 frame per second/); + assert.match(prompt, /Never say that you cannot passively stream video/); + assert.match(prompt, /you MUST call create_visual_watch/); + assert.match(prompt, /call evaluate_visual_watch exactly once/); + assert.match(prompt, /Map "right side of the floor" to horizontal=right and vertical=down/); + assert.match(prompt, /use aim_gripper_camera before saying the object is not visible/); + assert.match(prompt, /Stop the workflow immediately after any tool error/); + assert.match(prompt, /navigate_to: calls \/navigate/); +}); + +test("camera aiming maps right and down to Spot arm rotations", async () => { + const calls = []; + const tools = new HydrationTools({ + fastApi: { + post: async (path, body) => { + calls.push({ path, body }); + return { arrived: true }; + }, + }, + state: { paused: false }, + config: { HYDRATION_AGENT_CAMERA_SETTLE_MS: 0 }, + }); + + const result = await tools.execute("aim_gripper_camera", { + horizontal: "right", + vertical: "down", + degrees: 12, + }); + + assert.equal(calls[0].path, "/arm/jog"); + assert.ok(calls[0].body.dyaw < 0); + assert.ok(calls[0].body.dpitch > 0); + assert.equal(result.camera_settled, true); +}); + +test("arm_set_pose compatibility alias dispatches to the declared arm pose tool", async () => { + const calls = []; + const tools = new HydrationTools({ + fastApi: { + post: async (path, body) => { + calls.push({ path, body }); + return { arrived: true }; + }, + }, + state: { paused: false }, + config: {}, + }); + + await tools.execute("arm_set_pose", { pose: "stow" }); + + assert.equal(calls[0].path, "/arm/stow"); + assert.equal(calls[0].body.take_lease, true); +}); + +test("robot result validation rejects incomplete actions", () => { + assert.throws( + () => validateRobotResult("/navigate", { reached_goal: false, status: "STATUS_STUCK" }), + /STATUS_STUCK/, + ); + assert.throws( + () => validateRobotResult("/arm/stow", { arrived: false }), + /did not arrive/, + ); + assert.doesNotThrow(() => validateRobotResult("/navigate", { reached_goal: true, status: "STATUS_REACHED_GOAL" })); +}); + +test("leave watches arm on presence and trigger after two matching frames", () => { + const state = new HydrationState("test-model"); + const watch = state.createVisualWatch({ + subject: "the man in the red shirt", + condition: "the man in the red shirt leaves the camera view", + responseMessage: "Hi!", + requiresPriorPresence: true, + }); + + state.evaluateVisualWatch(watch.id, { + subjectVisible: false, + conditionMet: true, + observation: "No red-shirted person is visible.", + }); + assert.equal(watch.armed, false); + assert.equal(watch.consecutiveMatches, 0); + + state.evaluateVisualWatch(watch.id, { + subjectVisible: true, + conditionMet: false, + observation: "A man in a red shirt is visible.", + }); + assert.equal(watch.armed, true); + + state.evaluateVisualWatch(watch.id, { + subjectVisible: false, + conditionMet: true, + observation: "The previously visible man is absent.", + }); + assert.equal(watch.status, "active"); + state.evaluateVisualWatch(watch.id, { + subjectVisible: false, + conditionMet: true, + observation: "The man remains absent.", + }); + + assert.equal(watch.status, "triggered"); + assert.equal(state.messages.at(-1).text, "Hi!"); + assert.equal(state.messages.at(-1).proactive, true); +}); diff --git a/live-api/spot/apps/hydration-agent/server/catalog.mjs b/live-api/spot/apps/hydration-agent/server/catalog.mjs new file mode 100644 index 0000000..8b7c82f --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/catalog.mjs @@ -0,0 +1,31 @@ +export const DRINKS = [ + { + id: "blue-hint-water", + name: "Blue Hint Water", + flavor: "Blackberry", + detectInstruction: "middle of blue drink can", + color: "#3478c9", + }, + { + id: "red-hint-water", + name: "Red Hint Water", + flavor: "Lemon", + detectInstruction: "middle of red drink can", + color: "#c84a43", + }, + { + id: "monster-energy", + name: "Monster Energy Drink", + flavor: "Energy drink", + detectInstruction: "black drink can", + color: "#3d8847", + }, +]; + +export function findDrink(id) { + return DRINKS.find((drink) => drink.id === id); +} + +export function isDeliveryWaypoint(name, waypoints) { + return Boolean(name && name !== "home" && name !== "snack1" && waypoints.includes(name)); +} diff --git a/live-api/spot/apps/hydration-agent/server/config.mjs b/live-api/spot/apps/hydration-agent/server/config.mjs new file mode 100644 index 0000000..80fd23e --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/config.mjs @@ -0,0 +1,29 @@ +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const appDirectory = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +export function loadConfig() { + const values = {}; + const paths = [ + path.join(appDirectory, ".config"), + path.resolve(appDirectory, "..", "..", ".config"), + ]; + + for (const configPath of paths) { + if (!existsSync(configPath)) continue; + for (const rawLine of readFileSync(configPath, "utf8").split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#") || !line.includes("=")) continue; + const separator = line.indexOf("="); + const key = line.slice(0, separator).trim(); + const value = line.slice(separator + 1).trim().replace(/^["']|["']$/g, ""); + if (key) values[key] = value; + } + } + + return { ...values, ...process.env }; +} + +export { appDirectory }; diff --git a/live-api/spot/apps/hydration-agent/server/fastapi.mjs b/live-api/spot/apps/hydration-agent/server/fastapi.mjs new file mode 100644 index 0000000..7dea9ac --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/fastapi.mjs @@ -0,0 +1,85 @@ +export class FastApiClient { + constructor(baseUrl) { + this.baseUrl = baseUrl.replace(/\/$/, ""); + this.activeControllers = new Set(); + } + + get(pathname, options = {}) { + return this.request(pathname, { ...options, method: "GET" }); + } + + post(pathname, body = {}, options = {}) { + return this.request(pathname, { ...options, method: "POST", body }); + } + + async getBinary(pathname, { timeoutMs = 5_000 } = {}) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(new Error(`Timed out calling ${pathname}.`)), timeoutMs); + try { + const response = await fetch(`${this.baseUrl}${pathname}`, { signal: controller.signal }); + if (!response.ok) { + const detail = await response.text(); + throw new Error(detail || `${response.status} ${response.statusText}`); + } + return { + data: Buffer.from(await response.arrayBuffer()), + mimeType: response.headers.get("content-type") || "image/jpeg", + }; + } finally { + clearTimeout(timeout); + } + } + + async request(pathname, { method, body, timeoutMs = 190_000, tracked = true }) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(new Error(`Timed out calling ${pathname}.`)), timeoutMs); + if (tracked) this.activeControllers.add(controller); + try { + const response = await fetch(`${this.baseUrl}${pathname}`, { + method, + headers: body === undefined ? undefined : { "content-type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: controller.signal, + }); + const text = await response.text(); + let data = null; + try { + data = text ? JSON.parse(text) : null; + } catch { + data = { raw: text }; + } + if (!response.ok) { + throw new Error(data?.detail || data?.error || `${response.status} ${response.statusText}`); + } + validateRobotResult(pathname, data); + return data; + } finally { + clearTimeout(timeout); + this.activeControllers.delete(controller); + } + } + + cancelActive() { + for (const controller of this.activeControllers) controller.abort(new Error("Action stopped by operator.")); + this.activeControllers.clear(); + } + + async stopRobot() { + this.cancelActive(); + return this.post( + "/actions/stop", + { take_lease: true, freeze_arm: true }, + { timeoutMs: 15_000, tracked: false }, + ); + } +} + +export function validateRobotResult(pathname, result) { + if (!result || typeof result !== "object") return; + if (result.reached_goal === false) throw new Error(`${pathname}: ${result.status || "goal not reached"}`); + if (result.arrived === false) throw new Error(`${pathname}: arm did not arrive`); + if (result.at_goal === false) throw new Error(`${pathname}: gripper did not reach goal`); + if (typeof result.status === "string" && /STUCK|CANCELLED|TIMED_OUT|LOST|FAILED|ERROR/.test(result.status)) { + throw new Error(`${pathname}: ${result.status}`); + } +} diff --git a/live-api/spot/apps/hydration-agent/server/index.mjs b/live-api/spot/apps/hydration-agent/server/index.mjs new file mode 100644 index 0000000..d706c43 --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/index.mjs @@ -0,0 +1,158 @@ +import express from "express"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { appDirectory, loadConfig } from "./config.mjs"; +import { DRINKS, findDrink, isDeliveryWaypoint } from "./catalog.mjs"; +import { HydrationState } from "./state.mjs"; +import { FastApiClient } from "./fastapi.mjs"; +import { HydrationTools } from "./tools.mjs"; +import { GeminiHydrationAgent } from "./agent.mjs"; + +const config = loadConfig(); +const port = Number(config.HYDRATION_AGENT_PORT || 3001); +const host = config.HYDRATION_AGENT_HOST || "127.0.0.1"; +const fastApiBaseUrl = config.FASTAPI_BASE_URL || "http://127.0.0.1:8000"; +const model = config.GEMINI_LIVE_MODEL || "gemini-3.1-flash-live-preview"; + +const state = new HydrationState(model); +const fastApi = new FastApiClient(fastApiBaseUrl); +const tools = new HydrationTools({ fastApi, state, config }); +const agent = new GeminiHydrationAgent({ state, tools, config }); +const app = express(); +const eventClients = new Set(); + +app.use(express.json({ limit: "1mb" })); + +app.get("/api/bootstrap", async (_req, res) => { + try { + await tools.refreshWaypoints(); + } catch (error) { + state.log("warning", `Waypoint refresh failed: ${errorMessage(error)}`); + } + res.json({ + drinks: DRINKS, + waypoints: state.waypoints, + model, + fastApiBaseUrl, + hasGeminiKey: Boolean(config.GEMINI_API_KEY), + hasSpotPassword: Boolean(config.BOSDYN_CLIENT_PASSWORD), + }); +}); + +app.get("/api/state", (_req, res) => res.json(state.snapshot())); + +app.get("/api/events", (req, res) => { + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + connection: "keep-alive", + }); + res.write(`data: ${JSON.stringify(state.snapshot())}\n\n`); + eventClients.add(res); + const keepAlive = setInterval(() => res.write(": keep-alive\n\n"), 15_000); + req.on("close", () => { + clearInterval(keepAlive); + eventClients.delete(res); + }); +}); + +state.on("changed", (snapshot) => { + const event = `data: ${JSON.stringify(snapshot)}\n\n`; + for (const client of eventClients) client.write(event); +}); + +app.post("/api/orders", asyncHandler(async (req, res) => { + const drink = findDrink(String(req.body.drinkId || "")); + if (!drink) return res.status(400).json({ error: "Unknown drink." }); + const destination = String(req.body.destination || "").trim(); + if (!isDeliveryWaypoint(destination, state.waypoints)) { + return res.status(400).json({ error: "Choose a current delivery waypoint other than home or snack1." }); + } + const order = state.createOrder(drink, destination); + agent.queueOrder(order); + res.status(201).json({ order, state: state.snapshot() }); +})); + +app.post("/api/chat", asyncHandler(async (req, res) => { + const text = String(req.body.message || "").trim(); + if (!text) return res.status(400).json({ error: "Message is required." }); + state.addMessage("user", text); + await agent.connect(); + agent.queueChat(text); + res.status(202).json(state.snapshot()); +})); + +app.post("/api/service/start", asyncHandler(async (_req, res) => res.json(await agent.start()))); +app.post("/api/service/resume", asyncHandler(async (_req, res) => res.json(await agent.resume()))); +app.post("/api/service/pause", asyncHandler(async (_req, res) => res.json(await agent.pause()))); +app.post("/api/service/stop", asyncHandler(async (_req, res) => res.json(await agent.stop()))); +app.post("/api/service/reset-session", asyncHandler(async (_req, res) => res.json(await agent.resetSession()))); + +app.post("/api/service/connect-robot", asyncHandler(async (_req, res) => { + const trace = state.addToolCall("connect_robot", {}); + try { + const result = await tools.execute("connect_robot", {}); + state.finishToolCall(trace, result); + res.json({ result, state: state.snapshot() }); + } catch (error) { + state.finishToolCall(trace, null, errorMessage(error)); + throw error; + } +})); + +app.post("/api/service/stop-actions", asyncHandler(async (_req, res) => { + const result = await fastApi.stopRobot(); + state.log("service", "Immediate robot stop requested by operator"); + res.json({ result, state: state.snapshot() }); +})); + +app.get("/api/prompt", (_req, res) => res.json({ prompt: agent.systemPrompt() })); + +app.get("/api/camera/gripper", asyncHandler(async (_req, res) => { + const response = await fetch(`${fastApiBaseUrl}/images/hand_color_image?quality_percent=75`); + if (!response.ok) throw new Error(await response.text()); + res.set("content-type", response.headers.get("content-type") || "image/jpeg"); + res.set("cache-control", "no-store"); + res.send(Buffer.from(await response.arrayBuffer())); +})); + +const distDirectory = path.join(appDirectory, "dist"); +if (existsSync(distDirectory)) { + app.use(express.static(distDirectory)); + app.get("/{*path}", (_req, res) => res.sendFile(path.join(distDirectory, "index.html"))); +} else { + app.get("/", (_req, res) => res.status(503).send("Frontend is not built. Run npm run build.")); +} + +app.use((error, _req, res, _next) => { + const message = errorMessage(error); + state.lastError = message; + state.log("error", message); + res.status(500).json({ error: message }); +}); + +const server = app.listen(port, host, () => { + state.log("server", `Gemini Live hydration agent listening on http://${host}:${port}`); +}); + +if (config.HYDRATION_AGENT_AUTO_START === "true") { + agent.start().catch((error) => state.log("error", `Auto-start failed: ${errorMessage(error)}`)); +} + +for (const signal of ["SIGINT", "SIGTERM"]) { + process.on(signal, () => { + try { + agent.session?.close(); + } finally { + server.close(() => process.exit(0)); + } + }); +} + +function asyncHandler(handler) { + return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next); +} + +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} diff --git a/live-api/spot/apps/hydration-agent/server/prompt.mjs b/live-api/spot/apps/hydration-agent/server/prompt.mjs new file mode 100644 index 0000000..7579a30 --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/prompt.mjs @@ -0,0 +1,58 @@ +export function buildSystemPrompt({ drinks, waypoints, toolDescriptions }) { + const drinkLines = drinks.map((drink) => + `- ${drink.id}: ${drink.name} (${drink.flavor}); detection instruction: "${drink.detectInstruction}"`, + ).join("\n"); + const waypointLines = waypoints.length + ? waypoints.map((name) => `- ${name}`).join("\n") + : "- No delivery waypoint is currently available. Use list_waypoints after the robot map is loaded."; + + return `You are the Gemini Robotics Hydration Service agent controlling a Boston Dynamics Spot robot through validated function tools. + +Your job is to accept drink orders, execute them safely, answer operator questions, and follow manual debugging instructions. Robot motion only happens through function calls. Never claim an action succeeded until its function result confirms success. + +LIVE VISION +While Spot is connected, you continuously receive the gripper color camera as video at 1 frame per second. When the operator asks what you see, look at the latest video frame and describe the visible scene directly. Never say that you cannot passively stream video. Do not limit visual descriptions to the drink catalog. Use detect_object only when the operator requests precise object localization, a 3D pose, or a grasp target. + +SPATIALLY GUIDED OBJECT PICKUP +Treat relative location phrases such as left, right, on the floor, above, or behind the current view as camera-search instructions. When the operator asks to pick an object and provides a relative location, you MUST NOT reject the request merely because the object is absent from the initial frame. Check robot status and control, stand the robot, deploy the arm, then call aim_gripper_camera toward the supplied location. Map "right side of the floor" to horizontal=right and vertical=down. After the camera settles, call pick_object with the operator's object description. The pick tool performs a fresh detection from the new view and commands Spot's Manipulation API. If the pick succeeds, move the arm to carry pose. Make at most one additional small aim adjustment if the new frame clearly requires it; do not perform an unbounded search or move the base unless the operator asks. + +PROACTIVE VISUAL WATCHES +When the operator asks for a future visual notification, such as "say hi when the man in red leaves", you MUST call create_visual_watch. A verbal acknowledgment without that function call does not create monitoring. Set requires_prior_presence=true for leaves/disappears conditions and false for arrives/appears conditions. The server will send private visual-watch ticks containing active watches. During each tick, inspect only the latest frame and call evaluate_visual_watch exactly once for every listed watch, then respond with exactly DONE to close the private turn. Do not call robot-motion tools during a visual-watch tick. The server confirms a trigger across consecutive frames and emits the requested response message. + +AVAILABLE DRINKS +${drinkLines} + +CURRENT DELIVERY POSITIONS +The drink station is snack1 and the idle position is home. Orders may only be delivered to these named waypoints: +${waypointLines} + +TOOLS AND UNDERLYING FASTAPI +${toolDescriptions} + +NORMAL ORDER WORKFLOW +1. Call get_robot_status. If disconnected, call connect_robot. Take control if the lease is not held. +2. Mark the order running. +3. Navigate to snack1, then move backward 0.30 m. +4. Put the arm in carry pose and detect the ordered drink using its exact detection instruction. +5. Open the gripper to 60%, rotate the gripper camera 90 degrees clockwise, and approach the returned detection. +6. Close the gripper slowly and stow the arm before navigating. +7. Navigate to the delivery waypoint, put the arm in carry pose, and wait for the delivery handoff. +8. Mark the order finished only after the handoff tool reports triggered=true. +9. Serve another pending order when one exists. Otherwise navigate home and sit to save power. + +OPERATING RULES +- Execute physical action tools one at a time and wait for each result before choosing the next action. +- Stop the workflow immediately after any tool error. Do not call the next action. Briefly report the failure; the service will pause. +- Never invent waypoint names, drink IDs, detection IDs, poses, API results, or order completion. +- Base general visual answers on the latest gripper-camera video frame. If no recent frame is available, say that the camera frame is unavailable or stale. +- For a spatially guided pick request, use aim_gripper_camera before saying the object is not visible. A statement about the initial frame is not completion of the request. +- Use pick_object for arbitrary operator-requested pickups after aiming. Use the drink-specific detect, approach, and gripper workflow for hydration orders. +- Use detection IDs with approach_detection. Do not transcribe or modify pose coordinates yourself. +- Use open_gripper with fraction 0.6 before grasping and close_gripper with slow=true for the grasp. +- The arm pose tool is named set_arm_pose. Never invent or call arm_set_pose. +- Do not navigate while the arm is deployed. Stow it first unless the operator explicitly requests a stationary arm test. +- Treat stop_actions as immediate operator intent. +- For a natural-language order, call create_order exactly once. The scheduler will dispatch it after the current conversational turn. +- For a future visual condition, call create_visual_watch exactly once. Do not merely promise to keep watching. +- Keep chat replies short and operational. Tool calls and order tracking are already visible in the UI.`; +} diff --git a/live-api/spot/apps/hydration-agent/server/state.mjs b/live-api/spot/apps/hydration-agent/server/state.mjs new file mode 100644 index 0000000..bc93884 --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/state.mjs @@ -0,0 +1,203 @@ +import { EventEmitter } from "node:events"; + +export class HydrationState extends EventEmitter { + constructor(model) { + super(); + this.model = model; + this.serviceRunning = false; + this.paused = false; + this.agentConnected = false; + this.agentBusy = false; + this.robotConnected = false; + this.cameraStreaming = false; + this.cameraFrameAt = null; + this.cameraFramesSent = 0; + this.cameraError = null; + this.activeOrderId = null; + this.lastError = null; + this.nextOrderId = 1; + this.nextVisualWatchId = 1; + this.nextEntryId = 1; + this.orders = []; + this.messages = []; + this.toolCalls = []; + this.events = []; + this.waypoints = []; + this.visualWatches = []; + } + + snapshot() { + return { + model: this.model, + serviceRunning: this.serviceRunning, + paused: this.paused, + agentConnected: this.agentConnected, + agentBusy: this.agentBusy, + robotConnected: this.robotConnected, + cameraStreaming: this.cameraStreaming, + cameraFrameAt: this.cameraFrameAt, + cameraFramesSent: this.cameraFramesSent, + cameraError: this.cameraError, + activeOrderId: this.activeOrderId, + lastError: this.lastError, + orders: this.orders, + messages: this.messages.slice(-120), + toolCalls: this.toolCalls.slice(-80), + events: this.events.slice(-80), + waypoints: this.waypoints, + visualWatches: this.visualWatches.slice(-30), + }; + } + + changed() { + this.emit("changed", this.snapshot()); + } + + setWaypoints(waypoints) { + this.waypoints = [...new Set(waypoints)].sort((a, b) => a.localeCompare(b)); + this.changed(); + } + + createOrder(drink, destination, source = "ui") { + const now = new Date().toISOString(); + const order = { + id: this.nextOrderId++, + drinkId: drink.id, + drinkName: drink.name, + detectInstruction: drink.detectInstruction, + destination, + source, + status: "pending", + note: null, + createdAt: now, + updatedAt: now, + }; + this.orders.push(order); + this.log("order", `Order #${order.id} queued: ${order.drinkName} to ${destination}`); + return order; + } + + updateOrder(id, status, note = null) { + const order = this.orders.find((candidate) => candidate.id === Number(id)); + if (!order) throw new Error(`Unknown order #${id}.`); + order.status = status; + order.note = note || order.note; + order.updatedAt = new Date().toISOString(); + if (status === "running" && !order.startedAt) order.startedAt = order.updatedAt; + if (["finished", "failed", "cancelled"].includes(status)) order.finishedAt = order.updatedAt; + this.log("order", `Order #${order.id} marked ${status}${note ? `: ${note}` : ""}`); + return order; + } + + createVisualWatch({ subject, condition, responseMessage, requiresPriorPresence }) { + const now = new Date().toISOString(); + const watch = { + id: this.nextVisualWatchId++, + subject, + condition, + responseMessage, + requiresPriorPresence, + armed: !requiresPriorPresence, + consecutiveMatches: 0, + status: "active", + observation: null, + createdAt: now, + updatedAt: now, + }; + this.visualWatches.push(watch); + this.log("watch", `Visual watch #${watch.id} active: ${watch.condition}`); + return watch; + } + + evaluateVisualWatch(id, { subjectVisible, conditionMet, observation }) { + const watch = this.visualWatches.find((candidate) => candidate.id === Number(id)); + if (!watch) throw new Error(`Unknown visual watch #${id}.`); + if (watch.status !== "active") throw new Error(`Visual watch #${id} is ${watch.status}.`); + + if (subjectVisible) watch.armed = true; + const acceptedMatch = watch.armed && conditionMet; + watch.consecutiveMatches = acceptedMatch ? watch.consecutiveMatches + 1 : 0; + watch.observation = observation || null; + watch.updatedAt = new Date().toISOString(); + + if (watch.consecutiveMatches >= 2) { + watch.status = "triggered"; + watch.triggeredAt = watch.updatedAt; + this.addMessage("assistant", watch.responseMessage, { proactive: true, visualWatchId: watch.id }); + this.log("watch", `Visual watch #${watch.id} triggered: ${watch.observation || watch.condition}`); + } else { + this.changed(); + } + return watch; + } + + cancelVisualWatch(id) { + const watch = this.visualWatches.find((candidate) => candidate.id === Number(id)); + if (!watch) throw new Error(`Unknown visual watch #${id}.`); + if (watch.status === "active") { + watch.status = "cancelled"; + watch.updatedAt = new Date().toISOString(); + this.log("watch", `Visual watch #${watch.id} cancelled`); + } + return watch; + } + + addMessage(role, text, meta = {}) { + const message = { + id: this.nextEntryId++, + role, + text, + at: new Date().toISOString(), + ...meta, + }; + this.messages.push(message); + this.changed(); + return message; + } + + appendMessage(id, text) { + const message = this.messages.find((candidate) => candidate.id === id); + if (!message) return; + message.text += text; + this.changed(); + } + + addToolCall(name, args) { + const call = { + id: this.nextEntryId++, + name, + args, + status: "running", + startedAt: new Date().toISOString(), + }; + this.toolCalls.push(call); + this.changed(); + return call; + } + + finishToolCall(call, result, error = null) { + call.status = error ? "failed" : "finished"; + call.finishedAt = new Date().toISOString(); + if (error) call.error = error; + else call.result = summarize(result); + this.changed(); + } + + log(type, message) { + this.events.push({ id: this.nextEntryId++, type, message, at: new Date().toISOString() }); + this.changed(); + } +} + +function summarize(result) { + if (!result || typeof result !== "object") return result; + const summary = {}; + for (const key of [ + "ok", "status", "reached_goal", "arrived", "at_goal", "triggered", "reason", + "order_id", "detection_id", "pending_orders", "percentage", "connected", "holding_lease", + ]) { + if (key in result) summary[key] = result[key]; + } + if (result.pose) summary.pose = result.pose; + return Object.keys(summary).length ? summary : { ok: true }; +} diff --git a/live-api/spot/apps/hydration-agent/server/tools.mjs b/live-api/spot/apps/hydration-agent/server/tools.mjs new file mode 100644 index 0000000..1190ca4 --- /dev/null +++ b/live-api/spot/apps/hydration-agent/server/tools.mjs @@ -0,0 +1,583 @@ +import { Type } from "@google/genai"; +import { DRINKS, findDrink, isDeliveryWaypoint } from "./catalog.mjs"; + +const objectSchema = (properties = {}, required = []) => ({ + type: Type.OBJECT, + properties, + ...(required.length ? { required } : {}), +}); + +export const TOOL_DECLARATIONS = [ + { + name: "get_robot_status", + description: "Read FastAPI /health, /battery, and current service/order state before acting.", + }, + { + name: "connect_robot", + description: "Connect FastAPI to Spot using server-side .config credentials and take the lease. Never asks for or returns credentials.", + }, + { + name: "take_lease", + description: "Take control using FastAPI POST /lease/take.", + }, + { + name: "list_waypoints", + description: "List current named waypoints using FastAPI GET /waypoints.", + }, + { + name: "create_order", + description: "Queue a hydration order from natural-language chat. Do not serve it in this same conversational turn; the scheduler dispatches it next.", + parameters: objectSchema({ + drink_id: { type: Type.STRING, enum: DRINKS.map((drink) => drink.id) }, + destination: { type: Type.STRING, description: "Exact named delivery waypoint." }, + }, ["drink_id", "destination"]), + }, + { + name: "create_visual_watch", + description: "Register a persistent gripper-camera watch for a future visual event. Required when the operator says when, until, leaves, arrives, appears, or disappears; acknowledging verbally is not sufficient.", + parameters: objectSchema({ + subject: { type: Type.STRING, description: "The visual subject to track, such as 'the man in the red shirt'." }, + condition: { type: Type.STRING, description: "The exact future visual condition that triggers the response." }, + response_message: { type: Type.STRING, description: "Exact short message to emit when the condition is confirmed." }, + requires_prior_presence: { type: Type.BOOLEAN, description: "True for leaves/disappears conditions so the subject must first be seen; false for appears/arrives conditions." }, + }, ["subject", "condition", "response_message", "requires_prior_presence"]), + }, + { + name: "cancel_visual_watch", + description: "Cancel an active visual watch requested by the operator.", + parameters: objectSchema({ + watch_id: { type: Type.INTEGER }, + }, ["watch_id"]), + }, + { + name: "evaluate_visual_watch", + description: "Internal visual-tick tool. Evaluate one active watch against the latest frame. This tool is only valid during a server visual-watch tick.", + parameters: objectSchema({ + watch_id: { type: Type.INTEGER }, + subject_visible: { type: Type.BOOLEAN }, + condition_met: { type: Type.BOOLEAN }, + observation: { type: Type.STRING, description: "Brief factual observation from the latest frame." }, + }, ["watch_id", "subject_visible", "condition_met", "observation"]), + }, + { + name: "set_order_status", + description: "Update an order's tracked state. Use finished only after a confirmed handoff, and failed after an unrecoverable problem.", + parameters: objectSchema({ + order_id: { type: Type.INTEGER }, + status: { type: Type.STRING, enum: ["running", "finished", "failed", "cancelled"] }, + note: { type: Type.STRING }, + }, ["order_id", "status"]), + }, + { + name: "navigate_to", + description: "Navigate Spot to an exact named waypoint using FastAPI POST /navigate. Powers on, stands, and takes lease.", + parameters: objectSchema({ + waypoint: { type: Type.STRING }, + }, ["waypoint"]), + }, + { + name: "move_base", + description: "Move Spot's base up to 0.5 m using FastAPI POST /teleop/velocity. Intended for short station alignment, not waypoint navigation.", + parameters: objectSchema({ + direction: { type: Type.STRING, enum: ["forward", "backward", "left", "right"] }, + distance_m: { type: Type.NUMBER, description: "Distance from 0.01 to 0.5 meters." }, + }, ["direction", "distance_m"]), + }, + { + name: "set_arm_pose", + description: "Move the arm to carry, deploy/ready, stow, or freeze using the matching FastAPI /arm endpoint.", + parameters: objectSchema({ + pose: { type: Type.STRING, enum: ["carry", "deploy", "stow", "freeze"] }, + }, ["pose"]), + }, + { + name: "aim_gripper_camera", + description: "Aim the gripper camera toward a user-provided relative location through FastAPI POST /arm/jog. Use this before declaring a spatially described object absent.", + parameters: objectSchema({ + horizontal: { type: Type.STRING, enum: ["left", "center", "right"] }, + vertical: { type: Type.STRING, enum: ["up", "level", "down"] }, + degrees: { type: Type.NUMBER, description: "Small camera rotation from 3 to 25 degrees; normally 12." }, + }, ["horizontal", "vertical"]), + }, + { + name: "detect_object", + description: "Detect an object in the gripper RGB/depth images through FastAPI POST /detect. Returns a server detection_id and 3D pose.", + parameters: objectSchema({ + instruction: { type: Type.STRING, description: "Exact language instruction describing the target point." }, + }, ["instruction"]), + }, + { + name: "pick_object", + description: "Detect and pick an object currently in the gripper camera view through FastAPI POST /pick and Spot's Manipulation API. Aim the camera first when the operator gives a relative location.", + parameters: objectSchema({ + instruction: { type: Type.STRING, description: "Visual description of the object to pick, including useful location or appearance details." }, + }, ["instruction"]), + }, + { + name: "open_gripper", + description: "Open the gripper through FastAPI POST /gripper/open. Use fraction 0.6 before grasping a drink.", + parameters: objectSchema({ + fraction: { type: Type.NUMBER, description: "Open fraction from 0.0 to 1.0." }, + }, ["fraction"]), + }, + { + name: "close_gripper", + description: "Close the gripper through FastAPI POST /gripper/close. Set slow=true when grasping a drink.", + parameters: objectSchema({ + slow: { type: Type.BOOLEAN }, + }, ["slow"]), + }, + { + name: "rotate_gripper_camera", + description: "Roll the gripper camera through FastAPI POST /arm/camera-roll.", + parameters: objectSchema({ + direction: { type: Type.STRING, enum: ["clockwise", "counterclockwise"] }, + degrees: { type: Type.NUMBER, description: "Rotation angle from 1 to 180 degrees." }, + }, ["direction", "degrees"]), + }, + { + name: "approach_detection", + description: "Approach a stored 3D detection with coordinated arm and base motion through FastAPI POST /arm/approach-whole-body.", + parameters: objectSchema({ + detection_id: { type: Type.STRING }, + standoff_m: { type: Type.NUMBER, description: "Offset from target in meters; normally 0 for drink pickup." }, + }, ["detection_id"]), + }, + { + name: "wait_for_delivery", + description: "Monitor upward gripper motion, open for handoff, close, and stow using FastAPI POST /delivery/wait.", + parameters: objectSchema({ + monitor_seconds: { type: Type.NUMBER, description: "Monitoring duration, normally 30 seconds." }, + }), + }, + { + name: "stand_robot", + description: "Power on and stand using FastAPI POST /stand.", + }, + { + name: "sit_robot", + description: "Sit using FastAPI POST /sit.", + }, + { + name: "localize_robot", + description: "Initialize GraphNav localization at a named waypoint using FastAPI POST /localize.", + parameters: objectSchema({ + waypoint: { type: Type.STRING }, + }, ["waypoint"]), + }, + { + name: "clear_behavior_faults", + description: "Clear Spot behavior faults using FastAPI POST /faults/behavior/clear.", + }, + { + name: "stop_actions", + description: "Immediately cancel motion and freeze the arm using FastAPI POST /actions/stop.", + }, +]; + +const READ_ONLY_TOOLS = new Set([ + "get_robot_status", "list_waypoints", "create_order", "set_order_status", "create_visual_watch", + "cancel_visual_watch", "evaluate_visual_watch", "stop_actions", +]); + +export class HydrationTools { + constructor({ fastApi, state, config }) { + this.fastApi = fastApi; + this.state = state; + this.config = config; + this.detections = new Map(); + this.nextDetectionId = 1; + } + + descriptions() { + return TOOL_DECLARATIONS.map((tool) => `- ${tool.name}: ${tool.description}`).join("\n"); + } + + async refreshWaypoints() { + const items = await this.fastApi.get("/waypoints", { timeoutMs: 10_000 }); + const names = items.map((item) => item.name).filter(Boolean); + this.state.setWaypoints(names.filter((name) => name !== "home" && name !== "snack1")); + return names; + } + + async execute(name, args = {}) { + const resolvedName = TOOL_ALIASES[name] || name; + if (this.state.paused && !READ_ONLY_TOOLS.has(resolvedName)) { + throw new Error(`Service is paused; ${resolvedName} was not executed.`); + } + + switch (resolvedName) { + case "get_robot_status": + return this.getRobotStatus(); + case "connect_robot": + return this.connectRobot(); + case "take_lease": + return this.fastApi.post("/lease/take", {}, { timeoutMs: 20_000 }); + case "list_waypoints": + return { waypoints: await this.refreshWaypoints(), delivery_waypoints: this.state.waypoints }; + case "create_order": + return this.createOrder(args); + case "create_visual_watch": + return this.createVisualWatch(args); + case "cancel_visual_watch": + return this.cancelVisualWatch(args); + case "evaluate_visual_watch": + return this.evaluateVisualWatch(args); + case "set_order_status": + return this.setOrderStatus(args); + case "navigate_to": + return this.navigate(args); + case "move_base": + return this.moveBase(args); + case "set_arm_pose": + return this.setArmPose(args); + case "aim_gripper_camera": + return this.aimGripperCamera(args); + case "detect_object": + return this.detectObject(args); + case "pick_object": + return this.pickObject(args); + case "open_gripper": + return this.openGripper(args); + case "close_gripper": + return this.closeGripper(args); + case "rotate_gripper_camera": + return this.rotateCamera(args); + case "approach_detection": + return this.approachDetection(args); + case "wait_for_delivery": + return this.waitForDelivery(args); + case "stand_robot": + return this.fastApi.post("/stand", { power_on: true, take_lease: true, timeout: 15 }, { timeoutMs: 20_000 }); + case "sit_robot": + return this.fastApi.post("/sit", { take_lease: true, timeout: 15 }, { timeoutMs: 20_000 }); + case "localize_robot": + return this.fastApi.post("/localize", { waypoint_name: requiredString(args.waypoint, "waypoint") }, { timeoutMs: 40_000 }); + case "clear_behavior_faults": + return this.fastApi.post("/faults/behavior/clear", {}, { timeoutMs: 20_000 }); + case "stop_actions": + return this.fastApi.stopRobot(); + default: + throw new Error(`Unknown tool: ${resolvedName}`); + } + } + + async getRobotStatus() { + const health = await this.fastApi.get("/health", { timeoutMs: 10_000 }); + this.state.robotConnected = health.connected === true; + let battery = null; + if (health.connected) { + try { + battery = await this.fastApi.get("/battery", { timeoutMs: 10_000 }); + } catch (error) { + battery = { error: errorMessage(error) }; + } + } + return { ...health, battery, service: this.serviceSummary() }; + } + + async connectRobot() { + const password = this.config.BOSDYN_CLIENT_PASSWORD; + if (!password) throw new Error("BOSDYN_CLIENT_PASSWORD is missing from .config."); + const result = await this.fastApi.post("/connect", { + hostname: this.config.SPOT_HOSTNAME || this.config.BOSDYN_CLIENT_HOSTNAME || "192.168.80.3", + username: this.config.BOSDYN_CLIENT_USERNAME || "user", + password, + take_lease: true, + }, { timeoutMs: 60_000 }); + this.state.robotConnected = true; + this.state.changed(); + return result; + } + + createOrder(args) { + const drink = findDrink(requiredString(args.drink_id, "drink_id")); + if (!drink) throw new Error(`Unknown drink ID: ${args.drink_id}`); + const destination = requiredString(args.destination, "destination"); + if (!isDeliveryWaypoint(destination, this.state.waypoints)) { + throw new Error(`Invalid delivery waypoint: ${destination}`); + } + const order = this.state.createOrder(drink, destination, "chat"); + return { order_id: order.id, status: order.status, queued: true }; + } + + createVisualWatch(args) { + const watch = this.state.createVisualWatch({ + subject: requiredString(args.subject, "subject"), + condition: requiredString(args.condition, "condition"), + responseMessage: requiredString(args.response_message, "response_message"), + requiresPriorPresence: args.requires_prior_presence === true, + }); + return { + watch_id: watch.id, + status: watch.status, + armed: watch.armed, + condition: watch.condition, + }; + } + + cancelVisualWatch(args) { + const watch = this.state.cancelVisualWatch(Number(args.watch_id)); + return { watch_id: watch.id, status: watch.status }; + } + + evaluateVisualWatch(args) { + const watch = this.state.evaluateVisualWatch(Number(args.watch_id), { + subjectVisible: args.subject_visible === true, + conditionMet: args.condition_met === true, + observation: requiredString(args.observation, "observation"), + }); + return { + watch_id: watch.id, + status: watch.status, + armed: watch.armed, + consecutive_matches: watch.consecutiveMatches, + triggered: watch.status === "triggered", + }; + } + + setOrderStatus(args) { + const status = requiredString(args.status, "status"); + if (!["running", "finished", "failed", "cancelled"].includes(status)) { + throw new Error(`Invalid order status: ${status}`); + } + const order = this.state.updateOrder(Number(args.order_id), status, optionalString(args.note)); + const pending = this.state.orders.filter((candidate) => candidate.status === "pending"); + return { + order_id: order.id, + status: order.status, + pending_orders: pending.length, + next_action: pending.length ? "Serve the next pending order." : "Navigate home and sit.", + }; + } + + navigate(args) { + const waypoint = requiredString(args.waypoint, "waypoint"); + const known = ["home", "snack1", ...this.state.waypoints]; + if (!known.includes(waypoint)) throw new Error(`Unknown waypoint: ${waypoint}`); + return this.fastApi.post("/navigate", { + name: waypoint, + take_lease: true, + power_on: true, + stand: true, + timeout: 180, + }, { timeoutMs: 190_000 }); + } + + moveBase(args) { + const direction = requiredString(args.direction, "direction"); + const distance = boundedNumber(args.distance_m, "distance_m", 0.01, 0.5); + const speed = 0.25; + const signs = { + forward: { v_x: speed, v_y: 0 }, + backward: { v_x: -speed, v_y: 0 }, + left: { v_x: 0, v_y: speed }, + right: { v_x: 0, v_y: -speed }, + }; + if (!signs[direction]) throw new Error(`Invalid direction: ${direction}`); + return this.fastApi.post("/teleop/velocity", { + ...signs[direction], + v_rot: 0, + duration: distance / speed, + take_lease: true, + power_on: false, + stand: false, + }, { timeoutMs: 10_000 }); + } + + setArmPose(args) { + const pose = requiredString(args.pose, "pose"); + const endpoints = { + carry: "/arm/carry", + deploy: "/arm/deploy", + stow: "/arm/stow", + freeze: "/arm/freeze", + }; + const endpoint = endpoints[pose]; + if (!endpoint) throw new Error(`Invalid arm pose: ${pose}`); + return this.fastApi.post(endpoint, { + take_lease: true, + timeout: pose === "stow" ? 20 : 10, + ...(pose === "deploy" ? { power_on: true } : {}), + }, { timeoutMs: 30_000 }); + } + + async aimGripperCamera(args) { + const horizontal = requiredEnum(args.horizontal, "horizontal", ["left", "center", "right"]); + const vertical = requiredEnum(args.vertical, "vertical", ["up", "level", "down"]); + const degrees = args.degrees === undefined + ? 12 + : boundedNumber(args.degrees, "degrees", 3, 25); + const angle = degrees * Math.PI / 180; + const dyaw = horizontal === "left" ? angle : horizontal === "right" ? -angle : 0; + const dpitch = vertical === "up" ? -angle : vertical === "down" ? angle : 0; + if (dyaw === 0 && dpitch === 0) { + return { aimed: false, horizontal, vertical, degrees, reason: "Camera direction was unchanged." }; + } + + const result = await this.fastApi.post("/arm/jog", { + dyaw, + dpitch, + seconds: 1.0, + take_lease: true, + timeout: 4, + }, { timeoutMs: 10_000 }); + const settleMs = Math.max(0, Number(this.config.HYDRATION_AGENT_CAMERA_SETTLE_MS ?? 1_200)); + if (settleMs) await delay(settleMs); + return { + ...result, + aimed: true, + horizontal, + vertical, + degrees, + camera_settled: true, + }; + } + + async detectObject(args) { + const instruction = requiredString(args.instruction, "instruction"); + const scene = await this.fastApi.post("/detect", { + instruction, + api_key: this.config.GEMINI_API_KEY || null, + include_point_cloud: false, + }, { timeoutMs: 60_000 }); + if (!scene.pose) throw new Error(`Detection returned no 3D pose: ${JSON.stringify(scene.errors || {})}`); + const detectionId = `detection-${this.nextDetectionId++}`; + this.detections.set(detectionId, scene.pose); + return { + detection_id: detectionId, + pose: scene.pose, + detection: scene.detection || scene.point_2d || null, + image: scene.image ? { width: scene.image.width, height: scene.image.height } : null, + model_output: scene.model_output || scene.raw_model_output || null, + }; + } + + async pickObject(args) { + const instruction = requiredString(args.instruction, "instruction"); + const result = await this.fastApi.post("/pick", { + instruction, + model: "gemini-robotics-er-1.6-preview", + api_key: this.config.GEMINI_API_KEY || null, + take_lease: true, + timeout: 60, + }, { timeoutMs: 75_000 }); + const state = String(result.state || ""); + if (!/GRASP_SUCCEEDED|MANIP_STATE_DONE/.test(state)) { + throw new Error(`/pick did not confirm a grasp: ${state || "missing manipulation state"}`); + } + return result; + } + + openGripper(args) { + const fraction = boundedNumber(args.fraction, "fraction", 0, 1); + return this.fastApi.post("/gripper/open", { + open_fraction: fraction, + max_vel: 0.5, + max_acc: 1.0, + take_lease: true, + timeout: 5, + }, { timeoutMs: 10_000 }); + } + + closeGripper(args) { + const slow = args.slow !== false; + return this.fastApi.post("/gripper/close", { + max_vel: slow ? 0.25 : 0.5, + max_acc: slow ? 0.5 : 1.0, + take_lease: true, + timeout: 10, + }, { timeoutMs: 15_000 }); + } + + rotateCamera(args) { + const direction = requiredString(args.direction, "direction"); + if (!["clockwise", "counterclockwise"].includes(direction)) throw new Error(`Invalid rotation: ${direction}`); + const degrees = boundedNumber(args.degrees, "degrees", 1, 180); + return this.fastApi.post("/arm/camera-roll", { + direction, + angle_rad: degrees * Math.PI / 180, + seconds: Math.max(0.7, degrees / 90 * 0.7), + take_lease: true, + timeout: 5, + }, { timeoutMs: 10_000 }); + } + + approachDetection(args) { + const id = requiredString(args.detection_id, "detection_id"); + const pose = this.detections.get(id); + if (!pose) throw new Error(`Unknown or expired detection ID: ${id}`); + const standoff = args.standoff_m === undefined ? 0 : boundedNumber(args.standoff_m, "standoff_m", -0.1, 0.1); + return this.fastApi.post("/arm/approach-whole-body", { + pose, + standoff_m: standoff, + max_step_m: 0.8, + seconds: 2, + take_lease: true, + timeout: 10, + }, { timeoutMs: 20_000 }); + } + + async waitForDelivery(args) { + const monitorSeconds = args.monitor_seconds === undefined + ? 30 + : boundedNumber(args.monitor_seconds, "monitor_seconds", 5, 120); + const result = await this.fastApi.post("/delivery/wait", { + monitor_sec: monitorSeconds, + upward_threshold_m: 0.02, + sample_interval: 0.1, + open_duration_sec: 3, + take_lease: true, + gripper_timeout: 5, + stow_timeout: 10, + }, { timeoutMs: (monitorSeconds + 25) * 1000 }); + if (result.triggered !== true) throw new Error(result.reason || "Delivery handoff was not detected."); + return result; + } + + serviceSummary() { + return { + running: this.state.serviceRunning, + paused: this.state.paused, + active_order_id: this.state.activeOrderId, + pending_orders: this.state.orders.filter((order) => order.status === "pending").length, + }; + } +} + +function requiredString(value, name) { + const result = String(value ?? "").trim(); + if (!result) throw new Error(`${name} is required.`); + return result; +} + +function optionalString(value) { + const result = String(value ?? "").trim(); + return result || null; +} + +function boundedNumber(value, name, minimum, maximum) { + const number = Number(value); + if (!Number.isFinite(number) || number < minimum || number > maximum) { + throw new Error(`${name} must be between ${minimum} and ${maximum}.`); + } + return number; +} + +function requiredEnum(value, name, values) { + const result = requiredString(value, name); + if (!values.includes(result)) throw new Error(`${name} must be one of: ${values.join(", ")}.`); + return result; +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} + +const TOOL_ALIASES = Object.freeze({ + arm_set_pose: "set_arm_pose", +}); diff --git a/live-api/spot/apps/hydration-agent/src/main.jsx b/live-api/spot/apps/hydration-agent/src/main.jsx new file mode 100644 index 0000000..b3e76e9 --- /dev/null +++ b/live-api/spot/apps/hydration-agent/src/main.jsx @@ -0,0 +1,383 @@ +import React, { useEffect, useMemo, useRef, useState } from "react"; +import { createRoot } from "react-dom/client"; +import { + AlertTriangle, + Bot, + CirclePause, + CirclePlay, + Coffee, + Link, + MessageSquareText, + Power, + RefreshCw, + RotateCcw, + Send, + Square, + Wrench, +} from "lucide-react"; +import "./styles.css"; + +const EMPTY_STATE = { + orders: [], + messages: [], + toolCalls: [], + events: [], + waypoints: [], + serviceRunning: false, + paused: false, + agentConnected: false, + agentBusy: false, + robotConnected: false, + cameraStreaming: false, + cameraFrameAt: null, + cameraFramesSent: 0, + cameraError: null, + visualWatches: [], +}; + +function App() { + const [bootstrap, setBootstrap] = useState({ drinks: [], waypoints: [] }); + const [state, setState] = useState(EMPTY_STATE); + const [drinkId, setDrinkId] = useState(""); + const [destination, setDestination] = useState(""); + const [message, setMessage] = useState(""); + const [busy, setBusy] = useState(""); + const [error, setError] = useState(""); + const [cameraTick, setCameraTick] = useState(Date.now()); + const chatEnd = useRef(null); + + useEffect(() => { + apiGet("/api/bootstrap") + .then((data) => { + setBootstrap(data); + setDrinkId(data.drinks[0]?.id || ""); + setDestination(data.waypoints[0] || ""); + }) + .catch((nextError) => setError(nextError.message)); + + const events = new EventSource("/api/events"); + events.onmessage = (event) => setState(JSON.parse(event.data)); + events.onerror = () => setError("Lost the hydration agent event stream."); + return () => events.close(); + }, []); + + useEffect(() => { + chatEnd.current?.scrollIntoView({ behavior: "smooth", block: "end" }); + }, [state.messages]); + + useEffect(() => { + if (!state.robotConnected) return undefined; + const timer = setInterval(() => setCameraTick(Date.now()), 1000); + return () => clearInterval(timer); + }, [state.robotConnected]); + + const counts = useMemo(() => ({ + pending: state.orders.filter((order) => order.status === "pending").length, + running: state.orders.filter((order) => order.status === "running").length, + finished: state.orders.filter((order) => order.status === "finished").length, + failed: state.orders.filter((order) => order.status === "failed").length, + }), [state.orders]); + + async function runAction(name, path, body = {}) { + setBusy(name); + setError(""); + try { + const result = await apiPost(path, body); + if (result?.state) setState(result.state); + else if (result?.orders) setState(result); + return result; + } catch (nextError) { + setError(nextError.message); + return null; + } finally { + setBusy(""); + } + } + + async function submitOrder() { + await runAction("order", "/api/orders", { drinkId, destination }); + } + + async function submitChat(event) { + event.preventDefault(); + const text = message.trim(); + if (!text || busy === "chat") return; + setMessage(""); + await runAction("chat", "/api/chat", { message: text }); + } + + const serviceLabel = state.paused + ? "Paused" + : state.agentBusy + ? "Agent working" + : state.serviceRunning + ? "Ready" + : "Stopped"; + + return ( +
+
+
+ +
+

Gemini Robotics Hydration Agent

+ {state.model || bootstrap.model} +
+
+
+ + + + {serviceLabel} +
+
+ runAction("connect", "/api/service/connect-robot")} + /> + runAction("start", state.paused ? "/api/service/resume" : "/api/service/start")} + /> + runAction("pause", "/api/service/pause")} + /> + runAction("stop", "/api/service/stop-actions")} + /> + runAction("reset", "/api/service/reset-session")} + /> +
+
+ +
+ + +
+
+ + {state.agentBusy ? Working : null} +
+
+ {state.messages.length ? state.messages.map((item) => ( + + )) : ( +
+ + Agent ready +
+ )} +
+
+ {error || state.lastError ? ( +
{error || state.lastError}
+ ) : null} +
+