diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index b52442db3..7dbef8145 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -46,6 +46,8 @@ mod meet_call; mod meet_scanner; mod meet_video; mod native_notifications; +#[cfg(target_os = "macos")] +mod notch_window; mod notification_settings; mod process_kill; mod process_recovery; @@ -927,6 +929,33 @@ fn mascot_native_window_is_open() -> bool { false } +/// Show the notch activity indicator. macOS only — transparent NSPanel + WKWebView +/// anchored to the top-centre of the primary screen. Displays live voice and +/// agent status (listening, thinking, executing) in a pill that emerges from +/// the physical notch on supported MacBook Pros. +#[tauri::command] +fn notch_window_show(app: AppHandle) -> Result<(), String> { + log::info!("[notch-window] show requested"); + #[cfg(target_os = "macos")] + { + return notch_window::show(&app); + } + #[cfg(not(target_os = "macos"))] + { + let _ = app; + Ok(()) // No-op on non-macOS + } +} + +/// Hide the notch activity indicator. +#[tauri::command] +fn notch_window_hide(_app: AppHandle) -> Result<(), String> { + log::info!("[notch-window] hide requested"); + #[cfg(target_os = "macos")] + notch_window::hide(); + Ok(()) +} + /// Hide or show the OS top-level main-window frame on Windows by enumerating /// this process's top-level windows and matching the visible /// `Chrome_WidgetWin_1` host. `WebviewWindow::hwnd()` from the vendored CEF @@ -2800,6 +2829,14 @@ pub fn run() { // let _ = window.show(); // } + // Notch activity indicator: transparent pill at the top-centre of + // the primary screen. Shows live voice / agent state. macOS only + // (physical notch or menu-bar HUD on older hardware). + #[cfg(target_os = "macos")] + if let Err(err) = notch_window::show(&app.handle()) { + log::warn!("[notch-window] auto-show on startup failed: {err}"); + } + // Synthetic-input main-thread executor. enigo's macOS keyboard-layout // lookup (TSMGetInputSourceProperty) MUST run on the app main thread // or it traps (`_dispatch_assert_queue_fail`/EXC_BREAKPOINT) and @@ -3195,6 +3232,8 @@ pub fn run() { native_notifications::show_native_notification, mascot_window_show, mascot_window_hide, + notch_window_show, + notch_window_hide, file_logging::reveal_logs_folder, file_logging::logs_folder_path, workspace_paths::open_workspace_path, diff --git a/app/src-tauri/src/notch_window.rs b/app/src-tauri/src/notch_window.rs new file mode 100644 index 000000000..78653688c --- /dev/null +++ b/app/src-tauri/src/notch_window.rs @@ -0,0 +1,392 @@ +//! Native macOS NSPanel + WKWebView host for the notch activity indicator. +//! +//! A transparent, click-through floating panel anchored to the top-centre of +//! the primary screen. On MacBook Pros with a physical notch the pill visually +//! emerges from the notch; on older Macs it acts as a top-centre floating HUD. +//! +//! Architecture mirrors `mascot_native_window` — a native NSPanel avoids the +//! CEF transparency limitation (vendored tauri-cef cannot render transparent +//! windowed-mode browsers; only off-screen rendering supports transparency, +//! which the runtime does not enable). The WKWebView loads the same Vite entry +//! point at `?window=notch` so the React tree can branch in `main.tsx`. +//! +//! IPC strategy: no Tauri IPC bridge. The panel polls +//! `OPENHUMAN_CORE_RPC_URL` (set by `CoreProcessHandle` once the embedded +//! server is ready) and injects it via `evaluateJavaScript` so the React app +//! can open a Socket.IO connection to receive live voice and agent events. + +use std::cell::{Cell, RefCell}; +use std::path::PathBuf; +use std::ptr::NonNull; +use std::rc::Rc; + +use block2::RcBlock; +use objc2::rc::Retained; +use objc2::{msg_send, MainThreadMarker, MainThreadOnly}; +use objc2_app_kit::{ + NSBackingStoreType, NSColor, NSPanel, NSScreen, NSWindowCollectionBehavior, NSWindowStyleMask, +}; +use objc2_foundation::{NSNumber, NSPoint, NSRect, NSSize, NSString, NSTimer, NSURLRequest, NSURL}; +use objc2_web_kit::{WKWebView, WKWebViewConfiguration}; +use tauri::{AppHandle, Manager}; + +use crate::AppRuntime; + +/// Logical width of the notch panel. Wide enough to display voice/action text. +const PANEL_WIDTH: f64 = 380.0; +/// Logical height — covers the menu-bar / notch depth with headroom for the pill. +const PANEL_HEIGHT: f64 = 54.0; +/// URL-inject timer interval in seconds. +const INJECT_POLL_SECONDS: f64 = 1.0; +/// Ticks to wait before the first inject attempt (page-load delay). +const PAGE_LOAD_TICKS: u32 = 2; + +struct NotchPanel { + panel: Retained, + #[allow(dead_code)] + webview: Retained, + inject_timer: Retained, +} + +impl NotchPanel { + fn order_out(&self) { + self.inject_timer.invalidate(); + self.panel.orderOut(None); + } +} + +thread_local! { + /// Accessed only from the main thread. NSPanel/WKWebView are not Send/Sync + /// so a thread-local is the simplest safe storage. + static NOTCH: RefCell> = const { RefCell::new(None) }; +} + +pub(crate) fn is_open() -> bool { + NOTCH.with(|cell| cell.borrow().is_some()) +} + +pub(crate) fn hide() { + NOTCH.with(|cell| { + if let Some(existing) = cell.borrow_mut().take() { + log::info!("[notch-window] dropping panel"); + existing.order_out(); + } + }); +} + +pub(crate) fn show(app: &AppHandle) -> Result<(), String> { + if NOTCH.with(|cell| cell.borrow().is_some()) { + log::debug!("[notch-window] already open"); + return Ok(()); + } + + let mtm = MainThreadMarker::new() + .ok_or_else(|| "notch_window::show called off the main thread".to_string())?; + + let source = resolve_page_source(app)?; + // Log only the source *kind* — bundled paths contain `/Users//…` + // (PII), so never log the absolute resource paths. + log::info!( + "[notch-window] loading source_kind={}", + match &source { + PageSource::Dev { .. } => "dev", + PageSource::Bundled { .. } => "bundled", + } + ); + + let frame = top_center_frame(mtm); + log::debug!( + "[notch-window] frame origin=({:.0},{:.0}) size=({:.0},{:.0})", + frame.origin.x, + frame.origin.y, + frame.size.width, + frame.size.height + ); + + let panel = unsafe { build_panel(mtm, frame) }; + let webview = unsafe { build_webview(mtm, &panel, &source) }; + + panel.orderFrontRegardless(); + + let inject_timer = unsafe { spawn_inject_timer(webview.clone()) }; + + NOTCH.with(|cell| { + *cell.borrow_mut() = Some(NotchPanel { + panel, + webview, + inject_timer, + }); + }); + log::info!("[notch-window] panel shown at top-center"); + Ok(()) +} + +// ── Page source ─────────────────────────────────────────────────────────────── + +#[derive(Debug)] +enum PageSource { + Dev { url: String }, + Bundled { index_html: PathBuf, root: PathBuf }, +} + +fn resolve_page_source(app: &AppHandle) -> Result { + if let Some(mut url) = app.config().build.dev_url.as_ref().cloned() { + let query = url + .query() + .map(|q| format!("{q}&window=notch")) + .unwrap_or_else(|| "window=notch".into()); + url.set_query(Some(&query)); + return Ok(PageSource::Dev { + url: url.to_string(), + }); + } + + let resource_dir = app + .path() + .resource_dir() + .map_err(|e| format!("resolve resource_dir: {e}"))?; + for candidate in [ + resource_dir.join("index.html"), + resource_dir.join("dist").join("index.html"), + ] { + if candidate.is_file() { + let root = candidate + .parent() + .map(|p| p.to_path_buf()) + .unwrap_or_else(|| resource_dir.clone()); + return Ok(PageSource::Bundled { + index_html: candidate, + root, + }); + } + } + Err("notch bundled index.html not found under the app resource dir".to_string()) +} + +// ── Frame geometry ──────────────────────────────────────────────────────────── + +fn primary_screen_frame(mtm: MainThreadMarker) -> NSRect { + let screens = NSScreen::screens(mtm); + if let Some(primary) = screens.firstObject() { + return primary.frame(); + } + log::warn!("[notch-window] NSScreen::screens returned empty — falling back to 1440×900"); + NSRect::new(NSPoint::new(0.0, 0.0), NSSize::new(1440.0, 900.0)) +} + +/// Centre the panel horizontally at the very top of the primary screen. +/// +/// AppKit uses a bottom-left origin, so: +/// top-y = screen.origin.y + screen.height − PANEL_HEIGHT +/// center-x = screen.origin.x + (screen.width − PANEL_WIDTH) / 2 +fn top_center_frame(mtm: MainThreadMarker) -> NSRect { + let screen = primary_screen_frame(mtm); + let x = screen.origin.x + (screen.size.width - PANEL_WIDTH) / 2.0; + let y = screen.origin.y + screen.size.height - PANEL_HEIGHT; + NSRect::new(NSPoint::new(x, y), NSSize::new(PANEL_WIDTH, PANEL_HEIGHT)) +} + +// ── NSPanel construction ────────────────────────────────────────────────────── + +unsafe fn build_panel(mtm: MainThreadMarker, frame: NSRect) -> Retained { + let style = NSWindowStyleMask::Borderless | NSWindowStyleMask::NonactivatingPanel; + let panel: Retained = unsafe { + let allocated = NSPanel::alloc(mtm); + msg_send![ + allocated, + initWithContentRect: frame, + styleMask: style, + backing: NSBackingStoreType::Buffered, + defer: false, + ] + }; + + unsafe { + panel.setOpaque(false); + let clear = NSColor::clearColor(); + panel.setBackgroundColor(Some(&clear)); + panel.setHasShadow(false); + + // Float above the menu bar. NSStatusWindowLevel = 25, which sits above + // NSMainMenuWindowLevel = 24. Same recipe used by the mascot panel and + // the `configure_overlay_window_macos` helper. + panel.setLevel(25); + panel.setCollectionBehavior( + NSWindowCollectionBehavior::CanJoinAllSpaces + | NSWindowCollectionBehavior::Transient + | NSWindowCollectionBehavior::FullScreenAuxiliary + | NSWindowCollectionBehavior::IgnoresCycle, + ); + panel.setFloatingPanel(true); + panel.setHidesOnDeactivate(false); + panel.setBecomesKeyOnlyIfNeeded(true); + panel.setWorksWhenModal(true); + + // Fully click-through: the panel never steals mouse events. Menu-bar + // items remain clickable through the transparent regions. + panel.setIgnoresMouseEvents(true); + + let _: () = msg_send![&*panel, setExcludedFromWindowsMenu: true]; + } + + panel +} + +// ── WKWebView construction ──────────────────────────────────────────────────── + +unsafe fn build_webview( + mtm: MainThreadMarker, + panel: &NSPanel, + source: &PageSource, +) -> Retained { + let config: Retained = unsafe { + let alloc = WKWebViewConfiguration::alloc(mtm); + msg_send![alloc, init] + }; + + let frame = NSRect::new( + NSPoint::new(0.0, 0.0), + NSSize::new(PANEL_WIDTH, PANEL_HEIGHT), + ); + let webview: Retained = + unsafe { WKWebView::initWithFrame_configuration(WKWebView::alloc(mtm), frame, &config) }; + + unsafe { + // Disable WKWebView's own background so CSS `background: transparent` works. + // There is no public API for this on macOS — KVC against the private + // `drawsBackground` property is the canonical approach (used by wry, Electron). + let no = NSNumber::numberWithBool(false); + let key = NSString::from_str("drawsBackground"); + let _: () = msg_send![&*webview, setValue: &*no, forKey: &*key]; + + // Auto-resize to fill the panel content view. + let _: () = msg_send![&*webview, setAutoresizingMask: 18u64]; // width|height + + let webview_ref: &objc2::runtime::AnyObject = &*webview; + let webview_view = webview_ref as *const _ as *mut objc2::runtime::AnyObject; + let _: () = msg_send![panel, setContentView: webview_view]; + + match source { + PageSource::Dev { url } => { + let ns_url_str = NSString::from_str(url); + let ns_url = NSURL::URLWithString(&ns_url_str); + if let Some(ns_url) = ns_url { + let request = NSURLRequest::requestWithURL(&ns_url); + let _ = webview.loadRequest(&request); + } else { + log::warn!("[notch-window] could not parse dev url={url}"); + } + } + PageSource::Bundled { index_html, root } => { + let Ok(mut file_url) = url::Url::from_file_path(index_html) else { + log::warn!( + "[notch-window] index_html not absolute: {}", + index_html.display() + ); + return webview; + }; + file_url.set_query(Some("window=notch")); + let Ok(read_access_url) = url::Url::from_file_path(root) else { + log::warn!( + "[notch-window] resource root not absolute: {}", + root.display() + ); + return webview; + }; + let ns_url_str = NSString::from_str(file_url.as_str()); + let read_access_str = NSString::from_str(read_access_url.as_str()); + match ( + NSURL::URLWithString(&ns_url_str), + NSURL::URLWithString(&read_access_str), + ) { + (Some(ns_url), Some(read_access_ns)) => { + let _ = + webview.loadFileURL_allowingReadAccessToURL(&ns_url, &read_access_ns); + log::info!( + "[notch-window] loaded bundled page index={}", + index_html + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("index.html") + ); + } + _ => log::warn!( + "[notch-window] could not parse bundled file URLs (index={})", + index_html + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("index.html") + ), + } + } + } + } + + webview +} + +// ── Core-URL injection timer ────────────────────────────────────────────────── + +/// Spawn a 1 Hz repeating timer that waits for the embedded core to become +/// ready (indicated by `CoreProcessHandle` setting `OPENHUMAN_CORE_RPC_URL` +/// in the process env), then injects the base URL into the WKWebView. +/// +/// After the first successful inject the timer becomes a no-op until it is +/// invalidated by `NotchPanel::order_out()` when the panel is hidden. +unsafe fn spawn_inject_timer(webview: Retained) -> Retained { + let tick_count: Rc> = Rc::new(Cell::new(0)); + let injected: Rc> = Rc::new(Cell::new(false)); + + let block = RcBlock::new(move |_timer: NonNull| { + tick_count.set(tick_count.get() + 1); + + if injected.get() || tick_count.get() < PAGE_LOAD_TICKS { + return; + } + + let Ok(rpc_url) = std::env::var("OPENHUMAN_CORE_RPC_URL") else { + return; // Core not ready yet — try again next tick. + }; + + // Strip `/rpc` path suffix; Socket.IO connects to the base host. + let base_url = rpc_url.trim_end_matches("/rpc").to_string(); + + // The core Socket.IO handshake rejects unauthenticated clients, and this + // WKWebView has no Tauri IPC, so `getCoreRpcToken()` can't `invoke`. Hand + // the per-process bearer in via a global the same way as the URL (our own + // first-party webview — same trust as the renderer's `core_rpc_token`). + // The token is published *after* the URL env is set (post embedded spawn), + // so wait for it rather than injecting an empty token that gets rejected. + let token = match crate::core_process::current_rpc_token() { + Some(t) if !t.is_empty() => t, + _ => return, // bearer not published yet — retry next tick + }; + log::info!( + "[notch-window] injecting core url + bearer (token_len={})", + token.len() + ); + + // Set a global AND dispatch a custom event so React can pick up the URL + // regardless of whether the component mounted before or after this fires. + let js = format!( + "window.__OPENHUMAN_NOTCH_CORE_TOKEN__='{token}';\ + window.__OPENHUMAN_NOTCH_CORE_URL__='{base_url}';\ + window.dispatchEvent(new CustomEvent('notch:core-url',{{detail:{{url:'{base_url}'}}}}));" + ); + let js_str = NSString::from_str(&js); + unsafe { + let _: () = msg_send![ + &*webview, + evaluateJavaScript: &*js_str, + completionHandler: std::ptr::null::() + ]; + } + + injected.set(true); + log::debug!("[notch-window] injected core URL base={base_url}"); + }); + + unsafe { + NSTimer::scheduledTimerWithTimeInterval_repeats_block(INJECT_POLL_SECONDS, true, &block) + } +} diff --git a/app/src/index.css b/app/src/index.css index 888dbd678..c7f2a89e4 100644 --- a/app/src/index.css +++ b/app/src/index.css @@ -45,12 +45,51 @@ html[data-window='overlay'] #root, html[data-window='mascot'], html[data-window='mascot'] body, - html[data-window='mascot'] #root { + html[data-window='mascot'] #root, + html[data-window='notch'], + html[data-window='notch'] body, + html[data-window='notch'] #root { background: transparent; overflow: hidden; user-select: none; } + @keyframes notch-pill-in { + from { + opacity: 0; + transform: scaleX(0.4) scaleY(0.7); + } + to { + opacity: 1; + transform: scaleX(1) scaleY(1); + } + } + + @keyframes notch-bar { + 0%, + 100% { + transform: scaleY(0.5); + opacity: 0.6; + } + 50% { + transform: scaleY(1.2); + opacity: 1; + } + } + + @keyframes notch-dot { + 0%, + 80%, + 100% { + transform: scale(0.6); + opacity: 0.4; + } + 40% { + transform: scale(1); + opacity: 1; + } + } + @keyframes overlay-bubble-in { from { opacity: 0; diff --git a/app/src/main.tsx b/app/src/main.tsx index ac2b21be1..45bd27736 100644 --- a/app/src/main.tsx +++ b/app/src/main.tsx @@ -8,6 +8,7 @@ import App from './App'; import './index.css'; import { getCoreStateSnapshot } from './lib/coreState/store'; import MascotWindowApp from './mascot/MascotWindowApp'; +import NotchApp from './notch/NotchApp'; import OverlayApp from './overlay/OverlayApp'; import './polyfills'; import { initGA, initSentry, startUiInteractionTracking, trackEvent } from './services/analytics'; @@ -37,13 +38,16 @@ const urlWindowParam = (() => { } })(); const isMascotWindow = urlWindowParam === 'mascot'; +const isNotchWindow = urlWindowParam === 'notch'; const currentWindowLabel = isMascotWindow ? 'mascot' - : tauriRuntimeAvailable() - ? getCurrentWindow().label - : 'main'; + : isNotchWindow + ? 'notch' + : tauriRuntimeAvailable() + ? getCurrentWindow().label + : 'main'; const isOverlayWindow = currentWindowLabel === 'overlay'; -const isStandaloneWindow = isOverlayWindow || isMascotWindow; +const isStandaloneWindow = isOverlayWindow || isMascotWindow || isNotchWindow; const ensureDefaultHashRoute = () => { const hash = window.location.hash; @@ -83,17 +87,26 @@ if (!isStandaloneWindow) { // namespace from the first storage call. (#900) function bootRender() { const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); - const tree = isMascotWindow ? : isOverlayWindow ? : ; + const tree = isMascotWindow ? ( + + ) : isNotchWindow ? ( + + ) : isOverlayWindow ? ( + + ) : ( + + ); root.render({tree}); } -// The mascot lives in a native WKWebView (no Tauri IPC), so +// The mascot and notch windows live in native WKWebViews (no Tauri IPC), so // `getActiveUserIdFromCore()` would just reject after a roundtrip and -// delay first paint for nothing. Skip the bootstrap entirely in that -// path — the mascot UI doesn't read user-scoped storage anyway. -const activeUserBootstrap = isMascotWindow - ? Promise.resolve(null) - : getActiveUserIdFromCore(); +// delay first paint for nothing. Skip the bootstrap entirely in those +// paths — neither UI reads user-scoped storage. +const activeUserBootstrap = + isMascotWindow || isNotchWindow + ? Promise.resolve(null) + : getActiveUserIdFromCore(); activeUserBootstrap .then(id => primeActiveUserId(id)) diff --git a/app/src/notch/NotchApp.test.tsx b/app/src/notch/NotchApp.test.tsx new file mode 100644 index 000000000..82b28c624 --- /dev/null +++ b/app/src/notch/NotchApp.test.tsx @@ -0,0 +1,124 @@ +import { render, screen, waitFor } from '@testing-library/react'; +import { act } from 'react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { connectCoreSocket } from '../services/coreSocket'; +import NotchApp from './NotchApp'; + +// Key-passthrough i18n that honours the inline fallback the component passes +// (`t('notch.listening', 'Listening…')`). +vi.mock('../lib/i18n/I18nContext', () => ({ + useT: () => ({ t: (key: string, fallback?: string) => fallback ?? key }), +})); + +vi.mock('../services/coreSocket', () => ({ connectCoreSocket: vi.fn() })); + +// Minimal Socket.IO stand-in: records handlers so a test can replay events. +class MockSocket { + static last: MockSocket | null = null; + handlers = new Map void>(); + connected = false; + id = 'notch-test-socket'; + connect = vi.fn(() => { + this.connected = true; + return this; + }); + disconnect = vi.fn(() => { + this.connected = false; + return this; + }); + on = vi.fn((event: string, handler: (payload: unknown) => void) => { + this.handlers.set(event, handler); + return this; + }); + fire(event: string, payload: unknown) { + const handler = this.handlers.get(event); + if (!handler) throw new Error(`no handler registered for ${event}`); + act(() => handler(payload)); + } + constructor() { + MockSocket.last = this; + } +} + +describe('NotchApp', () => { + beforeEach(() => { + vi.clearAllMocks(); + MockSocket.last = null; + (window as { __OPENHUMAN_NOTCH_CORE_URL__?: string }).__OPENHUMAN_NOTCH_CORE_URL__ = + 'http://127.0.0.1:9999'; + vi.mocked(connectCoreSocket).mockImplementation( + async () => new MockSocket() as unknown as Awaited> + ); + }); + + afterEach(() => { + delete (window as { __OPENHUMAN_NOTCH_CORE_URL__?: string }).__OPENHUMAN_NOTCH_CORE_URL__; + }); + + const renderAndConnect = async () => { + render(); + // Idle baseline pill. + expect(screen.getByText('Ready')).toBeInTheDocument(); + // Wait for the async connect to register its socket handlers. + await waitFor(() => expect(MockSocket.last).not.toBeNull()); + await waitFor(() => + expect(MockSocket.last?.on).toHaveBeenCalledWith('overlay:attention', expect.any(Function)) + ); + return MockSocket.last as MockSocket; + }; + + it('connects using the preloaded core URL and shows the idle pill', async () => { + const socket = await renderAndConnect(); + expect(connectCoreSocket).toHaveBeenCalledTimes(1); + expect(socket.connect).toHaveBeenCalled(); + }); + + it('renders Listening on a dictation press', async () => { + const socket = await renderAndConnect(); + socket.fire('dictation:toggle', { type: 'pressed' }); + expect(await screen.findByText('Listening…')).toBeInTheDocument(); + }); + + it('renders the transcript text on dictation:transcription', async () => { + const socket = await renderAndConnect(); + socket.fire('dictation:transcription', { text: 'play some music' }); + expect(await screen.findByText('play some music')).toBeInTheDocument(); + }); + + it('maps companion:state_changed to a mode', async () => { + const socket = await renderAndConnect(); + socket.fire('companion:state_changed', { state: 'thinking' }); + expect(await screen.findByText('Processing…')).toBeInTheDocument(); + }); + + it('renders an overlay:attention message', async () => { + const socket = await renderAndConnect(); + socket.fire('overlay:attention', { message: 'Opening Music', ttl_ms: 5000 }); + expect(await screen.findByText('Opening Music')).toBeInTheDocument(); + }); + + it('handles speaking, released and idle transitions without throwing', async () => { + const socket = await renderAndConnect(); + socket.fire('companion:state_changed', { state: 'speaking' }); + expect(await screen.findByText('Speaking…')).toBeInTheDocument(); + // Released schedules a dismiss; idle drives an immediate dismiss — both + // exercise the scheduleDismiss branches. + socket.fire('dictation:toggle', { type: 'released' }); + socket.fire('companion:state_changed', { state: 'idle' }); + }); + + it('connects via the notch:core-url event when no URL was preloaded', async () => { + delete (window as { __OPENHUMAN_NOTCH_CORE_URL__?: string }).__OPENHUMAN_NOTCH_CORE_URL__; + render(); + expect(screen.getByText('Ready')).toBeInTheDocument(); + expect(connectCoreSocket).not.toHaveBeenCalled(); + + act(() => + window.dispatchEvent( + new CustomEvent('notch:core-url', { detail: { url: 'http://127.0.0.1:8888' } }) + ) + ); + await waitFor(() => expect(connectCoreSocket).toHaveBeenCalledTimes(1)); + }); +}); diff --git a/app/src/notch/NotchApp.tsx b/app/src/notch/NotchApp.tsx new file mode 100644 index 000000000..3cfcd8935 --- /dev/null +++ b/app/src/notch/NotchApp.tsx @@ -0,0 +1,331 @@ +/** + * NotchApp + * + * Standalone React root rendered inside the native macOS NSPanel that floats + * at the top-centre of the primary screen (see `app/src-tauri/src/notch_window.rs`). + * + * The panel has no Tauri IPC bridge (WKWebView outside the CEF runtime). The + * Rust host injects the core base URL via `evaluateJavaScript` once + * `OPENHUMAN_CORE_RPC_URL` is set by `CoreProcessHandle`, dispatching: + * `window.__OPENHUMAN_NOTCH_CORE_URL__` (global) + * `notch:core-url` CustomEvent (for late mounts) + * + * This component connects to the core over Socket.IO — identical to + * `OverlayApp` — and renders a pill that expands from the notch area when + * voice is active or the agent is performing an action. + * + * Events handled: + * dictation:toggle voice recording started / stopped + * dictation:transcription final transcript text + * companion:state_changed agent lifecycle (thinking, speaking, …) + * overlay:attention core broadcast message + */ +import { useCallback, useEffect, useRef, useState } from 'react'; +import type { Socket } from 'socket.io-client'; + +import { useT } from '../lib/i18n/I18nContext'; +import { connectCoreSocket } from '../services/coreSocket'; + +// ── Types ───────────────────────────────────────────────────────────────────── + +// 'ready' is the always-visible idle baseline (shows "Ready"); the pill never +// fully disappears so the user always knows the listener's status. +type NotchMode = 'ready' | 'listening' | 'transcribing' | 'thinking' | 'speaking' | 'attention'; + +interface NotchState { + mode: NotchMode; + text: string; +} + +interface DictationTogglePayload { + type?: string; +} +interface DictationTranscriptionPayload { + text?: string; +} +interface CompanionStatePayload { + state?: string; + message?: string; +} +interface AttentionPayload { + message?: string; + ttl_ms?: number; +} + +// ── Constants ───────────────────────────────────────────────────────────────── + +const LINGER_MS = 1800; +const DEFAULT_TTL_MS = 6000; + +// ── Waveform bars (voice activity animation) ────────────────────────────────── + +function WaveformBars() { + return ( + + ); +} + +// ── Spinner dots ────────────────────────────────────────────────────────────── + +function SpinnerDots() { + return ( + + ); +} + +// ── Icon glyph ──────────────────────────────────────────────────────────────── + +function ModeIcon({ mode }: { mode: NotchMode }) { + // Steady green dot when idle/ready — calm "I'm listening for the wake word". + if (mode === 'ready') return ; + if (mode === 'listening') return ; + if (mode === 'transcribing' || mode === 'thinking') return ; + if (mode === 'speaking') { + return ( + + ); + } + // attention / fallback + return ; +} + +// ── Main component ──────────────────────────────────────────────────────────── + +export default function NotchApp() { + const { t } = useT(); + const [state, setState] = useState({ mode: 'ready', text: '' }); + const dismissRef = useRef(null); + const socketRef = useRef(null); + + const clearDismiss = useCallback(() => { + if (dismissRef.current !== null) { + window.clearTimeout(dismissRef.current); + dismissRef.current = null; + } + }, []); + + const scheduleDismiss = useCallback( + (ms: number) => { + clearDismiss(); + dismissRef.current = window.setTimeout(() => { + // Fall back to the always-visible "Ready" baseline, never invisible. + setState({ mode: 'ready', text: '' }); + dismissRef.current = null; + }, ms); + }, + [clearDismiss] + ); + + // ── Socket.IO connection ──────────────────────────────────────────────────── + + const connectSocket = useCallback( + (baseUrl: string) => { + if (socketRef.current?.connected) return; + if (socketRef.current) { + socketRef.current.disconnect(); + } + + let disposed = false; + void (async () => { + try { + const socket = await connectCoreSocket({ + getBaseUrl: async () => baseUrl, + isDisposed: () => disposed, + }); + if (!socket || disposed) return; + socketRef.current = socket; + + socket.on('dictation:toggle', (payload: DictationTogglePayload) => { + const type = payload?.type ?? 'pressed'; + console.debug(`[notch] dictation:toggle type=${type}`); + if (type === 'pressed') { + clearDismiss(); + setState({ mode: 'listening', text: t('notch.listening', 'Listening…') }); + } else if (type === 'released') { + scheduleDismiss(LINGER_MS); + } + }); + + socket.on('dictation:transcription', (payload: DictationTranscriptionPayload) => { + const text = payload?.text?.trim(); + if (!text) return; + console.debug(`[notch] dictation:transcription chars=${text.length}`); + clearDismiss(); + setState({ + mode: 'transcribing', + text: text.length > 60 ? `${text.slice(0, 57)}…` : text, + }); + scheduleDismiss(LINGER_MS); + }); + + socket.on('companion:state_changed', (payload: CompanionStatePayload) => { + const agentState = payload?.state ?? 'idle'; + console.debug(`[notch] companion:state_changed state=${agentState}`); + + if (agentState === 'idle') { + scheduleDismiss(0); + return; + } + clearDismiss(); + + const modeMap: Partial> = { + listening: 'listening', + thinking: 'thinking', + speaking: 'speaking', + }; + const textMap: Partial> = { + listening: t('notch.listening', 'Listening…'), + thinking: t('notch.processing', 'Processing…'), + speaking: t('notch.speaking', 'Speaking…'), + }; + + setState({ + mode: modeMap[agentState] ?? 'thinking', + text: textMap[agentState] ?? agentState, + }); + }); + + socket.on('overlay:attention', (payload: AttentionPayload) => { + const message = payload?.message?.trim(); + if (!message) return; + console.debug(`[notch] overlay:attention chars=${message.length}`); + clearDismiss(); + // The voice listener uses two reserved status words to drive the + // pill: "Listening" (capturing speech) and "Processing" (running a + // command). Map them to the matching icon; everything else is a + // generic attention message. + const lower = message.toLowerCase(); + const mode: NotchMode = + lower === 'listening' + ? 'listening' + : lower === 'processing' + ? 'thinking' + : 'attention'; + setState({ mode, text: message.length > 60 ? `${message.slice(0, 57)}…` : message }); + scheduleDismiss(payload?.ttl_ms ?? DEFAULT_TTL_MS); + }); + + socket.connect(); + console.debug('[notch] socket connected', socket.id); + } catch (err) { + console.warn('[notch] failed to connect socket', err); + } + })(); + + return () => { + disposed = true; + }; + }, + [t, clearDismiss, scheduleDismiss] + ); + + // ── Core URL bootstrap ────────────────────────────────────────────────────── + + useEffect(() => { + // Track the in-flight connect's disposer so an unmount (or a new core-url) + // cancels a still-resolving connectCoreSocket — otherwise the async branch + // could attach listeners / setState after teardown. + let disposePendingConnect: (() => void) | undefined; + + // Check if Rust already injected the URL before this component mounted. + const preloaded = (window as { __OPENHUMAN_NOTCH_CORE_URL__?: string }) + .__OPENHUMAN_NOTCH_CORE_URL__; + if (preloaded) { + disposePendingConnect = connectSocket(preloaded); + } + + // Also listen for the event (fires when core becomes ready after mount). + const handler = (e: CustomEvent<{ url: string }>) => { + if (e.detail?.url) { + disposePendingConnect?.(); + disposePendingConnect = connectSocket(e.detail.url); + } + }; + window.addEventListener('notch:core-url', handler as EventListener); + + return () => { + window.removeEventListener('notch:core-url', handler as EventListener); + disposePendingConnect?.(); + socketRef.current?.disconnect(); + socketRef.current = null; + clearDismiss(); + }; + }, [connectSocket, clearDismiss]); + + // ── Render ────────────────────────────────────────────────────────────────── + + const { mode, text } = state; + + // The pill is ALWAYS visible so the user can always see the listener status: + // Ready (idle) · Listening (capturing speech) · Processing (running a command). + const label = text || (mode === 'ready' ? t('notch.ready', 'Ready') : ''); + + const pillBg = + mode === 'speaking' + ? 'bg-[rgba(10,40,10,0.92)]' + : mode === 'ready' + ? 'bg-[rgba(10,10,10,0.72)]' // dimmer when idle + : 'bg-[rgba(10,10,10,0.92)]'; + + return ( +
+
+ + {label && ( + + {label} + + )} +
+
+ ); +} diff --git a/app/src/services/__tests__/coreRpcClient.test.ts b/app/src/services/__tests__/coreRpcClient.test.ts index af2c7ce88..e82d577cf 100644 --- a/app/src/services/__tests__/coreRpcClient.test.ts +++ b/app/src/services/__tests__/coreRpcClient.test.ts @@ -1105,6 +1105,20 @@ describe('getCoreRpcToken (cloud-mode persistence)', () => { expect(headers.Authorization).toBe('Bearer cloud-token-abc'); }); + test('honours the host-injected notch core token before the cache/store', async () => { + // The notch / overlay WKWebViews have no Tauri IPC; the Rust host injects + // the bearer as a global, which must win ahead of the resolution cache. + (globalThis as { __OPENHUMAN_NOTCH_CORE_TOKEN__?: string }).__OPENHUMAN_NOTCH_CORE_TOKEN__ = + 'notch-bearer-xyz'; + try { + const { getCoreRpcToken } = await import('../coreRpcClient'); + await expect(getCoreRpcToken()).resolves.toBe('notch-bearer-xyz'); + } finally { + delete (globalThis as { __OPENHUMAN_NOTCH_CORE_TOKEN__?: string }) + .__OPENHUMAN_NOTCH_CORE_TOKEN__; + } + }); + test('clearCoreRpcTokenCache forces a re-resolve on the next call', async () => { let storedToken: string | null = 'first-token'; vi.doMock('../../utils/configPersistence', () => ({ diff --git a/app/src/services/coreRpcClient.ts b/app/src/services/coreRpcClient.ts index a1eb0e0b3..bb10608de 100644 --- a/app/src/services/coreRpcClient.ts +++ b/app/src/services/coreRpcClient.ts @@ -391,6 +391,18 @@ export async function getCoreRpcUrl(): Promise { * stored token is set so existing tests remain unaffected. */ export async function getCoreRpcToken(): Promise { + // Non-Tauri first-party webviews (the notch / overlay NSPanel WKWebViews have + // no Tauri IPC) receive the per-process bearer injected as a global by the + // Rust host. Honour it first — and not behind the resolution cache, so a late + // injection (the host injects on a timer once the core URL is ready) still wins. + const injected = (globalThis as { __OPENHUMAN_NOTCH_CORE_TOKEN__?: string }) + .__OPENHUMAN_NOTCH_CORE_TOKEN__; + if (typeof injected === 'string' && injected) { + resolvedCoreRpcToken = injected; + didResolveCoreRpcToken = true; + return injected; + } + if (didResolveCoreRpcToken) return resolvedCoreRpcToken; const storedToken = getStoredCoreToken();