From 16d99cf48a25cfffa8a9ca8bf800c92ccd30c525 Mon Sep 17 00:00:00 2001 From: aprv10 <1apoorvindia@gmail.com> Date: Tue, 21 Jul 2026 15:41:29 +0530 Subject: [PATCH 01/11] feat(browser): add session-scoped agent control --- backend/internal/browserruntime/broker.go | 259 +++++++++++ .../internal/browserruntime/broker_test.go | 135 ++++++ .../internal/browserruntime/listen_unix.go | 23 + .../internal/browserruntime/listen_windows.go | 33 ++ backend/internal/cli/browser.go | 306 +++++++++++++ backend/internal/cli/browser_test.go | 125 +++++ backend/internal/cli/root.go | 1 + backend/internal/daemon/daemon.go | 13 + backend/internal/httpd/api.go | 4 + backend/internal/httpd/apispec/openapi.yaml | 139 ++++++ .../internal/httpd/apispec/specgen/build.go | 37 ++ backend/internal/httpd/controllers/browser.go | 130 ++++++ .../httpd/controllers/browser_test.go | 98 ++++ backend/internal/httpd/controllers/dto.go | 31 ++ backend/internal/httpd/lan_listener.go | 1 + backend/internal/httpd/lan_listener_test.go | 3 +- backend/internal/session_manager/manager.go | 7 +- .../internal/session_manager/manager_test.go | 10 +- .../internal/skillassets/skillassets_test.go | 3 + .../internal/skillassets/using-ao/SKILL.md | 6 +- .../skillassets/using-ao/commands/browser.md | 40 ++ .../skillassets/using-ao/references.md | 4 + docs/STATUS.md | 5 + docs/architecture.md | 14 + docs/cli/README.md | 9 + frontend/src/api/schema.ts | 182 ++++++++ frontend/src/main.ts | 46 ++ .../src/main/browser-runtime-link.test.ts | 97 ++++ frontend/src/main/browser-runtime-link.ts | 171 +++++++ frontend/src/main/browser-view-host.test.ts | 138 ++++++ frontend/src/main/browser-view-host.ts | 430 +++++++++++++++++- 31 files changed, 2470 insertions(+), 30 deletions(-) create mode 100644 backend/internal/browserruntime/broker.go create mode 100644 backend/internal/browserruntime/broker_test.go create mode 100644 backend/internal/browserruntime/listen_unix.go create mode 100644 backend/internal/browserruntime/listen_windows.go create mode 100644 backend/internal/cli/browser.go create mode 100644 backend/internal/cli/browser_test.go create mode 100644 backend/internal/httpd/controllers/browser.go create mode 100644 backend/internal/httpd/controllers/browser_test.go create mode 100644 backend/internal/skillassets/using-ao/commands/browser.md create mode 100644 frontend/src/main/browser-runtime-link.test.ts create mode 100644 frontend/src/main/browser-runtime-link.ts diff --git a/backend/internal/browserruntime/broker.go b/backend/internal/browserruntime/broker.go new file mode 100644 index 0000000000..1269f04207 --- /dev/null +++ b/backend/internal/browserruntime/broker.go @@ -0,0 +1,259 @@ +// Package browserruntime brokers browser commands between the loopback daemon +// and the Electron process that owns AO's per-session WebContentsView targets. +package browserruntime + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "sync" + "time" + + "github.com/google/uuid" + + "github.com/aoagents/agent-orchestrator/backend/internal/domain" +) + +const ProtocolVersion = 1 + +var ErrUnavailable = errors.New("browser runtime is unavailable") + +type Status struct { + Connected bool + ConnectedAt time.Time +} + +type Command struct { + RequestID string `json:"requestId"` + SessionID domain.SessionID `json:"sessionId"` + Action string `json:"action"` + Args map[string]interface{} `json:"args,omitempty"` +} + +type CommandError struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (e CommandError) Error() string { + if e.Code == "" { + return e.Message + } + return fmt.Sprintf("%s (%s)", e.Message, e.Code) +} + +type Result struct { + RequestID string + Value interface{} +} + +type wireMessage struct { + Type string `json:"type"` + Version int `json:"version,omitempty"` + RequestID string `json:"requestId,omitempty"` + SessionID domain.SessionID `json:"sessionId,omitempty"` + Action string `json:"action,omitempty"` + Args map[string]interface{} `json:"args,omitempty"` + OK bool `json:"ok,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *CommandError `json:"error,omitempty"` +} + +type pendingResult struct { + value interface{} + err error +} + +// Broker owns the single active Electron runtime connection. Commands are +// correlated by request id, so independent AO sessions may use the bridge +// concurrently without sharing browser targets or results. +type Broker struct { + log *slog.Logger + + mu sync.Mutex + conn net.Conn + connectedAt time.Time + pending map[string]chan pendingResult + writeMu sync.Mutex +} + +func New(log *slog.Logger) *Broker { + if log == nil { + log = slog.Default() + } + return &Broker{log: log, pending: make(map[string]chan pendingResult)} +} + +func (b *Broker) Status() Status { + b.mu.Lock() + defer b.mu.Unlock() + return Status{Connected: b.conn != nil, ConnectedAt: b.connectedAt} +} + +func (b *Broker) Execute(ctx context.Context, sessionID domain.SessionID, action string, args map[string]interface{}) (Result, error) { + requestID := uuid.NewString() + resultCh := make(chan pendingResult, 1) + + b.mu.Lock() + conn := b.conn + if conn == nil { + b.mu.Unlock() + return Result{}, ErrUnavailable + } + b.pending[requestID] = resultCh + b.mu.Unlock() + + msg := wireMessage{ + Type: "command", + RequestID: requestID, + SessionID: sessionID, + Action: action, + Args: args, + } + if err := b.write(conn, msg); err != nil { + b.removePending(requestID) + b.disconnect(conn, fmt.Errorf("write browser command: %w", err)) + return Result{}, ErrUnavailable + } + + select { + case <-ctx.Done(): + b.removePending(requestID) + return Result{}, ctx.Err() + case result := <-resultCh: + if result.err != nil { + return Result{}, result.err + } + return Result{RequestID: requestID, Value: result.value}, nil + } +} + +// Serve accepts Electron runtime connections until ctx is cancelled. A new +// valid runtime replaces an older connection and fails its in-flight commands; +// this makes renderer reload/restart recovery deterministic. +func (b *Broker) Serve(ctx context.Context, ln net.Listener) error { + go func() { + <-ctx.Done() + _ = ln.Close() + }() + for { + conn, err := ln.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("accept browser runtime: %w", err) + } + go b.serveConn(ctx, conn) + } +} + +func (b *Broker) serveConn(ctx context.Context, conn net.Conn) { + dec := json.NewDecoder(conn) + var hello wireMessage + if err := dec.Decode(&hello); err != nil || hello.Type != "hello" || hello.Version != ProtocolVersion { + _ = conn.Close() + return + } + + b.mu.Lock() + old := b.conn + b.conn = conn + b.connectedAt = time.Now().UTC() + pending := b.takePendingLocked() + b.mu.Unlock() + if old != nil && old != conn { + _ = old.Close() + } + failPending(pending, ErrUnavailable) + b.log.Info("browser runtime connected") + + go func() { + <-ctx.Done() + _ = conn.Close() + }() + + for { + var msg wireMessage + if err := dec.Decode(&msg); err != nil { + b.disconnect(conn, err) + return + } + if msg.Type != "result" || msg.RequestID == "" { + continue + } + b.resolve(msg) + } +} + +func (b *Broker) write(conn net.Conn, msg wireMessage) error { + b.writeMu.Lock() + defer b.writeMu.Unlock() + return json.NewEncoder(conn).Encode(msg) +} + +func (b *Broker) resolve(msg wireMessage) { + b.mu.Lock() + ch := b.pending[msg.RequestID] + delete(b.pending, msg.RequestID) + b.mu.Unlock() + if ch == nil { + return + } + if !msg.OK { + if msg.Error == nil { + msg.Error = &CommandError{Code: "BROWSER_COMMAND_FAILED", Message: "Browser command failed"} + } + ch <- pendingResult{err: *msg.Error} + return + } + var value interface{} = map[string]interface{}{} + if len(msg.Result) > 0 && string(msg.Result) != "null" { + if err := json.Unmarshal(msg.Result, &value); err != nil { + ch <- pendingResult{err: fmt.Errorf("decode browser result: %w", err)} + return + } + } + ch <- pendingResult{value: value} +} + +func (b *Broker) disconnect(conn net.Conn, cause error) { + b.mu.Lock() + if b.conn != conn { + b.mu.Unlock() + return + } + b.conn = nil + b.connectedAt = time.Time{} + pending := b.takePendingLocked() + b.mu.Unlock() + _ = conn.Close() + failPending(pending, ErrUnavailable) + if cause != nil && !errors.Is(cause, io.EOF) && !errors.Is(cause, net.ErrClosed) { + b.log.Warn("browser runtime disconnected", "err", cause) + } else { + b.log.Info("browser runtime disconnected") + } +} + +func (b *Broker) removePending(requestID string) { + b.mu.Lock() + delete(b.pending, requestID) + b.mu.Unlock() +} + +func (b *Broker) takePendingLocked() map[string]chan pendingResult { + pending := b.pending + b.pending = make(map[string]chan pendingResult) + return pending +} + +func failPending(pending map[string]chan pendingResult, err error) { + for _, ch := range pending { + ch <- pendingResult{err: err} + } +} diff --git a/backend/internal/browserruntime/broker_test.go b/backend/internal/browserruntime/broker_test.go new file mode 100644 index 0000000000..85828e6881 --- /dev/null +++ b/backend/internal/browserruntime/broker_test.go @@ -0,0 +1,135 @@ +package browserruntime + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net" + "testing" + "time" +) + +func TestBrokerExecuteRoundTrip(t *testing.T) { + broker := New(slog.New(slog.NewTextHandler(io.Discard, nil))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = broker.Serve(ctx, ln) }() + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + enc := json.NewEncoder(conn) + dec := json.NewDecoder(conn) + if err := enc.Encode(wireMessage{Type: "hello", Version: ProtocolVersion}); err != nil { + t.Fatal(err) + } + waitConnected(t, broker) + + resultCh := make(chan Result, 1) + errCh := make(chan error, 1) + go func() { + result, err := broker.Execute(context.Background(), "session-1", "snapshot", map[string]interface{}{"interactive": true}) + if err != nil { + errCh <- err + return + } + resultCh <- result + }() + + var command wireMessage + if err := dec.Decode(&command); err != nil { + t.Fatal(err) + } + if command.Type != "command" || command.SessionID != "session-1" || command.Action != "snapshot" { + t.Fatalf("command = %#v", command) + } + if err := enc.Encode(wireMessage{ + Type: "result", + RequestID: command.RequestID, + OK: true, + Result: json.RawMessage(`{"text":"button Save [ref=e1]"}`), + }); err != nil { + t.Fatal(err) + } + + select { + case err := <-errCh: + t.Fatal(err) + case result := <-resultCh: + value := result.Value.(map[string]interface{}) + if value["text"] != "button Save [ref=e1]" { + t.Fatalf("result = %#v", result.Value) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for result") + } +} + +func TestBrokerMapsRuntimeError(t *testing.T) { + broker := New(slog.New(slog.NewTextHandler(io.Discard, nil))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = broker.Serve(ctx, ln) }() + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + enc := json.NewEncoder(conn) + dec := json.NewDecoder(conn) + _ = enc.Encode(wireMessage{Type: "hello", Version: ProtocolVersion}) + waitConnected(t, broker) + + errCh := make(chan error, 1) + go func() { + _, err := broker.Execute(context.Background(), "session-1", "click", map[string]interface{}{"ref": "e1"}) + errCh <- err + }() + var command wireMessage + if err := dec.Decode(&command); err != nil { + t.Fatal(err) + } + _ = enc.Encode(wireMessage{ + Type: "result", + RequestID: command.RequestID, + Error: &CommandError{Code: "STALE_REFERENCE", Message: "snapshot again"}, + }) + + select { + case err := <-errCh: + commandErr, ok := err.(CommandError) + if !ok || commandErr.Code != "STALE_REFERENCE" { + t.Fatalf("error = %#v", err) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for error") + } +} + +func TestBrokerUnavailableWithoutElectron(t *testing.T) { + broker := New(nil) + if _, err := broker.Execute(context.Background(), "session-1", "snapshot", nil); err != ErrUnavailable { + t.Fatalf("error = %v, want ErrUnavailable", err) + } +} + +func waitConnected(t *testing.T, broker *Broker) { + t.Helper() + deadline := time.Now().Add(time.Second) + for !broker.Status().Connected { + if time.Now().After(deadline) { + t.Fatal("browser runtime did not connect") + } + time.Sleep(time.Millisecond) + } +} diff --git a/backend/internal/browserruntime/listen_unix.go b/backend/internal/browserruntime/listen_unix.go new file mode 100644 index 0000000000..231ffca0b4 --- /dev/null +++ b/backend/internal/browserruntime/listen_unix.go @@ -0,0 +1,23 @@ +//go:build !windows + +package browserruntime + +import ( + "net" + "os" + "path/filepath" +) + +func Listen(runFilePath string) (net.Listener, string, error) { + sockPath := filepath.Join(filepath.Dir(runFilePath), "browser.sock") + _ = os.Remove(sockPath) + ln, err := net.Listen("unix", sockPath) + if err != nil { + return nil, "", err + } + if err := os.Chmod(sockPath, 0o600); err != nil { + _ = ln.Close() + return nil, "", err + } + return ln, sockPath, nil +} diff --git a/backend/internal/browserruntime/listen_windows.go b/backend/internal/browserruntime/listen_windows.go new file mode 100644 index 0000000000..8e88981b32 --- /dev/null +++ b/backend/internal/browserruntime/listen_windows.go @@ -0,0 +1,33 @@ +//go:build windows + +package browserruntime + +import ( + "net" + "path/filepath" + "regexp" + + "github.com/Microsoft/go-winio" +) + +var unsafePipeChars = regexp.MustCompile(`[^a-zA-Z0-9\-]`) + +func pipeNameFromRunFile(runFilePath string) string { + if runFilePath == "" { + return `\\.\pipe\ao-browser` + } + dir := filepath.Base(filepath.Dir(runFilePath)) + if dir == ".ao" || dir == "." || dir == "" { + return `\\.\pipe\ao-browser` + } + return `\\.\pipe\ao-browser-` + unsafePipeChars.ReplaceAllString(dir, "-") +} + +func Listen(runFilePath string) (net.Listener, string, error) { + name := pipeNameFromRunFile(runFilePath) + ln, err := winio.ListenPipe(name, nil) + if err != nil { + return nil, "", err + } + return ln, name, nil +} diff --git a/backend/internal/cli/browser.go b/backend/internal/cli/browser.go new file mode 100644 index 0000000000..f8f04a3379 --- /dev/null +++ b/backend/internal/cli/browser.go @@ -0,0 +1,306 @@ +package cli + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" +) + +type browserStatusDTO struct { + SessionID string `json:"sessionId"` + Connected bool `json:"connected"` + ConnectedAt time.Time `json:"connectedAt,omitempty"` + Transport string `json:"transport"` +} + +type browserCommandRequestDTO struct { + SessionID string `json:"sessionId"` + Action string `json:"action"` + Args map[string]any `json:"args,omitempty"` +} + +type browserCommandResponseDTO struct { + RequestID string `json:"requestId"` + SessionID string `json:"sessionId"` + Action string `json:"action"` + Result map[string]any `json:"result"` +} + +func newBrowserCommand(ctx *commandContext) *cobra.Command { + var jsonOutput bool + cmd := &cobra.Command{ + Use: "browser", + Short: "Inspect and control this AO session's shared desktop browser", + Long: "Inspect and control the target-isolated browser owned by the current AO session.\n\n" + + "The desktop app must be open. Commands operate the same live page the user sees,\n" + + "including while the Browser panel is hidden.", + Args: noArgs, + } + cmd.PersistentFlags().BoolVar(&jsonOutput, "json", false, "print the structured response as JSON") + + cmd.AddCommand(&cobra.Command{ + Use: "status", + Short: "Show whether the desktop browser runtime is connected", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + status, err := ctx.browserStatus(cmd.Context()) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(cmd.OutOrStdout(), status) + } + state := "disconnected" + if status.Connected { + state = "connected" + } + _, err = fmt.Fprintf(cmd.OutOrStdout(), "Browser runtime: %s (%s)\n", state, status.Transport) + return err + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "open ", + Short: "Open a URL in this session's browser", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "open", map[string]any{"url": args[0]}, jsonOutput) + }, + }) + + var interactiveOnly bool + snapshot := &cobra.Command{ + Use: "snapshot", + Short: "Print a compact accessibility snapshot with actionable element refs", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return ctx.runBrowserAction(cmd, "snapshot", map[string]any{"interactive": interactiveOnly}, jsonOutput) + }, + } + snapshot.Flags().BoolVar(&interactiveOnly, "interactive", false, "include only actionable elements") + cmd.AddCommand(snapshot) + + cmd.AddCommand(&cobra.Command{ + Use: "click ", + Short: "Click an element reference from the latest snapshot", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "click", map[string]any{"ref": args[0]}, jsonOutput) + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "fill ", + Short: "Replace the value of a form control", + Args: exactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "fill", map[string]any{"ref": args[0], "text": args[1]}, jsonOutput) + }, + }) + + var waitText, waitSelector, waitURL string + var waitMS, timeoutMS int + waitCmd := &cobra.Command{ + Use: "wait", + Short: "Wait for time, text, selector, or URL state", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + selected := 0 + for _, active := range []bool{waitText != "", waitSelector != "", waitURL != "", waitMS > 0} { + if active { + selected++ + } + } + if selected != 1 { + return usageError{errors.New("choose exactly one of --text, --selector, --url, or --ms")} + } + args := map[string]any{"timeoutMs": timeoutMS} + switch { + case waitText != "": + args["text"] = waitText + case waitSelector != "": + args["selector"] = waitSelector + case waitURL != "": + args["url"] = waitURL + default: + args["ms"] = waitMS + } + return ctx.runBrowserAction(cmd, "wait", args, jsonOutput) + }, + } + waitCmd.Flags().StringVar(&waitText, "text", "", "wait until visible page text contains this value") + waitCmd.Flags().StringVar(&waitSelector, "selector", "", "wait until this CSS selector exists") + waitCmd.Flags().StringVar(&waitURL, "url", "", "wait until the current URL contains this value") + waitCmd.Flags().IntVar(&waitMS, "ms", 0, "wait for a fixed number of milliseconds") + waitCmd.Flags().IntVar(&timeoutMS, "timeout", 10_000, "condition timeout in milliseconds") + cmd.AddCommand(waitCmd) + + cmd.AddCommand(&cobra.Command{ + Use: "screenshot [path]", + Short: "Capture the current page to a PNG file", + Args: atMostOneArg, + RunE: func(cmd *cobra.Command, args []string) error { + resp, err := ctx.browserAction(cmd.Context(), "screenshot", nil) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(cmd.OutOrStdout(), resp) + } + path := "" + if len(args) == 1 { + path = args[0] + } else { + path = "ao-browser-" + ctx.deps.Now().Format("20060102-150405.000") + ".png" + } + return writeBrowserScreenshot(cmd, resp.Result, path) + }, + }) + + for _, action := range []string{"console", "errors"} { + action := action + cmd.AddCommand(&cobra.Command{ + Use: action, + Short: "Print captured browser " + action, + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return ctx.runBrowserAction(cmd, action, nil, jsonOutput) + }, + }) + } + return cmd +} + +func exactArgs(n int) cobra.PositionalArgs { + return func(cmd *cobra.Command, args []string) error { + if err := cobra.ExactArgs(n)(cmd, args); err != nil { + return usageError{err} + } + return nil + } +} + +func currentBrowserSessionID() (string, error) { + sessionID := strings.TrimSpace(os.Getenv("AO_SESSION_ID")) + if sessionID == "" { + return "", usageError{errors.New("ao browser must run inside an AO session (AO_SESSION_ID is not set)")} + } + return sessionID, nil +} + +func (c *commandContext) browserStatus(ctx context.Context) (browserStatusDTO, error) { + sessionID, err := currentBrowserSessionID() + if err != nil { + return browserStatusDTO{}, err + } + var out browserStatusDTO + err = c.getJSON(ctx, "browser/status?sessionId="+url.QueryEscape(sessionID), &out) + return out, err +} + +func (c *commandContext) browserAction(ctx context.Context, action string, args map[string]any) (browserCommandResponseDTO, error) { + sessionID, err := currentBrowserSessionID() + if err != nil { + return browserCommandResponseDTO{}, err + } + var out browserCommandResponseDTO + err = c.postJSON(ctx, "browser/commands", browserCommandRequestDTO{SessionID: sessionID, Action: action, Args: args}, &out) + return out, err +} + +func (c *commandContext) runBrowserAction(cmd *cobra.Command, action string, args map[string]any, jsonOutput bool) error { + resp, err := c.browserAction(cmd.Context(), action, args) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(cmd.OutOrStdout(), resp) + } + return writeBrowserResult(cmd, action, resp.Result) +} + +func writeBrowserResult(cmd *cobra.Command, action string, result map[string]any) error { + if action == "snapshot" { + if text, ok := result["text"].(string); ok { + _, err := fmt.Fprintln(cmd.OutOrStdout(), text) + return err + } + } + if action == "console" || action == "errors" { + messages, _ := result["messages"].([]any) + if len(messages) == 0 { + _, err := fmt.Fprintln(cmd.OutOrStdout(), "No browser "+action+" captured.") + return err + } + for _, message := range messages { + if item, ok := message.(map[string]any); ok { + level, _ := item["level"].(string) + text, _ := item["message"].(string) + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "[%s] %s\n", level, text); err != nil { + return err + } + } + } + return nil + } + if currentURL, ok := result["url"].(string); ok && currentURL != "" { + _, err := fmt.Fprintln(cmd.OutOrStdout(), currentURL) + return err + } + _, err := fmt.Fprintln(cmd.OutOrStdout(), "Browser "+action+" completed.") + return err +} + +func writeBrowserScreenshot(cmd *cobra.Command, result map[string]any, target string) error { + encoded, _ := result["data"].(string) + if encoded == "" { + return errors.New("browser returned an empty screenshot") + } + data, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return fmt.Errorf("decode browser screenshot: %w", err) + } + abs, err := filepath.Abs(target) + if err != nil { + return err + } + file, err := os.OpenFile(abs, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("refusing to overwrite existing screenshot %s", abs) + } + return err + } + defer func() { _ = file.Close() }() + if _, err := file.Write(data); err != nil { + return err + } + width := numberString(result["width"]) + height := numberString(result["height"]) + size := "" + if width != "" && height != "" { + size = " (" + width + "x" + height + ")" + } + _, err = fmt.Fprintf(cmd.OutOrStdout(), "Saved %s%s\n", abs, size) + return err +} + +func numberString(v any) string { + switch n := v.(type) { + case float64: + return strconv.Itoa(int(n)) + case int: + return strconv.Itoa(n) + default: + return "" + } +} diff --git a/backend/internal/cli/browser_test.go b/backend/internal/cli/browser_test.go new file mode 100644 index 0000000000..1e5d1b7739 --- /dev/null +++ b/backend/internal/cli/browser_test.go @@ -0,0 +1,125 @@ +package cli + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +type browserRequestCapture struct { + path string + body browserCommandRequestDTO +} + +func browserCLIServer(t *testing.T, capture *browserRequestCapture) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capture.path = r.URL.RequestURI() + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet && r.URL.Path == "/api/v1/browser/status" { + _, _ = io.WriteString(w, `{"sessionId":"ao-1","connected":true,"transport":"electron-webcontents-debugger"}`) + return + } + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/browser/commands" { + http.NotFound(w, r) + return + } + if err := json.NewDecoder(r.Body).Decode(&capture.body); err != nil { + t.Fatalf("decode command: %v", err) + } + result := `{"ok":true}` + switch capture.body.Action { + case "snapshot": + result = `{"text":"button Save [ref=e1]"}` + case "screenshot": + result = `{"data":"cG5n","width":10,"height":20}` + } + _, _ = io.WriteString(w, `{"requestId":"r1","sessionId":"ao-1","action":"`+capture.body.Action+`","result":`+result+`}`) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestBrowserStatusAndSnapshot(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + + out, errOut, err := executeCLI(t, deps, "browser", "status") + if err != nil || !strings.Contains(out, "Browser runtime: connected") { + t.Fatalf("status err=%v stderr=%s stdout=%s", err, errOut, out) + } + if capture.path != "/api/v1/browser/status?sessionId=ao-1" { + t.Fatalf("status path = %q", capture.path) + } + out, errOut, err = executeCLI(t, deps, "browser", "snapshot", "--interactive") + if err != nil || !strings.Contains(out, "button Save [ref=e1]") { + t.Fatalf("snapshot err=%v stderr=%s stdout=%s", err, errOut, out) + } + if capture.body.SessionID != "ao-1" || capture.body.Action != "snapshot" || capture.body.Args["interactive"] != true { + t.Fatalf("command = %#v", capture.body) + } +} + +func TestBrowserClickAndWaitArguments(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + + if _, _, err := executeCLI(t, deps, "browser", "click", "e2"); err != nil { + t.Fatal(err) + } + if capture.body.Action != "click" || capture.body.Args["ref"] != "e2" { + t.Fatalf("click = %#v", capture.body) + } + if _, _, err := executeCLI(t, deps, "browser", "wait", "--text", "Ready", "--timeout", "2500"); err != nil { + t.Fatal(err) + } + if capture.body.Action != "wait" || capture.body.Args["text"] != "Ready" || capture.body.Args["timeoutMs"] != float64(2500) { + t.Fatalf("wait = %#v", capture.body) + } +} + +func TestBrowserScreenshotWritesWithoutOverwrite(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + target := filepath.Join(t.TempDir(), "shot.png") + + out, errOut, err := executeCLI(t, deps, "browser", "screenshot", target) + if err != nil { + t.Fatalf("screenshot err=%v stderr=%s", err, errOut) + } + data, err := os.ReadFile(target) + if err != nil || string(data) != "png" || !strings.Contains(out, "10x20") { + t.Fatalf("screenshot data=%q err=%v out=%s", data, err, out) + } + if _, _, err := executeCLI(t, deps, "browser", "screenshot", target); err == nil || !strings.Contains(err.Error(), "refusing to overwrite") { + t.Fatalf("overwrite error = %v", err) + } +} + +func TestBrowserRequiresSessionAndValidWait(t *testing.T) { + t.Setenv("AO_SESSION_ID", "") + if _, _, err := executeCLI(t, Deps{}, "browser", "status"); ExitCode(err) != 2 { + t.Fatalf("status error = %v code=%d", err, ExitCode(err)) + } + t.Setenv("AO_SESSION_ID", "ao-1") + if _, _, err := executeCLI(t, Deps{}, "browser", "wait", "--text", "x", "--url", "y"); ExitCode(err) != 2 { + t.Fatalf("wait error = %v code=%d", err, ExitCode(err)) + } +} diff --git a/backend/internal/cli/root.go b/backend/internal/cli/root.go index d7a4487a55..a44b6b3490 100644 --- a/backend/internal/cli/root.go +++ b/backend/internal/cli/root.go @@ -188,6 +188,7 @@ func NewRootCommand(deps Deps) *cobra.Command { root.AddCommand(newSpawnCommand(ctx)) root.AddCommand(newSendCommand(ctx)) root.AddCommand(newPreviewCommand(ctx)) + root.AddCommand(newBrowserCommand(ctx)) root.AddCommand(newHooksCommand(ctx)) root.AddCommand(newLaunchCommand(ctx)) root.AddCommand(newPtyHostCommand()) diff --git a/backend/internal/daemon/daemon.go b/backend/internal/daemon/daemon.go index d2c25c6188..faaf27916c 100644 --- a/backend/internal/daemon/daemon.go +++ b/backend/internal/daemon/daemon.go @@ -14,6 +14,7 @@ import ( "time" "github.com/aoagents/agent-orchestrator/backend/internal/adapters/runtime/runtimeselect" + "github.com/aoagents/agent-orchestrator/backend/internal/browserruntime" "github.com/aoagents/agent-orchestrator/backend/internal/config" "github.com/aoagents/agent-orchestrator/backend/internal/daemon/supervisor" "github.com/aoagents/agent-orchestrator/backend/internal/domain" @@ -151,6 +152,7 @@ func Run() error { DefaultPort: mobilebridge.DefaultPort, } mc := &controllers.MobileController{Bridge: bs} + browserBroker := browserruntime.New(log) srv, err := httpd.NewWithDeps(cfg, log, termMgr, httpd.APIDeps{ Projects: projectsvc.NewWithDeps(projectsvc.Deps{Store: store, Sessions: sessionSvc, DefaultHarness: domain.AgentHarness(cfg.Agent), Telemetry: telemetrySink}), @@ -165,6 +167,7 @@ func Run() error { Activity: lcStack.LCM, Telemetry: telemetrySink, Mobile: mc, + Browser: browserBroker, }) if err != nil { stop() @@ -175,6 +178,16 @@ func Run() error { return err } previewDone := preview.NewPoller(store, sessionSvc, "http://"+srv.Addr().String(), preview.PollerConfig{Logger: log}).Start(ctx) + if ln, addr, err := browserruntime.Listen(cfg.RunFilePath); err != nil { + log.Warn("browser runtime: listener unavailable; agent browser control disabled", "err", err) + } else { + log.Info("browser runtime: listening", "addr", addr) + go func() { + if err := browserBroker.Serve(ctx, ln); err != nil { + log.Warn("browser runtime: serve stopped with error", "err", err) + } + }() + } // Late-bind: the LAN listener shares the exact loopback router instance so // the LAN surface and loopback surface never drift apart. diff --git a/backend/internal/httpd/api.go b/backend/internal/httpd/api.go index d49c9c8ca6..a6660f86cc 100644 --- a/backend/internal/httpd/api.go +++ b/backend/internal/httpd/api.go @@ -32,6 +32,7 @@ type APIDeps struct { Events cdcSubscriber Telemetry ports.EventSink Mobile *controllers.MobileController + Browser controllers.BrowserRuntime } // API owns one controller per resource and is the single Register call the @@ -45,6 +46,7 @@ type API struct { reviews *controllers.ReviewsController notifications *controllers.NotificationsController imports *controllers.ImportController + browser *controllers.BrowserController events *EventsController } @@ -68,6 +70,7 @@ func NewAPI(cfg config.Config, deps APIDeps) *API { reviews: &controllers.ReviewsController{Svc: deps.Reviews}, notifications: &controllers.NotificationsController{Svc: deps.Notifications, Stream: deps.NotificationStream}, imports: &controllers.ImportController{Svc: deps.Import}, + browser: &controllers.BrowserController{Runtime: deps.Browser, Sessions: deps.Sessions}, events: &EventsController{Source: deps.CDC, Live: deps.Events}, } } @@ -93,6 +96,7 @@ func (a *API) Register(root chi.Router) { a.reviews.Register(r) a.notifications.Register(r) a.imports.Register(r) + a.browser.Register(r) // Sibling REST controllers plug in here. }) // Long-lived streams intentionally bypass the REST timeout middleware. diff --git a/backend/internal/httpd/apispec/openapi.yaml b/backend/internal/httpd/apispec/openapi.yaml index 6e7d0fadfe..0f75f0c22a 100644 --- a/backend/internal/httpd/apispec/openapi.yaml +++ b/backend/internal/httpd/apispec/openapi.yaml @@ -97,6 +97,99 @@ paths: summary: Refresh the cached local agent adapter catalog tags: - agents + /api/v1/browser/commands: + post: + operationId: executeBrowserCommand + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BrowserCommandRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/BrowserCommandResponse' + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Bad Request + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Found + "409": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Conflict + "422": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Unprocessable Entity + "501": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Implemented + "503": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Service Unavailable + summary: Execute a target-scoped command in a session's desktop browser + tags: + - browser + /api/v1/browser/status: + get: + operationId: getBrowserStatus + parameters: + - description: AO session identifier. + in: query + name: sessionId + schema: + description: AO session identifier. + type: string + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/BrowserStatusResponse' + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Bad Request + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Found + "501": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Implemented + summary: Check whether the desktop browser runtime is connected for a session + tags: + - browser /api/v1/events: get: operationId: streamEvents @@ -1859,6 +1952,50 @@ components: - id - label type: object + BrowserCommandRequest: + properties: + action: + type: string + args: + additionalProperties: {} + type: object + sessionId: + type: string + required: + - sessionId + - action + type: object + BrowserCommandResponse: + properties: + action: + type: string + requestId: + type: string + result: {} + sessionId: + type: string + required: + - requestId + - sessionId + - action + - result + type: object + BrowserStatusResponse: + properties: + connected: + type: boolean + connectedAt: + format: date-time + type: string + sessionId: + type: string + transport: + type: string + required: + - sessionId + - connected + - transport + type: object CancelReviewResponse: properties: reviewerHandleId: @@ -3179,3 +3316,5 @@ tags: name: import - description: Connect Mobile LAN bridge control (loopback/desktop only) name: mobile +- description: Target-isolated desktop browser runtime (loopback only) + name: browser diff --git a/backend/internal/httpd/apispec/specgen/build.go b/backend/internal/httpd/apispec/specgen/build.go index cfe93382a6..76eb285cd6 100644 --- a/backend/internal/httpd/apispec/specgen/build.go +++ b/backend/internal/httpd/apispec/specgen/build.go @@ -73,6 +73,8 @@ func Build() ([]byte, error) { "Legacy AO project import (availability probe and run)"), *(&openapi31.Tag{Name: "mobile"}).WithDescription( "Connect Mobile LAN bridge control (loopback/desktop only)"), + *(&openapi31.Tag{Name: "browser"}).WithDescription( + "Target-isolated desktop browser runtime (loopback only)"), } for _, op := range operations() { @@ -149,6 +151,10 @@ var schemaNames = map[string]string{ "ControllersSessionResponse": "SessionResponse", "ControllersSessionPreviewResponse": "SessionPreviewResponse", "ControllersSetSessionPreviewRequest": "SetSessionPreviewRequest", + "ControllersBrowserStatusQuery": "BrowserStatusQuery", + "ControllersBrowserStatusResponse": "BrowserStatusResponse", + "ControllersBrowserCommandRequest": "BrowserCommandRequest", + "ControllersBrowserCommandResponse": "BrowserCommandResponse", "ControllersRenameSessionRequest": "RenameSessionRequest", "ControllersRenameSessionResponse": "RenameSessionResponse", "ControllersRestoreSessionResponse": "RestoreSessionResponse", @@ -305,9 +311,40 @@ func operations() []operation { ops = append(ops, notificationOperations()...) ops = append(ops, importOperations()...) ops = append(ops, mobileOperations()...) + ops = append(ops, browserOperations()...) return ops } +func browserOperations() []operation { + return []operation{ + { + method: http.MethodGet, path: "/api/v1/browser/status", id: "getBrowserStatus", tag: "browser", + summary: "Check whether the desktop browser runtime is connected for a session", + pathParams: []any{controllers.BrowserStatusQuery{}}, + resps: []respUnit{ + {http.StatusOK, controllers.BrowserStatusResponse{}}, + {http.StatusBadRequest, envelope.APIError{}}, + {http.StatusNotFound, envelope.APIError{}}, + {http.StatusNotImplemented, envelope.APIError{}}, + }, + }, + { + method: http.MethodPost, path: "/api/v1/browser/commands", id: "executeBrowserCommand", tag: "browser", + summary: "Execute a target-scoped command in a session's desktop browser", + reqBody: controllers.BrowserCommandRequest{}, + resps: []respUnit{ + {http.StatusOK, controllers.BrowserCommandResponse{}}, + {http.StatusBadRequest, envelope.APIError{}}, + {http.StatusNotFound, envelope.APIError{}}, + {http.StatusConflict, envelope.APIError{}}, + {http.StatusUnprocessableEntity, envelope.APIError{}}, + {http.StatusServiceUnavailable, envelope.APIError{}}, + {http.StatusNotImplemented, envelope.APIError{}}, + }, + }, + } +} + func agentOperations() []operation { return []operation{ { diff --git a/backend/internal/httpd/controllers/browser.go b/backend/internal/httpd/controllers/browser.go new file mode 100644 index 0000000000..26050e346c --- /dev/null +++ b/backend/internal/httpd/controllers/browser.go @@ -0,0 +1,130 @@ +package controllers + +import ( + "context" + "errors" + "net/http" + "strings" + + "github.com/go-chi/chi/v5" + + "github.com/aoagents/agent-orchestrator/backend/internal/browserruntime" + "github.com/aoagents/agent-orchestrator/backend/internal/domain" + "github.com/aoagents/agent-orchestrator/backend/internal/httpd/apispec" + "github.com/aoagents/agent-orchestrator/backend/internal/httpd/envelope" +) + +var browserActions = map[string]struct{}{ + "open": {}, + "snapshot": {}, + "click": {}, + "fill": {}, + "wait": {}, + "screenshot": {}, + "console": {}, + "errors": {}, +} + +type BrowserRuntime interface { + Status() browserruntime.Status + Execute(ctx context.Context, sessionID domain.SessionID, action string, args map[string]interface{}) (browserruntime.Result, error) +} + +type BrowserSessionReader interface { + Get(ctx context.Context, id domain.SessionID) (domain.Session, error) +} + +type BrowserController struct { + Runtime BrowserRuntime + Sessions BrowserSessionReader +} + +func (c *BrowserController) Register(r chi.Router) { + r.Get("/browser/status", c.status) + r.Post("/browser/commands", c.execute) +} + +func (c *BrowserController) status(w http.ResponseWriter, r *http.Request) { + if c.Runtime == nil || c.Sessions == nil { + apispec.NotImplemented(w, r, http.MethodGet, "/api/v1/browser/status") + return + } + sessionID := domain.SessionID(strings.TrimSpace(r.URL.Query().Get("sessionId"))) + if sessionID == "" { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "SESSION_ID_REQUIRED", "sessionId is required", nil) + return + } + if _, err := c.Sessions.Get(r.Context(), sessionID); err != nil { + envelope.WriteError(w, r, err) + return + } + status := c.Runtime.Status() + envelope.WriteJSON(w, http.StatusOK, BrowserStatusResponse{ + SessionID: sessionID, + Connected: status.Connected, + ConnectedAt: status.ConnectedAt, + Transport: "electron-webcontents-debugger", + }) +} + +func (c *BrowserController) execute(w http.ResponseWriter, r *http.Request) { + if c.Runtime == nil || c.Sessions == nil { + apispec.NotImplemented(w, r, http.MethodPost, "/api/v1/browser/commands") + return + } + var in BrowserCommandRequest + if err := decodeJSON(r, &in); err != nil { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "INVALID_JSON", "Invalid JSON body", nil) + return + } + in.Action = strings.ToLower(strings.TrimSpace(in.Action)) + if in.SessionID == "" { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "SESSION_ID_REQUIRED", "sessionId is required", nil) + return + } + if _, ok := browserActions[in.Action]; !ok { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "BROWSER_ACTION_UNSUPPORTED", "Unsupported browser action", nil) + return + } + if _, err := c.Sessions.Get(r.Context(), in.SessionID); err != nil { + envelope.WriteError(w, r, err) + return + } + result, err := c.Runtime.Execute(r.Context(), in.SessionID, in.Action, in.Args) + if err != nil { + writeBrowserError(w, r, err) + return + } + envelope.WriteJSON(w, http.StatusOK, BrowserCommandResponse{ + RequestID: result.RequestID, + SessionID: in.SessionID, + Action: in.Action, + Result: result.Value, + }) +} + +func writeBrowserError(w http.ResponseWriter, r *http.Request, err error) { + if errors.Is(err, browserruntime.ErrUnavailable) { + envelope.WriteAPIError(w, r, http.StatusServiceUnavailable, "unavailable", "BROWSER_RUNTIME_UNAVAILABLE", "Desktop browser runtime is not connected", nil) + return + } + var commandErr browserruntime.CommandError + if errors.As(err, &commandErr) { + status := http.StatusUnprocessableEntity + typeName := "unprocessable" + switch commandErr.Code { + case "INVALID_ARGUMENT", "URL_REQUIRED", "REFERENCE_REQUIRED": + status = http.StatusBadRequest + typeName = "bad_request" + case "STALE_REFERENCE": + status = http.StatusConflict + typeName = "conflict" + case "BROWSER_TARGET_UNAVAILABLE": + status = http.StatusServiceUnavailable + typeName = "unavailable" + } + envelope.WriteAPIError(w, r, status, typeName, commandErr.Code, commandErr.Message, nil) + return + } + envelope.WriteError(w, r, err) +} diff --git a/backend/internal/httpd/controllers/browser_test.go b/backend/internal/httpd/controllers/browser_test.go new file mode 100644 index 0000000000..f9e40f46cd --- /dev/null +++ b/backend/internal/httpd/controllers/browser_test.go @@ -0,0 +1,98 @@ +package controllers_test + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/aoagents/agent-orchestrator/backend/internal/browserruntime" + "github.com/aoagents/agent-orchestrator/backend/internal/config" + "github.com/aoagents/agent-orchestrator/backend/internal/domain" + "github.com/aoagents/agent-orchestrator/backend/internal/httpd" +) + +type fakeBrowserRuntime struct { + status browserruntime.Status + action string + args map[string]interface{} + err error +} + +func (f *fakeBrowserRuntime) Status() browserruntime.Status { return f.status } + +func (f *fakeBrowserRuntime) Execute( + _ context.Context, + _ domain.SessionID, + action string, + args map[string]interface{}, +) (browserruntime.Result, error) { + f.action, f.args = action, args + if f.err != nil { + return browserruntime.Result{}, f.err + } + return browserruntime.Result{RequestID: "request-1", Value: map[string]interface{}{"text": "button Save [ref=e1]"}}, nil +} + +func browserServer(t *testing.T, runtime *fakeBrowserRuntime) *httptest.Server { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + srv := httptest.NewServer(httpd.NewRouterWithControl(config.Config{}, log, nil, httpd.APIDeps{ + Sessions: newFakeSessionService(), + Browser: runtime, + }, httpd.ControlDeps{})) + t.Cleanup(srv.Close) + return srv +} + +func TestBrowserStatusAndSnapshot(t *testing.T) { + connectedAt := time.Now().UTC().Truncate(time.Second) + runtime := &fakeBrowserRuntime{status: browserruntime.Status{Connected: true, ConnectedAt: connectedAt}} + srv := browserServer(t, runtime) + + body, status, _ := doRequest(t, srv, http.MethodGet, "/api/v1/browser/status?sessionId=ao-1", "") + if status != http.StatusOK || !containsAll(body, `"connected":true`, `"transport":"electron-webcontents-debugger"`) { + t.Fatalf("status = %d body=%s", status, body) + } + body, status, _ = doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"snapshot","args":{"interactive":true}}`) + if status != http.StatusOK || !containsAll(body, `"requestId":"request-1"`, `"button Save [ref=e1]"`) { + t.Fatalf("command = %d body=%s", status, body) + } + if runtime.action != "snapshot" || runtime.args["interactive"] != true { + t.Fatalf("runtime command = %q %#v", runtime.action, runtime.args) + } +} + +func TestBrowserCommandValidationAndErrors(t *testing.T) { + runtime := &fakeBrowserRuntime{} + srv := browserServer(t, runtime) + + body, status, _ := doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"eval"}`) + if status != http.StatusBadRequest || !containsAll(body, `"code":"BROWSER_ACTION_UNSUPPORTED"`) { + t.Fatalf("unsupported = %d body=%s", status, body) + } + runtime.err = browserruntime.ErrUnavailable + body, status, _ = doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"snapshot"}`) + if status != http.StatusServiceUnavailable || !containsAll(body, `"code":"BROWSER_RUNTIME_UNAVAILABLE"`) { + t.Fatalf("unavailable = %d body=%s", status, body) + } + runtime.err = browserruntime.CommandError{Code: "STALE_REFERENCE", Message: "snapshot again"} + body, status, _ = doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"click"}`) + if status != http.StatusConflict || !containsAll(body, `"code":"STALE_REFERENCE"`) { + t.Fatalf("stale = %d body=%s", status, body) + } +} + +func containsAll(body []byte, parts ...string) bool { + value := string(body) + for _, part := range parts { + if !strings.Contains(value, part) { + return false + } + } + return true +} diff --git a/backend/internal/httpd/controllers/dto.go b/backend/internal/httpd/controllers/dto.go index f725453fe4..473168e1f2 100644 --- a/backend/internal/httpd/controllers/dto.go +++ b/backend/internal/httpd/controllers/dto.go @@ -220,6 +220,37 @@ type SetSessionPreviewRequest struct { URL string `json:"url,omitempty" description:"Preview target URL. When empty, the daemon autodetects a static entry point in the session workspace."` } +// BrowserStatusQuery selects the session whose logical browser is inspected. +type BrowserStatusQuery struct { + SessionID domain.SessionID `query:"sessionId" description:"AO session identifier."` +} + +// BrowserStatusResponse reports whether the desktop-owned browser transport is +// ready. A connected runtime can create the session target while its panel is +// hidden; panel visibility is intentionally not part of this state. +type BrowserStatusResponse struct { + SessionID domain.SessionID `json:"sessionId"` + Connected bool `json:"connected"` + ConnectedAt time.Time `json:"connectedAt,omitempty"` + Transport string `json:"transport"` +} + +// BrowserCommandRequest is the stable daemon-facing command envelope. Action +// arguments remain action-specific JSON so new target-scoped operations do not +// require a new transport or Electron IPC surface. +type BrowserCommandRequest struct { + SessionID domain.SessionID `json:"sessionId"` + Action string `json:"action"` + Args map[string]interface{} `json:"args,omitempty"` +} + +type BrowserCommandResponse struct { + RequestID string `json:"requestId"` + SessionID domain.SessionID `json:"sessionId"` + Action string `json:"action"` + Result interface{} `json:"result"` +} + // RenameSessionResponse is the body of PATCH /api/v1/sessions/{sessionId}. type RenameSessionResponse struct { OK bool `json:"ok"` diff --git a/backend/internal/httpd/lan_listener.go b/backend/internal/httpd/lan_listener.go index 68dc0ce04a..1afe95e981 100644 --- a/backend/internal/httpd/lan_listener.go +++ b/backend/internal/httpd/lan_listener.go @@ -55,6 +55,7 @@ var lanControlBlockedPrefixes = []string{ "/shutdown", "/internal/", "/api/v1/mobile", + "/api/v1/browser", } // lanControlBlock returns 404 for any request whose path is, or is nested diff --git a/backend/internal/httpd/lan_listener_test.go b/backend/internal/httpd/lan_listener_test.go index c408c9bb79..8957227b3e 100644 --- a/backend/internal/httpd/lan_listener_test.go +++ b/backend/internal/httpd/lan_listener_test.go @@ -44,7 +44,7 @@ func TestLANManagerAuthGatesSharedHandler(t *testing.T) { } // TestLANManagerBlocksLoopbackOnlyControlRoutes proves the LAN listener never -// serves /shutdown, /internal/*, or /api/v1/mobile* — even when the request +// serves /shutdown, /internal/*, /api/v1/mobile*, or /api/v1/browser* — even when the request // carries a spoofed Host: 127.0.0.1 and valid LAN auth, since gating on Host // alone (localControlRequest) is what let a LAN client reach these routes. func TestLANManagerBlocksLoopbackOnlyControlRoutes(t *testing.T) { @@ -64,6 +64,7 @@ func TestLANManagerBlocksLoopbackOnlyControlRoutes(t *testing.T) { "/shutdown", "/internal/telemetry/cli-invoked", "/api/v1/mobile/status", + "/api/v1/browser/status", } for _, path := range blocked { req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d%s", port, path), nil) diff --git a/backend/internal/session_manager/manager.go b/backend/internal/session_manager/manager.go index 71280b6c35..de1af51921 100644 --- a/backend/internal/session_manager/manager.go +++ b/backend/internal/session_manager/manager.go @@ -1956,8 +1956,13 @@ func (m *Manager) aoSkillPointer() string { dir := skillassets.Dir(m.dataDir) skillFile := filepath.Join(dir, "SKILL.md") commandsGlob := filepath.Join(dir, "commands", "*.md") + browserFile := filepath.Join(dir, "commands", "browser.md") return "\n\n" + "## Using the ao CLI\n\n" + - "When you need to use the `ao` CLI, read `" + skillFile + "` first (and the relevant `" + commandsGlob + "`) for the full command catalog, flags, and examples." + "When you need to use the `ao` CLI, read `" + skillFile + "` first (and the relevant `" + commandsGlob + "`) for the full command catalog, flags, and examples.\n\n" + + "## AO desktop Browser panel\n\n" + + "When the user asks you to inspect, test, click, or type in the page shown in AO's desktop Browser panel, read `" + browserFile + "` and use `ao browser` from this AO session. " + + "Do not use Codex/host in-app browser connectors, `agent.browsers.get(\"iab\")`, or a browser MCP for the AO Browser panel: those are separate browser runtimes and cannot see or control AO's session-owned page. " + + "`ao browser` deliberately operates the same live page the user sees in that panel." } func (m *Manager) workspaceProjectPrompt(ctx context.Context, kind domain.SessionKind, projectID domain.ProjectID) (string, error) { diff --git a/backend/internal/session_manager/manager_test.go b/backend/internal/session_manager/manager_test.go index 51e2d93f7c..c51ebeaf69 100644 --- a/backend/internal/session_manager/manager_test.go +++ b/backend/internal/session_manager/manager_test.go @@ -2093,7 +2093,10 @@ func TestSpawnOrchestrator_UsesCoordinatorPrompt(t *testing.T) { "`ao session ls --project mer`", "`ao session get `", "Delegate implementation, fixes, tests, and PR ownership to worker sessions", - "skills/using-ao/SKILL.md", + filepath.Join("skills", "using-ao", "SKILL.md"), + "AO desktop Browser panel", + "agent.browsers.get(\"iab\")", + "same live page the user sees", } { if !strings.Contains(systemPrompt, want) { t.Fatalf("system prompt missing %q:\n%s", want, systemPrompt) @@ -2240,9 +2243,12 @@ func TestSystemPrompt_AppendsConfidentialityGuard(t *testing.T) { if !strings.Contains(sp, "role boundaries, delegation policy, CI/review follow-up expectations, PR/MR workflow when applicable, and privacy rules") { t.Fatalf("%s: system prompt missing generic behavior categories:\n%s", tc.name, sp) } - if !strings.Contains(sp, "skills/using-ao/SKILL.md") { + if !strings.Contains(sp, filepath.Join("skills", "using-ao", "SKILL.md")) { t.Fatalf("%s: system prompt missing using-ao skill pointer:\n%s", tc.name, sp) } + if !strings.Contains(sp, "AO desktop Browser panel") || !strings.Contains(sp, "agent.browsers.get(\"iab\")") { + t.Fatalf("%s: system prompt missing AO browser routing guidance:\n%s", tc.name, sp) + } }) } } diff --git a/backend/internal/skillassets/skillassets_test.go b/backend/internal/skillassets/skillassets_test.go index 4757e22f18..9209386ba1 100644 --- a/backend/internal/skillassets/skillassets_test.go +++ b/backend/internal/skillassets/skillassets_test.go @@ -26,6 +26,9 @@ func TestInstall_WritesSkillAndIsIdempotent(t *testing.T) { if _, err := os.Stat(filepath.Join(Dir(dataDir), "commands", "spawn.md")); err != nil { t.Fatalf("commands/spawn.md missing: %v", err) } + if _, err := os.Stat(filepath.Join(Dir(dataDir), "commands", "browser.md")); err != nil { + t.Fatalf("commands/browser.md missing: %v", err) + } // A stale file inside the skill dir must not survive a reinstall (clobber). stale := filepath.Join(Dir(dataDir), "stale.md") diff --git a/backend/internal/skillassets/using-ao/SKILL.md b/backend/internal/skillassets/using-ao/SKILL.md index 7be54f5ba9..88600ef31d 100644 --- a/backend/internal/skillassets/using-ao/SKILL.md +++ b/backend/internal/skillassets/using-ao/SKILL.md @@ -1,7 +1,7 @@ --- name: using-ao -description: Catalog of the AO (Agent Orchestrator) `ao` CLI: spawning workers, managing sessions and projects, sending messages, previewing pages, and daemon control. Use when using the ao CLI, spawning workers, or managing AO sessions in an AO workspace. -trigger: Using the ao CLI in an AO workspace: spawning workers, managing sessions/projects, sending messages, previewing pages. +description: Catalog of the AO (Agent Orchestrator) `ao` CLI: spawning workers, managing sessions and projects, sending messages, controlling the shared browser, previewing pages, and daemon control. Use when using the ao CLI, spawning workers, or managing AO sessions in an AO workspace. +trigger: Using the ao CLI in an AO workspace: spawning workers, managing sessions/projects, sending messages, controlling or previewing pages. --- # AO CLI Catalog @@ -17,6 +17,7 @@ trigger: Using the ao CLI in an AO workspace: spawning workers, managing session | `review` | Submit a reviewer result for a worker's PR | Completing a code review loop | [commands/review.md](commands/review.md) | | `send` | Send a message to a running agent session | Correcting or directing a live agent | [commands/send.md](commands/send.md) | | `preview` | Open a URL in the desktop browser panel | Demoing a local server or file from inside a session | [commands/preview.md](commands/preview.md) | +| `browser` | Inspect and control the session's shared live browser | Verifying a web app through snapshots, interactions, waits, screenshots, console, and errors | [commands/browser.md](commands/browser.md) | | `start` | Fetch (if needed) and open the AO desktop app | Launching the app | [commands/start.md](commands/start.md) | | `stop` | Stop the AO daemon | Shutting down AO | [commands/stop.md](commands/stop.md) | | `status` | Show daemon status | Verifying the daemon is up and healthy | [commands/status.md](commands/status.md) | @@ -32,5 +33,6 @@ trigger: Using the ao CLI in an AO workspace: spawning workers, managing session - Session and project ids are shown by `ao session ls` and `ao project ls`. - `--agent` is an alias for `--harness` on `ao spawn`. - Every command accepts `-h / --help` for the full flag list. +- For frontend work, prefer `ao browser`: check status, open the app, snapshot, interact, wait for the update, snapshot again, and inspect `ao browser errors` before completing. See [references.md](references.md) for natural-language-to-command mappings. diff --git a/backend/internal/skillassets/using-ao/commands/browser.md b/backend/internal/skillassets/using-ao/commands/browser.md new file mode 100644 index 0000000000..3507130b46 --- /dev/null +++ b/backend/internal/skillassets/using-ao/commands/browser.md @@ -0,0 +1,40 @@ +# ao browser + +Inspect and control the current AO session's target-isolated browser. The desktop app must be open. The agent and user share the same live page, cookies, navigation state, and `WebContentsView`; the runtime remains usable while the Browser panel is hidden. + +`AO_SESSION_ID` selects the target, so run these commands from inside an AO worker session. + +This is the automation interface for AO's visible desktop Browser panel. Do not use Codex/host in-app browser connectors, `agent.browsers.get("iab")`, or a browser MCP for this panel: those belong to separate browser runtimes and will not discover or update AO's session-owned page. + +## Core workflow + +```bash +ao browser status +ao browser open http://localhost:5173 +ao browser snapshot +ao browser click e1 +ao browser fill e2 "hello" +ao browser wait --text "Saved" +ao browser snapshot +ao browser errors +``` + +Element references such as `e1` are short-lived. After navigation or a substantial DOM replacement, take another snapshot. A stale reference fails explicitly and never falls through to another session or page. + +## Commands + +```text +ao browser status [--json] +ao browser open [--json] +ao browser snapshot [--interactive] [--json] +ao browser click [--json] +ao browser fill [--json] +ao browser wait (--text | --selector | --url | --ms ) [--timeout ] [--json] +ao browser screenshot [path] [--json] +ao browser console [--json] +ao browser errors [--json] +``` + +Without `--json`, `screenshot` writes a PNG and refuses to overwrite an existing file. With `--json`, it returns the structured response including base64 image data. + +`ao preview` remains available for the passive URL/static-file workflow. Use `ao browser` when the agent needs to inspect or verify the page. diff --git a/backend/internal/skillassets/using-ao/references.md b/backend/internal/skillassets/using-ao/references.md index ef9fc5507b..fe7515cd44 100644 --- a/backend/internal/skillassets/using-ao/references.md +++ b/backend/internal/skillassets/using-ao/references.md @@ -5,6 +5,10 @@ Natural-language-to-command mappings for common AO tasks. | You want to... | Command | |---|---| | Show me this webpage / open this page | `ao preview ""` | +| Inspect and verify this webpage as the agent | `ao browser open ""`, then `ao browser snapshot` | +| Click or fill a page element | `ao browser snapshot`, then `ao browser click ` or `ao browser fill ""` | +| Check frontend runtime failures | `ao browser errors` and `ao browser console` | +| Capture the page | `ao browser screenshot [path]` | | Spawn a worker on issue N | `ao spawn --project

--issue N --name "<=20 chars>" --prompt "..."` | | Message a running agent | `ao send --session --message "..."` | | Kill a session | `ao session kill ` | diff --git a/docs/STATUS.md b/docs/STATUS.md index 7281d2f987..07a99127d8 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -61,6 +61,11 @@ surface (`npm run sqlc`, `npm run api`). ### Frontend (Electron + React) - Electron + React 19 + TanStack Router/Query + Tailwind + shadcn primitives. +- Target-isolated per-session browser-control spike: a dedicated local + daemon↔Electron bridge drives only the selected session's `WebContentsView` + through Electron's bound debugger transport. `ao browser` supports open, + compact accessibility snapshots and refs, click/fill, waits, screenshots, + console messages, and page errors while the Browser panel is hidden. - Real daemon wiring via the generated `openapi-fetch` typed client (`src/api/schema.ts`); mock data only in `VITE_NO_ELECTRON` web-preview mode. - Electron main handles daemon discovery, launch, and status reporting. diff --git a/docs/architecture.md b/docs/architecture.md index 91fb1ffe1f..0dc4f59ab8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -15,6 +15,7 @@ Agent Orchestrator is a long-running Go daemon that supervises multiple parallel - [Observation Loops](#observation-loops) - [HTTP Layer](#http-layer) - [Terminal Multiplexing](#terminal-multiplexing) +- [Browser Runtime Bridge](#browser-runtime-bridge) --- @@ -832,6 +833,19 @@ sequenceDiagram Mux->>Runtime: Close PTY ``` +## Browser Runtime Bridge + +Browser automation uses a dedicated local socket (`browser.sock` on Unix, +`ao-browser[-dev]` named pipe on Windows) between the daemon and Electron. The +daemon owns command authorization/correlation; Electron owns the actual browser +targets. Commands never use the supervisor liveness socket and never enable an +unauthenticated remote-debugging port. + +Electron attaches its debugger directly to the selected session's +`WebContentsView`, so the protocol transport cannot enumerate or attach to the +AO renderer or a different session. The loopback `/api/v1/browser` surface is +blocked entirely on the opt-in LAN listener. + --- ## Load-Bearing Rules diff --git a/docs/cli/README.md b/docs/cli/README.md index a94952ff65..93a270ca2d 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -56,6 +56,7 @@ Every product command resolves to a daemon HTTP route. Run `ao | `ao orchestrator ls` | `GET /api/v1/orchestrators` | | `ao send` | `POST /api/v1/sessions/{id}/send` | | `ao preview [url]` | `POST /api/v1/sessions/{id}/preview` | +| `ao browser ...` | `GET /api/v1/browser/status`, `POST /api/v1/browser/commands` | | `ao hooks ` | `POST /api/v1/sessions/{id}/activity` (hidden) | `ao agent ls` prints the daemon-supported agent catalog with local install/auth @@ -80,6 +81,14 @@ spawn remains the authoritative runtime validation point. Use autodetects an `index.html` in the session workspace; with a URL argument it opens that URL verbatim (`file://`, `http`, `https`). +`ao browser` also resolves its target from `AO_SESSION_ID`, but controls the +session-owned live Electron browser rather than only setting its preview URL. +The initial target-isolated command set is `status`, `open`, `snapshot`, +`click`, `fill`, `wait`, `screenshot`, `console`, and `errors`. The AO desktop +app must be open because Electron owns the `WebContentsView`. References from a +snapshot are invalidated after navigation or DOM replacement; take another +snapshot when a command reports `STALE_REFERENCE`. + `go run .` in `backend/` remains a compatibility wrapper around the daemon. PR and review actions (merge, resolve-comments, review execute/send) are diff --git a/frontend/src/api/schema.ts b/frontend/src/api/schema.ts index 3a44b570a0..944b01359f 100644 --- a/frontend/src/api/schema.ts +++ b/frontend/src/api/schema.ts @@ -55,6 +55,40 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/browser/commands": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Execute a target-scoped command in a session's desktop browser */ + post: operations["executeBrowserCommand"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/browser/status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Check whether the desktop browser runtime is connected for a session */ + get: operations["getBrowserStatus"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/events": { parameters: { query?: never; @@ -708,6 +742,26 @@ export interface components { id: string; label: string; }; + BrowserCommandRequest: { + action: string; + args?: { + [key: string]: unknown; + }; + sessionId: string; + }; + BrowserCommandResponse: { + action: string; + requestId: string; + result: unknown; + sessionId: string; + }; + BrowserStatusResponse: { + connected: boolean; + /** Format: date-time */ + connectedAt?: string; + sessionId: string; + transport: string; + }; CancelReviewResponse: { reviewerHandleId: string; reviews: components["schemas"]["PRReviewState"][]; @@ -1334,6 +1388,134 @@ export interface operations { }; }; }; + executeBrowserCommand: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BrowserCommandRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BrowserCommandResponse"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Conflict */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Implemented */ + 501: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Service Unavailable */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + }; + }; + getBrowserStatus: { + parameters: { + query?: { + /** @description AO session identifier. */ + sessionId?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BrowserStatusResponse"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Implemented */ + 501: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + }; + }; streamEvents: { parameters: { query?: { diff --git a/frontend/src/main.ts b/frontend/src/main.ts index 6d152ee2c7..218dfb5969 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -55,6 +55,7 @@ import { DEFAULT_POSTHOG_HOST, DEFAULT_POSTHOG_PROJECT_KEY } from "./shared/post import { buildTelemetryBootstrap } from "./shared/telemetry"; import { createBrowserViewHost, type BrowserViewHost } from "./main/browser-view-host"; import { connectSupervisor, type SupervisorLinkHandle } from "./main/supervisor-link"; +import { connectBrowserRuntime, type BrowserRuntimeLinkHandle } from "./main/browser-runtime-link"; import { shouldLinkOnAttach } from "./main/daemon-owner"; import { readMigrationState, updateMigration, writeAppStateMarker, type MigrationState } from "./main/app-state"; import { isAllowedAppExternalURL, openAllowedAppExternalURL } from "./main/external-open"; @@ -101,6 +102,7 @@ let daemonStartPromise: Promise | null = null; let daemonStartEpoch = 0; let daemonStatus: DaemonStatus = { state: "stopped" }; let browserViewHost: BrowserViewHost | null = null; +let browserRuntimeLink: BrowserRuntimeLinkHandle | null = null; // Held for the app lifetime. Dropping it (on any exit) triggers daemon self-stop. let supervisorLink: SupervisorLinkHandle | null = null; @@ -237,6 +239,9 @@ function applyRuntimeAppIcon(): void { function setDaemonStatus(nextStatus: DaemonStatus): void { daemonStatus = nextStatus; mainWindow?.webContents.send("daemon:status", daemonStatus); + if (nextStatus.state === "ready" && browserViewHost) { + establishBrowserRuntimeLink(); + } } // Role-based menu installed on Windows where the native menu bar is hidden. The @@ -358,6 +363,7 @@ function createWindow(): void { rendererOrigin: RENDERER_ORIGIN, isMac, }); + if (daemonStatus.state === "ready") establishBrowserRuntimeLink(); void mainWindow.loadURL(rendererUrl()); @@ -368,6 +374,8 @@ function createWindow(): void { } mainWindow.on("closed", () => { + browserRuntimeLink?.dispose(); + browserRuntimeLink = null; browserViewHost?.dispose(); browserViewHost = null; mainWindow = null; @@ -568,6 +576,40 @@ function supervisorPipeFromRunFile(rfp: string | null): string { return "\\\\.\\pipe\\ao-supervise-" + dir.replace(/[^a-zA-Z0-9-]/g, "-"); } +function browserRuntimePipeFromRunFile(rfp: string | null): string { + if (!rfp) return "\\\\.\\pipe\\ao-browser"; + const dir = path.basename(path.dirname(rfp)); + if (dir === ".ao" || dir === "." || dir === "") return "\\\\.\\pipe\\ao-browser"; + return "\\\\.\\pipe\\ao-browser-" + dir.replace(/[^a-zA-Z0-9-]/g, "-"); +} + +function establishBrowserRuntimeLink(): void { + if (!browserViewHost || browserRuntimeLink) return; + const rfp = runFilePath(); + const address = + process.platform === "win32" + ? browserRuntimePipeFromRunFile(rfp) + : rfp + ? path.join(path.dirname(rfp), "browser.sock") + : null; + if (!address) { + console.warn("AO: browser runtime link skipped; run-file path unavailable"); + return; + } + browserRuntimeLink = connectBrowserRuntime(address, { + execute: (command) => { + const host = browserViewHost; + if (!host) { + throw Object.assign(new Error("Browser target owner is unavailable"), { + code: "BROWSER_TARGET_UNAVAILABLE", + }); + } + return host.execute(command.sessionId, command.action, command.args); + }, + log: (message) => console.log(`AO: ${message}`), + }); +} + function establishSupervisorLink(): void { const rfp = runFilePath(); const addr = @@ -965,6 +1007,8 @@ function stopDaemon(): DaemonStatus { // A later daemon:start re-establishes the link via reportBoundPort. supervisorLink?.dispose(); supervisorLink = null; + browserRuntimeLink?.dispose(); + browserRuntimeLink = null; killDaemon(daemonProcess); setDaemonStatus({ state: "stopped" }); return daemonStatus; @@ -1412,6 +1456,8 @@ app.whenReady().then(async () => { // The supervisorLink fd is NOT explicitly closed on quit; the OS closes it when // the process exits for any reason (Cmd+Q, crash, SIGKILL). Sessions survive. app.on("before-quit", () => { + browserRuntimeLink?.dispose(); + browserRuntimeLink = null; browserViewHost?.dispose(); browserViewHost = null; }); diff --git a/frontend/src/main/browser-runtime-link.test.ts b/frontend/src/main/browser-runtime-link.test.ts new file mode 100644 index 0000000000..d2eb1801ac --- /dev/null +++ b/frontend/src/main/browser-runtime-link.test.ts @@ -0,0 +1,97 @@ +import net from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { connectBrowserRuntime, type BrowserRuntimeLinkHandle } from "./browser-runtime-link"; + +const handles: BrowserRuntimeLinkHandle[] = []; +const servers: net.Server[] = []; + +afterEach(async () => { + handles.splice(0).forEach((handle) => handle.dispose()); + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ), + ); +}); + +describe("browser runtime link", () => { + it("handshakes and correlates a command result", async () => { + const execute = vi.fn(async () => ({ text: "button Save [ref=e1]" })); + let serverSocket: net.Socket | null = null; + let inbound = ""; + const messages: unknown[] = []; + const server = net.createServer((socket) => { + serverSocket = socket; + socket.on("data", (chunk) => { + inbound += chunk.toString("utf8"); + for (;;) { + const newline = inbound.indexOf("\n"); + if (newline < 0) return; + messages.push(JSON.parse(inbound.slice(0, newline))); + inbound = inbound.slice(newline + 1); + } + }); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address() as net.AddressInfo; + const handle = connectBrowserRuntime({ host: address.address, port: address.port }, { execute }); + handles.push(handle); + await vi.waitFor(() => expect(handle.connected).toBe(true)); + await vi.waitFor(() => expect(messages).toContainEqual({ type: "hello", version: 1 })); + + serverSocket!.write( + `${JSON.stringify({ type: "command", requestId: "r1", sessionId: "s1", action: "snapshot", args: {} })}\n`, + ); + + await vi.waitFor(() => expect(execute).toHaveBeenCalledWith(expect.objectContaining({ requestId: "r1" }))); + await vi.waitFor(() => + expect(messages).toContainEqual({ + type: "result", + requestId: "r1", + ok: true, + result: { text: "button Save [ref=e1]" }, + }), + ); + }); + + it("returns structured command errors", async () => { + let serverSocket: net.Socket | null = null; + let inbound = ""; + const messages: unknown[] = []; + const server = net.createServer((socket) => { + serverSocket = socket; + socket.on("data", (chunk) => { + inbound += chunk.toString("utf8"); + const lines = inbound.split("\n"); + inbound = lines.pop() ?? ""; + for (const line of lines) if (line) messages.push(JSON.parse(line)); + }); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address() as net.AddressInfo; + const handle = connectBrowserRuntime( + { host: address.address, port: address.port }, + { + execute: async () => { + throw { code: "STALE_REFERENCE", message: "snapshot again" }; + }, + }, + ); + handles.push(handle); + await vi.waitFor(() => expect(handle.connected).toBe(true)); + serverSocket!.write(`${JSON.stringify({ type: "command", requestId: "r2", sessionId: "s1", action: "click" })}\n`); + await vi.waitFor(() => + expect(messages).toContainEqual({ + type: "result", + requestId: "r2", + ok: false, + error: { code: "STALE_REFERENCE", message: "snapshot again" }, + }), + ); + }); +}); diff --git a/frontend/src/main/browser-runtime-link.ts b/frontend/src/main/browser-runtime-link.ts new file mode 100644 index 0000000000..feb766c4ce --- /dev/null +++ b/frontend/src/main/browser-runtime-link.ts @@ -0,0 +1,171 @@ +import net from "node:net"; + +const PROTOCOL_VERSION = 1; +const BACKOFF_INIT_MS = 200; +const BACKOFF_MAX_MS = 2_000; +const MAX_COMMAND_BYTES = 1 << 20; + +export type BrowserRuntimeCommand = { + type: "command"; + requestId: string; + sessionId: string; + action: string; + args?: Record; +}; + +export type BrowserRuntimeCommandError = { + code: string; + message: string; +}; + +export interface BrowserRuntimeLinkHandle { + readonly connected: boolean; + dispose(): void; +} + +type BrowserRuntimeLinkOptions = { + execute: (command: BrowserRuntimeCommand) => Promise; + log?: (message: string) => void; +}; + +export function connectBrowserRuntime( + address: string | net.TcpNetConnectOpts, + options: BrowserRuntimeLinkOptions, +): BrowserRuntimeLinkHandle { + const log = options.log ?? (() => undefined); + let disposed = false; + let connected = false; + let socket: net.Socket | null = null; + let retryTimer: ReturnType | null = null; + let backoff = BACKOFF_INIT_MS; + let buffer = ""; + let commandChain = Promise.resolve(); + + const clearRetry = () => { + if (retryTimer !== null) { + clearTimeout(retryTimer); + retryTimer = null; + } + }; + + const destroySocket = () => { + if (!socket) return; + socket.removeAllListeners(); + socket.destroy(); + socket = null; + }; + + const send = (message: unknown) => { + if (!socket || socket.destroyed) return; + socket.write(`${JSON.stringify(message)}\n`); + }; + + const respond = async (command: BrowserRuntimeCommand) => { + try { + const result = await options.execute(command); + send({ type: "result", requestId: command.requestId, ok: true, result }); + } catch (error) { + const normalized = normalizeCommandError(error); + send({ type: "result", requestId: command.requestId, ok: false, error: normalized }); + } + }; + + const consumeLine = (line: string) => { + if (!line.trim()) return; + let command: BrowserRuntimeCommand; + try { + command = JSON.parse(line) as BrowserRuntimeCommand; + } catch { + return; + } + if ( + command.type !== "command" || + typeof command.requestId !== "string" || + typeof command.sessionId !== "string" || + typeof command.action !== "string" + ) { + return; + } + commandChain = commandChain.then(() => respond(command)); + }; + + const consume = (chunk: Buffer) => { + buffer += chunk.toString("utf8"); + if (Buffer.byteLength(buffer, "utf8") > MAX_COMMAND_BYTES) { + log("browser-runtime-link: oversized command frame; reconnecting"); + socket?.destroy(); + return; + } + for (;;) { + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + consumeLine(line); + } + }; + + const scheduleReconnect = () => { + if (disposed) return; + clearRetry(); + const delay = backoff; + backoff = Math.min(backoff * 2, BACKOFF_MAX_MS); + retryTimer = setTimeout(connect, delay); + }; + + function connect() { + if (disposed) return; + destroySocket(); + buffer = ""; + const next = typeof address === "string" ? net.connect(address) : net.connect(address); + socket = next; + next.on("connect", () => { + if (disposed) { + next.destroy(); + return; + } + connected = true; + backoff = BACKOFF_INIT_MS; + send({ type: "hello", version: PROTOCOL_VERSION }); + log("browser-runtime-link: connected"); + }); + next.on("data", consume); + next.on("error", (error) => log(`browser-runtime-link: error: ${error.message}`)); + next.on("close", () => { + connected = false; + if (!disposed) scheduleReconnect(); + }); + } + + connect(); + return { + get connected() { + return connected; + }, + dispose() { + disposed = true; + connected = false; + clearRetry(); + destroySocket(); + }, + }; +} + +function normalizeCommandError(error: unknown): BrowserRuntimeCommandError { + if (isCommandError(error)) { + return { code: error.code, message: error.message }; + } + return { + code: "BROWSER_COMMAND_FAILED", + message: error instanceof Error ? error.message : "Browser command failed", + }; +} + +function isCommandError(error: unknown): error is BrowserRuntimeCommandError { + return Boolean( + error && + typeof error === "object" && + typeof (error as BrowserRuntimeCommandError).code === "string" && + typeof (error as BrowserRuntimeCommandError).message === "string", + ); +} diff --git a/frontend/src/main/browser-view-host.test.ts b/frontend/src/main/browser-view-host.test.ts index f32d60d9e8..f0c3f30e07 100644 --- a/frontend/src/main/browser-view-host.test.ts +++ b/frontend/src/main/browser-view-host.test.ts @@ -18,6 +18,14 @@ function setupHost() { let currentURL = ""; let displayHandler: DisplayHandler | null = null; const webContentsListeners = new Map void>(); + const debuggerListeners = new Map void>(); + let debuggerAttached = false; + const debuggerSendCommand = vi.fn(async (method: string): Promise => { + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.resolveNode") return { object: { objectId: "object-1" } }; + if (method === "Runtime.evaluate") return { result: { value: true } }; + return {}; + }); const webContents = { id: 99, mainFrame: { frameToken: "preview-frame" }, @@ -26,7 +34,20 @@ function setupHost() { capturePage: vi.fn(async () => ({ isEmpty: () => false, toJPEG: () => Buffer.from("snapshot"), + toPNG: () => Buffer.from("png-snapshot"), + getSize: () => ({ width: 640, height: 480 }), })), + debugger: { + attach: vi.fn(() => { + debuggerAttached = true; + }), + detach: vi.fn(() => { + debuggerAttached = false; + }), + isAttached: () => debuggerAttached, + on: (event: string, listener: (...args: never[]) => void) => debuggerListeners.set(event, listener), + sendCommand: debuggerSendCommand, + }, clearHistory: () => undefined, getTitle: () => "", getURL: () => currentURL, @@ -126,6 +147,9 @@ function setupHost() { shellSend, view, webContents, + webContentsListeners, + debuggerListeners, + debuggerSendCommand, }; } @@ -230,6 +254,120 @@ describe("browser:capture", () => { }); }); +describe("agent browser runtime", () => { + it("creates one hidden target per session and reuses it when the panel mounts", async () => { + const { host, invoke, view } = setupHost(); + await host.execute("sess-1", "open", { url: "http://localhost:4173" }); + + const state = await invoke("browser:ensure", "sess-1"); + + expect(state.viewId).toBe("0:sess-1"); + expect(view.webContents.loadURL).toHaveBeenCalledTimes(1); + }); + + it("returns compact refs and targets only the referenced WebContents node", async () => { + const { debuggerSendCommand, host } = setupHost(); + debuggerSendCommand.mockImplementation(async (method: string) => { + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { nodeId: "1", role: { value: "document" }, name: { value: "Demo" } }, + { + nodeId: "2", + parentId: "1", + backendDOMNodeId: 42, + role: { value: "button" }, + name: { value: "Save" }, + }, + ], + }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "save-button" } }; + return {}; + }); + + const snapshot = (await host.execute("sess-1", "snapshot", {})) as { text: string }; + await host.execute("sess-1", "click", { ref: "e1" }); + + expect(snapshot.text).toContain('button "Save" [ref=e1]'); + expect(debuggerSendCommand).toHaveBeenCalledWith("DOM.resolveNode", { backendNodeId: 42 }); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Runtime.callFunctionOn", + expect.objectContaining({ objectId: "save-button" }), + ); + }); + + it("fills the same session target mounted in the visible browser panel", async () => { + const { debuggerSendCommand, emit, host, invoke, view } = setupHost(); + debuggerSendCommand.mockImplementation(async (method: string) => { + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { + nodeId: "1", + backendDOMNodeId: 77, + role: { value: "textbox" }, + name: { value: "Profile" }, + }, + ], + }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "profile-input" } }; + return {}; + }); + + await host.execute("sess-1", "snapshot", { interactive: true }); + const panelState = await invoke("browser:ensure", "sess-1"); + emit("browser:setBounds", 1, { + viewId: panelState.viewId, + rect: { x: 20, y: 30, width: 400, height: 300 }, + visible: true, + }); + await host.execute("sess-1", "fill", { ref: "e1", text: "hello i am AO" }); + + expect(panelState.viewId).toBe("0:sess-1"); + expect(view.setVisible).toHaveBeenLastCalledWith(true); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Runtime.callFunctionOn", + expect.objectContaining({ + objectId: "profile-input", + arguments: [{ value: "hello i am AO" }], + }), + ); + }); + + it("invalidates refs after navigation", async () => { + const { debuggerSendCommand, host, webContentsListeners } = setupHost(); + debuggerSendCommand.mockImplementation(async (method: string) => { + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [{ nodeId: "1", backendDOMNodeId: 42, role: { value: "button" }, name: { value: "Save" } }], + }; + } + return {}; + }); + await host.execute("sess-1", "snapshot", {}); + webContentsListeners.get("did-start-loading")?.(); + + await expect(host.execute("sess-1", "click", { ref: "e1" })).rejects.toMatchObject({ + code: "STALE_REFERENCE", + }); + }); + + it("captures a PNG and separates errors from other console messages", async () => { + const { host, webContentsListeners } = setupHost(); + const screenshot = (await host.execute("sess-1", "screenshot")) as { data: string; width: number }; + const consoleListener = webContentsListeners.get("console-message"); + consoleListener?.({} as never, { level: "info", message: "ready" } as never); + consoleListener?.({} as never, { level: "error", message: "boom" } as never); + + const errors = (await host.execute("sess-1", "errors")) as { messages: Array<{ message: string }> }; + expect(screenshot.data).toBe(Buffer.from("png-snapshot").toString("base64")); + expect(screenshot.width).toBe(640); + expect(errors.messages.map((entry) => entry.message)).toEqual(["boom"]); + }); +}); + describe("browser:requestMirror", () => { it("grants the display-media request from the frame that armed the mirror", async () => { const { getDisplayHandler, invoke, rendererFrame, webContents } = setupHost(); diff --git a/frontend/src/main/browser-view-host.ts b/frontend/src/main/browser-view-host.ts index f1d73b58ff..2865986d1b 100644 --- a/frontend/src/main/browser-view-host.ts +++ b/frontend/src/main/browser-view-host.ts @@ -48,6 +48,7 @@ type BrowserWebContents = Pick< | "canGoForward" | "capturePage" | "clearHistory" + | "debugger" | "mainFrame" | "getTitle" | "getURL" @@ -104,6 +105,7 @@ export type BrowserViewHost = { dispose: () => void; destroy: (viewId: string) => void; destroyAll: () => void; + execute: (sessionId: string, action: string, args?: Record) => Promise; // webContents of the most recently focused browser panel (or null); the titlebar menu targets it for Edit/Reload/Zoom/DevTools. getLastFocusedPanelContents: () => WebContents | null; // Drop the remembered panel; call when the shell gains focus for a real reason so a stale panel stops absorbing menu actions. @@ -111,9 +113,22 @@ export type BrowserViewHost = { }; type BrowserEntry = { + sessionId: string; view: BrowserViewLike; state: BrowserNavState; annotationEnabled: boolean; + refGeneration: number; + refs: Map; + consoleMessages: BrowserLogEntry[]; + errors: BrowserLogEntry[]; +}; + +type BrowserLogEntry = { + level: string; + message: string; + source?: string; + line?: number; + timestamp: string; }; const OFFSCREEN_BOUNDS: BrowserRect = { x: -10_000, y: -10_000, width: 0, height: 0 }; @@ -177,6 +192,8 @@ export function scaleBoundsForZoom(rect: BrowserRect, zoomFactor: number): Brows export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserViewHost { const entries = new Map(); + const viewIdsBySessionId = new Map(); + const rendererOwnersByViewId = new Map>(); const viewIdsByWebContentsId = new Map(); const ipcDisposers: Array<() => void> = []; // viewId of the panel that most recently held focus; cleared when it is hidden or destroyed. @@ -218,7 +235,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }); } - const ensure = (viewId: string): BrowserEntry => { + const ensure = (viewId: string, sessionId: string): BrowserEntry => { const existing = entries.get(viewId); if (existing) return existing; @@ -235,11 +252,22 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV options.mainWindow.contentView.addChildView(view); const state: BrowserNavState = emptyNavState(viewId); - const entry = { view, state, annotationEnabled: false }; + const entry: BrowserEntry = { + sessionId, + view, + state, + annotationEnabled: false, + refGeneration: 0, + refs: new Map(), + consoleMessages: [], + errors: [], + }; entries.set(viewId, entry); + viewIdsBySessionId.set(sessionId, viewId); viewIdsByWebContentsId.set(view.webContents.id, viewId); hardenWebContents(view.webContents, options, entry); wireNavEvents(view.webContents, options, entry); + wireAutomationEvents(view.webContents, entry); // The preview is a separate WebContentsView, so renderer-window keydown // listeners never see keys typed here. Forward application shortcuts to the // shell renderer so they still work with the panel focused. @@ -250,6 +278,21 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV return entry; }; + const ensureSession = (sessionId: string, rendererId?: number): BrowserEntry => { + const existingViewId = viewIdsBySessionId.get(sessionId); + const viewId = existingViewId ?? `${rendererId ?? 0}:${sessionId}`; + const entry = ensure(viewId, sessionId); + if (rendererId !== undefined) { + const owners = rendererOwnersByViewId.get(viewId) ?? new Set(); + owners.add(rendererId); + rendererOwnersByViewId.set(viewId, owners); + } + return entry; + }; + + const isRendererOwned = (event: IpcMainInvokeEvent | IpcMainEvent, viewId: string): boolean => + rendererOwnersByViewId.get(viewId)?.has(event.sender.id) ?? false; + const setBounds = ({ viewId, rect, visible, parked }: BrowserBoundsInput, zoomFactor = 1): void => { const entry = entries.get(viewId); if (!entry) return; @@ -276,7 +319,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }; const navigate = async ({ viewId, url }: BrowserNavigateInput): Promise => { - const entry = ensure(viewId); + const entry = entries.get(viewId); + if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); cancelAnnotation(options, entry, "navigation"); const normalized = normalizeBrowserURL(url); if (!isAllowedBrowserURL(normalized.href, options.rendererOrigin)) { @@ -300,7 +344,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV // readNavState normalizes it back to an empty url so the panel shows its // empty state. const clear = async (viewId: string): Promise => { - const entry = ensure(viewId); + const entry = entries.get(viewId); + if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); cancelAnnotation(options, entry, "navigation"); entry.view.setVisible?.(false); entry.view.setBounds(OFFSCREEN_BOUNDS); @@ -326,6 +371,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV const entry = entries.get(viewId); if (!entry) return; entries.delete(viewId); + viewIdsBySessionId.delete(entry.sessionId); + rendererOwnersByViewId.delete(viewId); viewIdsByWebContentsId.delete(entry.view.webContents.id); forgetIfFocused(viewId); // When the window is already gone (dispose fired from mainWindow "closed"), @@ -335,6 +382,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV entry.view.setVisible?.(false); entry.view.setBounds(OFFSCREEN_BOUNDS); options.mainWindow.contentView.removeChildView?.(entry.view); + if (entry.view.webContents.debugger?.isAttached()) { + entry.view.webContents.debugger.detach(); + } entry.view.webContents.close?.(); }; @@ -351,7 +401,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }; const setAnnotationMode = (event: IpcMainInvokeEvent, input: BrowserAnnotationModeInput): void => { - if (!isRendererOwnedViewId(event, input.viewId)) return; + if (!isRendererOwned(event, input.viewId)) return; const entry = entries.get(input.viewId); if (!entry) return; entry.annotationEnabled = input.enabled; @@ -410,24 +460,44 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV ipcDisposers.push(() => options.ipcMain.off(channel, fn)); }; - handle("browser:ensure", (event, sessionId: string) => pushNavState(options, ensure(scopedViewId(event, sessionId)))); - on("browser:setBounds", (event, input: BrowserBoundsInput) => setBounds(input, event.sender.getZoomFactor())); - handle("browser:navigate", (_event, input: BrowserNavigateInput) => navigate(input)); - handle("browser:clear", (_event, viewId: string) => clear(viewId)); - handle("browser:capture", (event, viewId: string) => (isRendererOwnedViewId(event, viewId) ? capture(viewId) : "")); + handle("browser:ensure", (event, sessionId: string) => + pushNavState(options, ensureSession(sessionId, event.sender.id)), + ); + on("browser:setBounds", (event, input: BrowserBoundsInput) => { + if (isRendererOwned(event, input.viewId)) setBounds(input, event.sender.getZoomFactor()); + }); + handle("browser:navigate", (event, input: BrowserNavigateInput) => + isRendererOwned(event, input.viewId) ? navigate(input) : emptyNavState(input.viewId), + ); + handle("browser:clear", (event, viewId: string) => + isRendererOwned(event, viewId) ? clear(viewId) : emptyNavState(viewId), + ); + handle("browser:capture", (event, viewId: string) => (isRendererOwned(event, viewId) ? capture(viewId) : "")); handle("browser:requestMirror", (event, viewId: string) => { - if (!mirrorSupported || !isRendererOwnedViewId(event, viewId) || !entries.has(viewId)) return false; + if (!mirrorSupported || !isRendererOwned(event, viewId) || !entries.has(viewId)) return false; const frame = event.senderFrame; if (!frame) return false; pendingMirror = { viewId, expires: Date.now() + 5000, frame }; return true; }); - handle("browser:goBack", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.goBack(), true)); - handle("browser:goForward", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.goForward(), true)); - handle("browser:reload", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.reload(), true)); - handle("browser:stop", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.stop())); + handle("browser:goBack", (event, viewId: string) => + isRendererOwned(event, viewId) ? invokeNav(viewId, (contents) => contents.goBack(), true) : emptyNavState(viewId), + ); + handle("browser:goForward", (event, viewId: string) => + isRendererOwned(event, viewId) + ? invokeNav(viewId, (contents) => contents.goForward(), true) + : emptyNavState(viewId), + ); + handle("browser:reload", (event, viewId: string) => + isRendererOwned(event, viewId) ? invokeNav(viewId, (contents) => contents.reload(), true) : emptyNavState(viewId), + ); + handle("browser:stop", (event, viewId: string) => + isRendererOwned(event, viewId) ? invokeNav(viewId, (contents) => contents.stop()) : emptyNavState(viewId), + ); handle("browser:annotation:setMode", (event, input: BrowserAnnotationModeInput) => setAnnotationMode(event, input)); - on("browser:destroy", (_event, viewId: string) => destroy(viewId)); + on("browser:destroy", (event, viewId: string) => { + if (isRendererOwned(event, viewId)) destroy(viewId); + }); on("browser:annotation:submit", (event, payload: BrowserAnnotationPageSubmitPayload) => forwardAnnotationSubmit(event, payload), ); @@ -436,6 +506,38 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV ); return { + execute: async (sessionId, action, args = {}) => { + if (!sessionId.trim()) throw browserError("INVALID_ARGUMENT", "sessionId is required"); + const entry = ensureSession(sessionId); + switch (action) { + case "open": { + const url = stringArg(args, "url", "URL_REQUIRED", "url is required"); + const state = await navigate({ viewId: entry.state.viewId, url }); + if (state.error) throw browserError("NAVIGATION_FAILED", state.error); + return state; + } + case "snapshot": + return snapshotEntry(entry, Boolean(args.interactive)); + case "click": + return clickEntry(entry, stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required")); + case "fill": + return fillEntry( + entry, + stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required"), + stringArg(args, "text", "INVALID_ARGUMENT", "text is required", true), + ); + case "wait": + return waitForEntry(entry, args); + case "screenshot": + return screenshotEntry(entry); + case "console": + return { messages: [...entry.consoleMessages] }; + case "errors": + return { messages: [...entry.errors] }; + default: + throw browserError("INVALID_ARGUMENT", `Unsupported browser action: ${action}`); + } + }, dispose: () => { ipcDisposers.splice(0).forEach((dispose) => dispose()); for (const viewId of [...entries.keys()]) { @@ -505,14 +607,6 @@ function emptyNavState(viewId: string): BrowserNavState { }; } -function scopedViewId(event: IpcMainInvokeEvent, sessionId: string): string { - return `${event.sender.id}:${sessionId}`; -} - -function isRendererOwnedViewId(event: IpcMainInvokeEvent, viewId: string): boolean { - return viewId.startsWith(`${event.sender.id}:`); -} - function hardenWebContents(contents: BrowserWebContents, options: BrowserViewHostOptions, entry: BrowserEntry): void { contents.setWindowOpenHandler(({ url }) => { if (isAllowedBrowserURL(url, options.rendererOrigin)) { @@ -542,12 +636,18 @@ function wireNavEvents(contents: BrowserWebContents, options: BrowserViewHostOpt contents.on("did-navigate-in-page", update); contents.on("page-title-updated", update); contents.on("did-start-loading", () => { + invalidateRefs(entry); cancelAnnotation(options, entry, "navigation"); update(); }); contents.on("did-stop-loading", update); contents.on("did-fail-load", (_event, errorCode, errorDescription) => { if (errorCode === -3) return; + pushBrowserLog(entry.errors, { + level: "error", + message: String(errorDescription || `Navigation failed (${errorCode})`), + timestamp: new Date().toISOString(), + }); entry.view.setVisible?.(false); entry.state = { ...readNavState(entry), error: String(errorDescription || "Unable to load page") }; options.mainWindow.webContents.send("browser:navState", entry.state); @@ -586,3 +686,285 @@ function readNavState(entry: BrowserEntry): BrowserNavState { isLoading: webContents.isLoading(), }; } + +type AXValue = { value?: unknown }; +type AXNode = { + nodeId: string; + parentId?: string; + ignored?: boolean; + backendDOMNodeId?: number; + role?: AXValue; + name?: AXValue; + value?: AXValue; + properties?: Array<{ name: string; value?: AXValue }>; +}; + +const INTERACTIVE_ROLES = new Set([ + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "scrollbar", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +]); + +function wireAutomationEvents(contents: BrowserWebContents, entry: BrowserEntry): void { + contents.on("console-message", (...eventArgs: unknown[]) => { + const details = eventArgs.find( + (value) => value && typeof value === "object" && typeof (value as { message?: unknown }).message === "string", + ) as { level?: string; message: string; lineNumber?: number; sourceId?: string } | undefined; + const legacyLevel = typeof eventArgs[1] === "number" ? eventArgs[1] : 1; + const legacyMessage = typeof eventArgs[2] === "string" ? eventArgs[2] : ""; + const level = details?.level ?? ["debug", "info", "warning", "error"][legacyLevel] ?? "info"; + const log: BrowserLogEntry = { + level, + message: details?.message ?? legacyMessage, + source: details?.sourceId ?? (typeof eventArgs[4] === "string" ? eventArgs[4] : undefined), + line: details?.lineNumber ?? (typeof eventArgs[3] === "number" ? eventArgs[3] : undefined), + timestamp: new Date().toISOString(), + }; + pushBrowserLog(entry.consoleMessages, log); + if (level === "error") pushBrowserLog(entry.errors, log); + }); + contents.on("render-process-gone", (_event, details) => { + pushBrowserLog(entry.errors, { + level: "error", + message: `Browser renderer exited: ${details.reason}`, + timestamp: new Date().toISOString(), + }); + }); + const targetDebugger = contents.debugger; + if (!targetDebugger) return; + targetDebugger.on("message", (_event, method, params) => { + if (method === "DOM.documentUpdated") { + invalidateRefs(entry); + return; + } + if (method === "Runtime.exceptionThrown") { + const detail = params as { exceptionDetails?: { text?: string; url?: string; lineNumber?: number } }; + const exception = detail.exceptionDetails; + pushBrowserLog(entry.errors, { + level: "error", + message: exception?.text ?? "Uncaught browser exception", + source: exception?.url, + line: exception?.lineNumber, + timestamp: new Date().toISOString(), + }); + } + }); +} + +function pushBrowserLog(target: BrowserLogEntry[], entry: BrowserLogEntry): void { + target.push(entry); + if (target.length > 200) target.splice(0, target.length - 200); +} + +function invalidateRefs(entry: BrowserEntry): void { + entry.refGeneration += 1; + entry.refs.clear(); +} + +async function ensureDebugger(entry: BrowserEntry): Promise { + const debug = entry.view.webContents.debugger; + if (!debug) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser debugger is unavailable"); + if (!debug.isAttached()) { + try { + debug.attach("1.3"); + } catch (error) { + throw browserError( + "BROWSER_TARGET_UNAVAILABLE", + error instanceof Error ? error.message : "Unable to attach to browser target", + ); + } + } + await debug.sendCommand("Runtime.enable"); + await debug.sendCommand("DOM.enable"); +} + +async function snapshotEntry(entry: BrowserEntry, interactiveOnly: boolean): Promise { + await ensureDebugger(entry); + await entry.view.webContents.debugger.sendCommand("Accessibility.enable"); + const response = (await entry.view.webContents.debugger.sendCommand("Accessibility.getFullAXTree")) as { + nodes?: AXNode[]; + }; + const nodes = response.nodes ?? []; + entry.refGeneration += 1; + entry.refs.clear(); + const generation = entry.refGeneration; + const depths = new Map(); + const lines: string[] = []; + const elements: Array<{ ref: string; role: string; name: string }> = []; + let refIndex = 0; + for (const node of nodes.slice(0, 1_000)) { + if (node.ignored) continue; + const role = stringValue(node.role) || "generic"; + const name = stringValue(node.name); + const value = stringValue(node.value); + const interactive = + INTERACTIVE_ROLES.has(role) || node.properties?.some((property) => property.name === "focusable"); + let ref = ""; + if (interactive && node.backendDOMNodeId) { + ref = `e${++refIndex}`; + entry.refs.set(ref, { backendNodeId: node.backendDOMNodeId, generation }); + elements.push({ ref, role, name }); + } + if (interactiveOnly && !ref) continue; + if (!ref && !name && !value) continue; + const parentDepth = node.parentId ? (depths.get(node.parentId) ?? -1) : -1; + const depth = Math.max(0, parentDepth + 1); + depths.set(node.nodeId, depth); + const label = name ? ` \"${compactText(name)}\"` : ""; + const currentValue = value && value !== name ? ` value=\"${compactText(value)}\"` : ""; + const reference = ref ? ` [ref=${ref}]` : ""; + lines.push(`${" ".repeat(Math.min(depth, 8))}${role}${label}${currentValue}${reference}`); + } + return { + url: entry.view.webContents.getURL(), + title: entry.view.webContents.getTitle(), + generation, + text: lines.join("\n") || "(empty accessibility snapshot)", + elements, + }; +} + +async function clickEntry(entry: BrowserEntry, refName: string): Promise { + const objectId = await resolveRef(entry, refName); + await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: + "function(){ this.scrollIntoView({block:'center',inline:'center'}); this.focus(); this.click(); }", + awaitPromise: true, + }); + return { ref: refName, url: entry.view.webContents.getURL() }; +} + +async function fillEntry(entry: BrowserEntry, refName: string, text: string): Promise { + const objectId = await resolveRef(entry, refName); + await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function(next){ + this.scrollIntoView({block:'center',inline:'center'}); + this.focus(); + const proto = Object.getPrototypeOf(this); + const descriptor = proto && Object.getOwnPropertyDescriptor(proto, 'value'); + if (descriptor && descriptor.set) descriptor.set.call(this, next); else this.value = next; + this.dispatchEvent(new Event('input', {bubbles:true, composed:true})); + this.dispatchEvent(new Event('change', {bubbles:true, composed:true})); + }`, + arguments: [{ value: text }], + awaitPromise: true, + }); + return { ref: refName, value: text, url: entry.view.webContents.getURL() }; +} + +async function resolveRef(entry: BrowserEntry, refName: string): Promise { + await ensureDebugger(entry); + const ref = entry.refs.get(refName); + if (!ref || ref.generation !== entry.refGeneration) { + throw browserError("STALE_REFERENCE", `Element reference ${refName} is stale; run ao browser snapshot again`); + } + try { + const resolved = (await entry.view.webContents.debugger.sendCommand("DOM.resolveNode", { + backendNodeId: ref.backendNodeId, + })) as { object?: { objectId?: string } }; + if (!resolved.object?.objectId) throw new Error("node has no runtime object"); + return resolved.object.objectId; + } catch { + entry.refs.delete(refName); + throw browserError("STALE_REFERENCE", `Element reference ${refName} is stale; run ao browser snapshot again`); + } +} + +async function waitForEntry(entry: BrowserEntry, args: Record): Promise { + const fixedMS = numberArg(args.ms, 0, 60_000); + if (fixedMS > 0) { + await delay(fixedMS); + return { waitedMs: fixedMS, url: entry.view.webContents.getURL() }; + } + const timeoutMS = numberArg(args.timeoutMs, 1, 60_000) || 10_000; + let expression = ""; + let condition = ""; + if (typeof args.text === "string" && args.text) { + expression = `Boolean(document.body && document.body.innerText.includes(${JSON.stringify(args.text)}))`; + condition = `text ${JSON.stringify(args.text)}`; + } else if (typeof args.selector === "string" && args.selector) { + expression = `Boolean(document.querySelector(${JSON.stringify(args.selector)}))`; + condition = `selector ${JSON.stringify(args.selector)}`; + } else if (typeof args.url === "string" && args.url) { + expression = `location.href.includes(${JSON.stringify(args.url)})`; + condition = `URL ${JSON.stringify(args.url)}`; + } else { + throw browserError("INVALID_ARGUMENT", "wait requires text, selector, url, or ms"); + } + await ensureDebugger(entry); + const deadline = Date.now() + timeoutMS; + while (Date.now() <= deadline) { + const evaluated = (await entry.view.webContents.debugger.sendCommand("Runtime.evaluate", { + expression, + returnByValue: true, + })) as { result?: { value?: unknown } }; + if (evaluated.result?.value === true) { + return { condition, url: entry.view.webContents.getURL() }; + } + await delay(100); + } + throw browserError("WAIT_TIMEOUT", `Timed out after ${timeoutMS}ms waiting for ${condition}`); +} + +async function screenshotEntry(entry: BrowserEntry): Promise { + const image = await entry.view.webContents.capturePage(); + if (image.isEmpty()) throw browserError("BROWSER_COMMAND_FAILED", "Browser screenshot is empty"); + const size = image.getSize(); + return { + mimeType: "image/png", + data: image.toPNG().toString("base64"), + width: size.width, + height: size.height, + url: entry.view.webContents.getURL(), + }; +} + +function stringArg( + args: Record, + name: string, + code: string, + message: string, + allowEmpty = false, +): string { + const value = args[name]; + if (typeof value !== "string" || (!allowEmpty && !value.trim())) throw browserError(code, message); + return value; +} + +function numberArg(value: unknown, min: number, max: number): number { + if (typeof value !== "number" || !Number.isFinite(value)) return 0; + return Math.max(min, Math.min(max, Math.round(value))); +} + +function stringValue(value: AXValue | undefined): string { + return typeof value?.value === "string" ? value.value : value?.value == null ? "" : String(value.value); +} + +function compactText(value: string): string { + return value.replace(/\s+/g, " ").replace(/\"/g, '\\"').trim().slice(0, 240); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function browserError(code: string, message: string): Error & { code: string } { + return Object.assign(new Error(message), { code }); +} From 4e77e197d40d66e3ddb7251e46300f178031f9fe Mon Sep 17 00:00:00 2001 From: aprv10 <1apoorvindia@gmail.com> Date: Thu, 23 Jul 2026 15:40:53 +0530 Subject: [PATCH 02/11] feat(browser): add session-scoped agent control # Conflicts: # backend/internal/daemon/daemon.go # backend/internal/httpd/api.go # backend/internal/httpd/apispec/openapi.yaml # backend/internal/httpd/lan_listener.go # backend/internal/httpd/lan_listener_test.go # backend/internal/skillassets/using-ao/SKILL.md # frontend/src/api/schema.ts # frontend/src/main.ts --- backend/internal/browserruntime/broker.go | 259 +++++++++++ .../internal/browserruntime/broker_test.go | 135 ++++++ .../internal/browserruntime/listen_unix.go | 23 + .../internal/browserruntime/listen_windows.go | 33 ++ backend/internal/cli/browser.go | 306 +++++++++++++ backend/internal/cli/browser_test.go | 125 +++++ backend/internal/cli/root.go | 1 + backend/internal/daemon/daemon.go | 13 + backend/internal/httpd/api.go | 4 + backend/internal/httpd/apispec/openapi.yaml | 139 ++++++ .../internal/httpd/apispec/specgen/build.go | 37 ++ backend/internal/httpd/controllers/browser.go | 130 ++++++ .../httpd/controllers/browser_test.go | 98 ++++ backend/internal/httpd/controllers/dto.go | 31 ++ backend/internal/httpd/lan_listener.go | 1 + backend/internal/httpd/lan_listener_test.go | 9 +- backend/internal/session_manager/manager.go | 7 +- .../internal/session_manager/manager_test.go | 10 +- .../internal/skillassets/skillassets_test.go | 3 + .../internal/skillassets/using-ao/SKILL.md | 6 +- .../skillassets/using-ao/commands/browser.md | 40 ++ .../skillassets/using-ao/references.md | 4 + docs/STATUS.md | 5 + docs/architecture.md | 14 + docs/cli/README.md | 9 + frontend/src/api/schema.ts | 182 ++++++++ frontend/src/main.ts | 46 ++ .../src/main/browser-runtime-link.test.ts | 97 ++++ frontend/src/main/browser-runtime-link.ts | 171 +++++++ frontend/src/main/browser-view-host.test.ts | 138 ++++++ frontend/src/main/browser-view-host.ts | 430 +++++++++++++++++- 31 files changed, 2473 insertions(+), 33 deletions(-) create mode 100644 backend/internal/browserruntime/broker.go create mode 100644 backend/internal/browserruntime/broker_test.go create mode 100644 backend/internal/browserruntime/listen_unix.go create mode 100644 backend/internal/browserruntime/listen_windows.go create mode 100644 backend/internal/cli/browser.go create mode 100644 backend/internal/cli/browser_test.go create mode 100644 backend/internal/httpd/controllers/browser.go create mode 100644 backend/internal/httpd/controllers/browser_test.go create mode 100644 backend/internal/skillassets/using-ao/commands/browser.md create mode 100644 frontend/src/main/browser-runtime-link.test.ts create mode 100644 frontend/src/main/browser-runtime-link.ts diff --git a/backend/internal/browserruntime/broker.go b/backend/internal/browserruntime/broker.go new file mode 100644 index 0000000000..1269f04207 --- /dev/null +++ b/backend/internal/browserruntime/broker.go @@ -0,0 +1,259 @@ +// Package browserruntime brokers browser commands between the loopback daemon +// and the Electron process that owns AO's per-session WebContentsView targets. +package browserruntime + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "sync" + "time" + + "github.com/google/uuid" + + "github.com/aoagents/agent-orchestrator/backend/internal/domain" +) + +const ProtocolVersion = 1 + +var ErrUnavailable = errors.New("browser runtime is unavailable") + +type Status struct { + Connected bool + ConnectedAt time.Time +} + +type Command struct { + RequestID string `json:"requestId"` + SessionID domain.SessionID `json:"sessionId"` + Action string `json:"action"` + Args map[string]interface{} `json:"args,omitempty"` +} + +type CommandError struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (e CommandError) Error() string { + if e.Code == "" { + return e.Message + } + return fmt.Sprintf("%s (%s)", e.Message, e.Code) +} + +type Result struct { + RequestID string + Value interface{} +} + +type wireMessage struct { + Type string `json:"type"` + Version int `json:"version,omitempty"` + RequestID string `json:"requestId,omitempty"` + SessionID domain.SessionID `json:"sessionId,omitempty"` + Action string `json:"action,omitempty"` + Args map[string]interface{} `json:"args,omitempty"` + OK bool `json:"ok,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *CommandError `json:"error,omitempty"` +} + +type pendingResult struct { + value interface{} + err error +} + +// Broker owns the single active Electron runtime connection. Commands are +// correlated by request id, so independent AO sessions may use the bridge +// concurrently without sharing browser targets or results. +type Broker struct { + log *slog.Logger + + mu sync.Mutex + conn net.Conn + connectedAt time.Time + pending map[string]chan pendingResult + writeMu sync.Mutex +} + +func New(log *slog.Logger) *Broker { + if log == nil { + log = slog.Default() + } + return &Broker{log: log, pending: make(map[string]chan pendingResult)} +} + +func (b *Broker) Status() Status { + b.mu.Lock() + defer b.mu.Unlock() + return Status{Connected: b.conn != nil, ConnectedAt: b.connectedAt} +} + +func (b *Broker) Execute(ctx context.Context, sessionID domain.SessionID, action string, args map[string]interface{}) (Result, error) { + requestID := uuid.NewString() + resultCh := make(chan pendingResult, 1) + + b.mu.Lock() + conn := b.conn + if conn == nil { + b.mu.Unlock() + return Result{}, ErrUnavailable + } + b.pending[requestID] = resultCh + b.mu.Unlock() + + msg := wireMessage{ + Type: "command", + RequestID: requestID, + SessionID: sessionID, + Action: action, + Args: args, + } + if err := b.write(conn, msg); err != nil { + b.removePending(requestID) + b.disconnect(conn, fmt.Errorf("write browser command: %w", err)) + return Result{}, ErrUnavailable + } + + select { + case <-ctx.Done(): + b.removePending(requestID) + return Result{}, ctx.Err() + case result := <-resultCh: + if result.err != nil { + return Result{}, result.err + } + return Result{RequestID: requestID, Value: result.value}, nil + } +} + +// Serve accepts Electron runtime connections until ctx is cancelled. A new +// valid runtime replaces an older connection and fails its in-flight commands; +// this makes renderer reload/restart recovery deterministic. +func (b *Broker) Serve(ctx context.Context, ln net.Listener) error { + go func() { + <-ctx.Done() + _ = ln.Close() + }() + for { + conn, err := ln.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return nil + } + return fmt.Errorf("accept browser runtime: %w", err) + } + go b.serveConn(ctx, conn) + } +} + +func (b *Broker) serveConn(ctx context.Context, conn net.Conn) { + dec := json.NewDecoder(conn) + var hello wireMessage + if err := dec.Decode(&hello); err != nil || hello.Type != "hello" || hello.Version != ProtocolVersion { + _ = conn.Close() + return + } + + b.mu.Lock() + old := b.conn + b.conn = conn + b.connectedAt = time.Now().UTC() + pending := b.takePendingLocked() + b.mu.Unlock() + if old != nil && old != conn { + _ = old.Close() + } + failPending(pending, ErrUnavailable) + b.log.Info("browser runtime connected") + + go func() { + <-ctx.Done() + _ = conn.Close() + }() + + for { + var msg wireMessage + if err := dec.Decode(&msg); err != nil { + b.disconnect(conn, err) + return + } + if msg.Type != "result" || msg.RequestID == "" { + continue + } + b.resolve(msg) + } +} + +func (b *Broker) write(conn net.Conn, msg wireMessage) error { + b.writeMu.Lock() + defer b.writeMu.Unlock() + return json.NewEncoder(conn).Encode(msg) +} + +func (b *Broker) resolve(msg wireMessage) { + b.mu.Lock() + ch := b.pending[msg.RequestID] + delete(b.pending, msg.RequestID) + b.mu.Unlock() + if ch == nil { + return + } + if !msg.OK { + if msg.Error == nil { + msg.Error = &CommandError{Code: "BROWSER_COMMAND_FAILED", Message: "Browser command failed"} + } + ch <- pendingResult{err: *msg.Error} + return + } + var value interface{} = map[string]interface{}{} + if len(msg.Result) > 0 && string(msg.Result) != "null" { + if err := json.Unmarshal(msg.Result, &value); err != nil { + ch <- pendingResult{err: fmt.Errorf("decode browser result: %w", err)} + return + } + } + ch <- pendingResult{value: value} +} + +func (b *Broker) disconnect(conn net.Conn, cause error) { + b.mu.Lock() + if b.conn != conn { + b.mu.Unlock() + return + } + b.conn = nil + b.connectedAt = time.Time{} + pending := b.takePendingLocked() + b.mu.Unlock() + _ = conn.Close() + failPending(pending, ErrUnavailable) + if cause != nil && !errors.Is(cause, io.EOF) && !errors.Is(cause, net.ErrClosed) { + b.log.Warn("browser runtime disconnected", "err", cause) + } else { + b.log.Info("browser runtime disconnected") + } +} + +func (b *Broker) removePending(requestID string) { + b.mu.Lock() + delete(b.pending, requestID) + b.mu.Unlock() +} + +func (b *Broker) takePendingLocked() map[string]chan pendingResult { + pending := b.pending + b.pending = make(map[string]chan pendingResult) + return pending +} + +func failPending(pending map[string]chan pendingResult, err error) { + for _, ch := range pending { + ch <- pendingResult{err: err} + } +} diff --git a/backend/internal/browserruntime/broker_test.go b/backend/internal/browserruntime/broker_test.go new file mode 100644 index 0000000000..85828e6881 --- /dev/null +++ b/backend/internal/browserruntime/broker_test.go @@ -0,0 +1,135 @@ +package browserruntime + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net" + "testing" + "time" +) + +func TestBrokerExecuteRoundTrip(t *testing.T) { + broker := New(slog.New(slog.NewTextHandler(io.Discard, nil))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = broker.Serve(ctx, ln) }() + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + enc := json.NewEncoder(conn) + dec := json.NewDecoder(conn) + if err := enc.Encode(wireMessage{Type: "hello", Version: ProtocolVersion}); err != nil { + t.Fatal(err) + } + waitConnected(t, broker) + + resultCh := make(chan Result, 1) + errCh := make(chan error, 1) + go func() { + result, err := broker.Execute(context.Background(), "session-1", "snapshot", map[string]interface{}{"interactive": true}) + if err != nil { + errCh <- err + return + } + resultCh <- result + }() + + var command wireMessage + if err := dec.Decode(&command); err != nil { + t.Fatal(err) + } + if command.Type != "command" || command.SessionID != "session-1" || command.Action != "snapshot" { + t.Fatalf("command = %#v", command) + } + if err := enc.Encode(wireMessage{ + Type: "result", + RequestID: command.RequestID, + OK: true, + Result: json.RawMessage(`{"text":"button Save [ref=e1]"}`), + }); err != nil { + t.Fatal(err) + } + + select { + case err := <-errCh: + t.Fatal(err) + case result := <-resultCh: + value := result.Value.(map[string]interface{}) + if value["text"] != "button Save [ref=e1]" { + t.Fatalf("result = %#v", result.Value) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for result") + } +} + +func TestBrokerMapsRuntimeError(t *testing.T) { + broker := New(slog.New(slog.NewTextHandler(io.Discard, nil))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = broker.Serve(ctx, ln) }() + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + enc := json.NewEncoder(conn) + dec := json.NewDecoder(conn) + _ = enc.Encode(wireMessage{Type: "hello", Version: ProtocolVersion}) + waitConnected(t, broker) + + errCh := make(chan error, 1) + go func() { + _, err := broker.Execute(context.Background(), "session-1", "click", map[string]interface{}{"ref": "e1"}) + errCh <- err + }() + var command wireMessage + if err := dec.Decode(&command); err != nil { + t.Fatal(err) + } + _ = enc.Encode(wireMessage{ + Type: "result", + RequestID: command.RequestID, + Error: &CommandError{Code: "STALE_REFERENCE", Message: "snapshot again"}, + }) + + select { + case err := <-errCh: + commandErr, ok := err.(CommandError) + if !ok || commandErr.Code != "STALE_REFERENCE" { + t.Fatalf("error = %#v", err) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for error") + } +} + +func TestBrokerUnavailableWithoutElectron(t *testing.T) { + broker := New(nil) + if _, err := broker.Execute(context.Background(), "session-1", "snapshot", nil); err != ErrUnavailable { + t.Fatalf("error = %v, want ErrUnavailable", err) + } +} + +func waitConnected(t *testing.T, broker *Broker) { + t.Helper() + deadline := time.Now().Add(time.Second) + for !broker.Status().Connected { + if time.Now().After(deadline) { + t.Fatal("browser runtime did not connect") + } + time.Sleep(time.Millisecond) + } +} diff --git a/backend/internal/browserruntime/listen_unix.go b/backend/internal/browserruntime/listen_unix.go new file mode 100644 index 0000000000..231ffca0b4 --- /dev/null +++ b/backend/internal/browserruntime/listen_unix.go @@ -0,0 +1,23 @@ +//go:build !windows + +package browserruntime + +import ( + "net" + "os" + "path/filepath" +) + +func Listen(runFilePath string) (net.Listener, string, error) { + sockPath := filepath.Join(filepath.Dir(runFilePath), "browser.sock") + _ = os.Remove(sockPath) + ln, err := net.Listen("unix", sockPath) + if err != nil { + return nil, "", err + } + if err := os.Chmod(sockPath, 0o600); err != nil { + _ = ln.Close() + return nil, "", err + } + return ln, sockPath, nil +} diff --git a/backend/internal/browserruntime/listen_windows.go b/backend/internal/browserruntime/listen_windows.go new file mode 100644 index 0000000000..8e88981b32 --- /dev/null +++ b/backend/internal/browserruntime/listen_windows.go @@ -0,0 +1,33 @@ +//go:build windows + +package browserruntime + +import ( + "net" + "path/filepath" + "regexp" + + "github.com/Microsoft/go-winio" +) + +var unsafePipeChars = regexp.MustCompile(`[^a-zA-Z0-9\-]`) + +func pipeNameFromRunFile(runFilePath string) string { + if runFilePath == "" { + return `\\.\pipe\ao-browser` + } + dir := filepath.Base(filepath.Dir(runFilePath)) + if dir == ".ao" || dir == "." || dir == "" { + return `\\.\pipe\ao-browser` + } + return `\\.\pipe\ao-browser-` + unsafePipeChars.ReplaceAllString(dir, "-") +} + +func Listen(runFilePath string) (net.Listener, string, error) { + name := pipeNameFromRunFile(runFilePath) + ln, err := winio.ListenPipe(name, nil) + if err != nil { + return nil, "", err + } + return ln, name, nil +} diff --git a/backend/internal/cli/browser.go b/backend/internal/cli/browser.go new file mode 100644 index 0000000000..f8f04a3379 --- /dev/null +++ b/backend/internal/cli/browser.go @@ -0,0 +1,306 @@ +package cli + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" +) + +type browserStatusDTO struct { + SessionID string `json:"sessionId"` + Connected bool `json:"connected"` + ConnectedAt time.Time `json:"connectedAt,omitempty"` + Transport string `json:"transport"` +} + +type browserCommandRequestDTO struct { + SessionID string `json:"sessionId"` + Action string `json:"action"` + Args map[string]any `json:"args,omitempty"` +} + +type browserCommandResponseDTO struct { + RequestID string `json:"requestId"` + SessionID string `json:"sessionId"` + Action string `json:"action"` + Result map[string]any `json:"result"` +} + +func newBrowserCommand(ctx *commandContext) *cobra.Command { + var jsonOutput bool + cmd := &cobra.Command{ + Use: "browser", + Short: "Inspect and control this AO session's shared desktop browser", + Long: "Inspect and control the target-isolated browser owned by the current AO session.\n\n" + + "The desktop app must be open. Commands operate the same live page the user sees,\n" + + "including while the Browser panel is hidden.", + Args: noArgs, + } + cmd.PersistentFlags().BoolVar(&jsonOutput, "json", false, "print the structured response as JSON") + + cmd.AddCommand(&cobra.Command{ + Use: "status", + Short: "Show whether the desktop browser runtime is connected", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + status, err := ctx.browserStatus(cmd.Context()) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(cmd.OutOrStdout(), status) + } + state := "disconnected" + if status.Connected { + state = "connected" + } + _, err = fmt.Fprintf(cmd.OutOrStdout(), "Browser runtime: %s (%s)\n", state, status.Transport) + return err + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "open ", + Short: "Open a URL in this session's browser", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "open", map[string]any{"url": args[0]}, jsonOutput) + }, + }) + + var interactiveOnly bool + snapshot := &cobra.Command{ + Use: "snapshot", + Short: "Print a compact accessibility snapshot with actionable element refs", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return ctx.runBrowserAction(cmd, "snapshot", map[string]any{"interactive": interactiveOnly}, jsonOutput) + }, + } + snapshot.Flags().BoolVar(&interactiveOnly, "interactive", false, "include only actionable elements") + cmd.AddCommand(snapshot) + + cmd.AddCommand(&cobra.Command{ + Use: "click ", + Short: "Click an element reference from the latest snapshot", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "click", map[string]any{"ref": args[0]}, jsonOutput) + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "fill ", + Short: "Replace the value of a form control", + Args: exactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "fill", map[string]any{"ref": args[0], "text": args[1]}, jsonOutput) + }, + }) + + var waitText, waitSelector, waitURL string + var waitMS, timeoutMS int + waitCmd := &cobra.Command{ + Use: "wait", + Short: "Wait for time, text, selector, or URL state", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + selected := 0 + for _, active := range []bool{waitText != "", waitSelector != "", waitURL != "", waitMS > 0} { + if active { + selected++ + } + } + if selected != 1 { + return usageError{errors.New("choose exactly one of --text, --selector, --url, or --ms")} + } + args := map[string]any{"timeoutMs": timeoutMS} + switch { + case waitText != "": + args["text"] = waitText + case waitSelector != "": + args["selector"] = waitSelector + case waitURL != "": + args["url"] = waitURL + default: + args["ms"] = waitMS + } + return ctx.runBrowserAction(cmd, "wait", args, jsonOutput) + }, + } + waitCmd.Flags().StringVar(&waitText, "text", "", "wait until visible page text contains this value") + waitCmd.Flags().StringVar(&waitSelector, "selector", "", "wait until this CSS selector exists") + waitCmd.Flags().StringVar(&waitURL, "url", "", "wait until the current URL contains this value") + waitCmd.Flags().IntVar(&waitMS, "ms", 0, "wait for a fixed number of milliseconds") + waitCmd.Flags().IntVar(&timeoutMS, "timeout", 10_000, "condition timeout in milliseconds") + cmd.AddCommand(waitCmd) + + cmd.AddCommand(&cobra.Command{ + Use: "screenshot [path]", + Short: "Capture the current page to a PNG file", + Args: atMostOneArg, + RunE: func(cmd *cobra.Command, args []string) error { + resp, err := ctx.browserAction(cmd.Context(), "screenshot", nil) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(cmd.OutOrStdout(), resp) + } + path := "" + if len(args) == 1 { + path = args[0] + } else { + path = "ao-browser-" + ctx.deps.Now().Format("20060102-150405.000") + ".png" + } + return writeBrowserScreenshot(cmd, resp.Result, path) + }, + }) + + for _, action := range []string{"console", "errors"} { + action := action + cmd.AddCommand(&cobra.Command{ + Use: action, + Short: "Print captured browser " + action, + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return ctx.runBrowserAction(cmd, action, nil, jsonOutput) + }, + }) + } + return cmd +} + +func exactArgs(n int) cobra.PositionalArgs { + return func(cmd *cobra.Command, args []string) error { + if err := cobra.ExactArgs(n)(cmd, args); err != nil { + return usageError{err} + } + return nil + } +} + +func currentBrowserSessionID() (string, error) { + sessionID := strings.TrimSpace(os.Getenv("AO_SESSION_ID")) + if sessionID == "" { + return "", usageError{errors.New("ao browser must run inside an AO session (AO_SESSION_ID is not set)")} + } + return sessionID, nil +} + +func (c *commandContext) browserStatus(ctx context.Context) (browserStatusDTO, error) { + sessionID, err := currentBrowserSessionID() + if err != nil { + return browserStatusDTO{}, err + } + var out browserStatusDTO + err = c.getJSON(ctx, "browser/status?sessionId="+url.QueryEscape(sessionID), &out) + return out, err +} + +func (c *commandContext) browserAction(ctx context.Context, action string, args map[string]any) (browserCommandResponseDTO, error) { + sessionID, err := currentBrowserSessionID() + if err != nil { + return browserCommandResponseDTO{}, err + } + var out browserCommandResponseDTO + err = c.postJSON(ctx, "browser/commands", browserCommandRequestDTO{SessionID: sessionID, Action: action, Args: args}, &out) + return out, err +} + +func (c *commandContext) runBrowserAction(cmd *cobra.Command, action string, args map[string]any, jsonOutput bool) error { + resp, err := c.browserAction(cmd.Context(), action, args) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(cmd.OutOrStdout(), resp) + } + return writeBrowserResult(cmd, action, resp.Result) +} + +func writeBrowserResult(cmd *cobra.Command, action string, result map[string]any) error { + if action == "snapshot" { + if text, ok := result["text"].(string); ok { + _, err := fmt.Fprintln(cmd.OutOrStdout(), text) + return err + } + } + if action == "console" || action == "errors" { + messages, _ := result["messages"].([]any) + if len(messages) == 0 { + _, err := fmt.Fprintln(cmd.OutOrStdout(), "No browser "+action+" captured.") + return err + } + for _, message := range messages { + if item, ok := message.(map[string]any); ok { + level, _ := item["level"].(string) + text, _ := item["message"].(string) + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "[%s] %s\n", level, text); err != nil { + return err + } + } + } + return nil + } + if currentURL, ok := result["url"].(string); ok && currentURL != "" { + _, err := fmt.Fprintln(cmd.OutOrStdout(), currentURL) + return err + } + _, err := fmt.Fprintln(cmd.OutOrStdout(), "Browser "+action+" completed.") + return err +} + +func writeBrowserScreenshot(cmd *cobra.Command, result map[string]any, target string) error { + encoded, _ := result["data"].(string) + if encoded == "" { + return errors.New("browser returned an empty screenshot") + } + data, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return fmt.Errorf("decode browser screenshot: %w", err) + } + abs, err := filepath.Abs(target) + if err != nil { + return err + } + file, err := os.OpenFile(abs, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("refusing to overwrite existing screenshot %s", abs) + } + return err + } + defer func() { _ = file.Close() }() + if _, err := file.Write(data); err != nil { + return err + } + width := numberString(result["width"]) + height := numberString(result["height"]) + size := "" + if width != "" && height != "" { + size = " (" + width + "x" + height + ")" + } + _, err = fmt.Fprintf(cmd.OutOrStdout(), "Saved %s%s\n", abs, size) + return err +} + +func numberString(v any) string { + switch n := v.(type) { + case float64: + return strconv.Itoa(int(n)) + case int: + return strconv.Itoa(n) + default: + return "" + } +} diff --git a/backend/internal/cli/browser_test.go b/backend/internal/cli/browser_test.go new file mode 100644 index 0000000000..1e5d1b7739 --- /dev/null +++ b/backend/internal/cli/browser_test.go @@ -0,0 +1,125 @@ +package cli + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +type browserRequestCapture struct { + path string + body browserCommandRequestDTO +} + +func browserCLIServer(t *testing.T, capture *browserRequestCapture) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capture.path = r.URL.RequestURI() + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet && r.URL.Path == "/api/v1/browser/status" { + _, _ = io.WriteString(w, `{"sessionId":"ao-1","connected":true,"transport":"electron-webcontents-debugger"}`) + return + } + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/browser/commands" { + http.NotFound(w, r) + return + } + if err := json.NewDecoder(r.Body).Decode(&capture.body); err != nil { + t.Fatalf("decode command: %v", err) + } + result := `{"ok":true}` + switch capture.body.Action { + case "snapshot": + result = `{"text":"button Save [ref=e1]"}` + case "screenshot": + result = `{"data":"cG5n","width":10,"height":20}` + } + _, _ = io.WriteString(w, `{"requestId":"r1","sessionId":"ao-1","action":"`+capture.body.Action+`","result":`+result+`}`) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestBrowserStatusAndSnapshot(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + + out, errOut, err := executeCLI(t, deps, "browser", "status") + if err != nil || !strings.Contains(out, "Browser runtime: connected") { + t.Fatalf("status err=%v stderr=%s stdout=%s", err, errOut, out) + } + if capture.path != "/api/v1/browser/status?sessionId=ao-1" { + t.Fatalf("status path = %q", capture.path) + } + out, errOut, err = executeCLI(t, deps, "browser", "snapshot", "--interactive") + if err != nil || !strings.Contains(out, "button Save [ref=e1]") { + t.Fatalf("snapshot err=%v stderr=%s stdout=%s", err, errOut, out) + } + if capture.body.SessionID != "ao-1" || capture.body.Action != "snapshot" || capture.body.Args["interactive"] != true { + t.Fatalf("command = %#v", capture.body) + } +} + +func TestBrowserClickAndWaitArguments(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + + if _, _, err := executeCLI(t, deps, "browser", "click", "e2"); err != nil { + t.Fatal(err) + } + if capture.body.Action != "click" || capture.body.Args["ref"] != "e2" { + t.Fatalf("click = %#v", capture.body) + } + if _, _, err := executeCLI(t, deps, "browser", "wait", "--text", "Ready", "--timeout", "2500"); err != nil { + t.Fatal(err) + } + if capture.body.Action != "wait" || capture.body.Args["text"] != "Ready" || capture.body.Args["timeoutMs"] != float64(2500) { + t.Fatalf("wait = %#v", capture.body) + } +} + +func TestBrowserScreenshotWritesWithoutOverwrite(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + target := filepath.Join(t.TempDir(), "shot.png") + + out, errOut, err := executeCLI(t, deps, "browser", "screenshot", target) + if err != nil { + t.Fatalf("screenshot err=%v stderr=%s", err, errOut) + } + data, err := os.ReadFile(target) + if err != nil || string(data) != "png" || !strings.Contains(out, "10x20") { + t.Fatalf("screenshot data=%q err=%v out=%s", data, err, out) + } + if _, _, err := executeCLI(t, deps, "browser", "screenshot", target); err == nil || !strings.Contains(err.Error(), "refusing to overwrite") { + t.Fatalf("overwrite error = %v", err) + } +} + +func TestBrowserRequiresSessionAndValidWait(t *testing.T) { + t.Setenv("AO_SESSION_ID", "") + if _, _, err := executeCLI(t, Deps{}, "browser", "status"); ExitCode(err) != 2 { + t.Fatalf("status error = %v code=%d", err, ExitCode(err)) + } + t.Setenv("AO_SESSION_ID", "ao-1") + if _, _, err := executeCLI(t, Deps{}, "browser", "wait", "--text", "x", "--url", "y"); ExitCode(err) != 2 { + t.Fatalf("wait error = %v code=%d", err, ExitCode(err)) + } +} diff --git a/backend/internal/cli/root.go b/backend/internal/cli/root.go index 0dbd9918a0..3f2f823d4b 100644 --- a/backend/internal/cli/root.go +++ b/backend/internal/cli/root.go @@ -188,6 +188,7 @@ func NewRootCommand(deps Deps) *cobra.Command { root.AddCommand(newSpawnCommand(ctx)) root.AddCommand(newSendCommand(ctx)) root.AddCommand(newPreviewCommand(ctx)) + root.AddCommand(newBrowserCommand(ctx)) root.AddCommand(newHooksCommand(ctx)) root.AddCommand(newLaunchCommand(ctx)) root.AddCommand(newPtyHostCommand()) diff --git a/backend/internal/daemon/daemon.go b/backend/internal/daemon/daemon.go index ab395516d3..c261058eb7 100644 --- a/backend/internal/daemon/daemon.go +++ b/backend/internal/daemon/daemon.go @@ -14,6 +14,7 @@ import ( "time" "github.com/aoagents/agent-orchestrator/backend/internal/adapters/runtime/runtimeselect" + "github.com/aoagents/agent-orchestrator/backend/internal/browserruntime" "github.com/aoagents/agent-orchestrator/backend/internal/config" "github.com/aoagents/agent-orchestrator/backend/internal/daemon/supervisor" "github.com/aoagents/agent-orchestrator/backend/internal/domain" @@ -168,6 +169,7 @@ func Run() error { DefaultPort: mobilebridge.DefaultPort, } mc := &controllers.MobileController{Bridge: bs} + browserBroker := browserruntime.New(log) // Push-device registry: persisted phones that receive OS push notifications. // A load failure must not block boot — degrade to no push rather than refusing @@ -216,6 +218,7 @@ func Run() error { Activity: lcStack.LCM, Telemetry: telemetrySink, Mobile: mc, + Browser: browserBroker, }) if err != nil { stop() @@ -226,6 +229,16 @@ func Run() error { return err } previewDone := preview.NewPoller(store, sessionSvc, "http://"+srv.Addr().String(), preview.PollerConfig{Logger: log}).Start(ctx) + if ln, addr, err := browserruntime.Listen(cfg.RunFilePath); err != nil { + log.Warn("browser runtime: listener unavailable; agent browser control disabled", "err", err) + } else { + log.Info("browser runtime: listening", "addr", addr) + go func() { + if err := browserBroker.Serve(ctx, ln); err != nil { + log.Warn("browser runtime: serve stopped with error", "err", err) + } + }() + } // Late-bind: the LAN listener shares the exact loopback router instance so // the LAN surface and loopback surface never drift apart. diff --git a/backend/internal/httpd/api.go b/backend/internal/httpd/api.go index 30e2eaddf2..69a46e7558 100644 --- a/backend/internal/httpd/api.go +++ b/backend/internal/httpd/api.go @@ -34,6 +34,7 @@ type APIDeps struct { Events cdcSubscriber Telemetry ports.EventSink Mobile *controllers.MobileController + Browser controllers.BrowserRuntime } // API owns one controller per resource and is the single Register call the @@ -49,6 +50,7 @@ type API struct { push *controllers.PushController imports *controllers.ImportController dev *controllers.DevController + browser *controllers.BrowserController events *EventsController } @@ -74,6 +76,7 @@ func NewAPI(cfg config.Config, deps APIDeps) *API { push: &controllers.PushController{Registry: deps.Push}, imports: &controllers.ImportController{Svc: deps.Import}, dev: &controllers.DevController{Import: deps.DevImport}, + browser: &controllers.BrowserController{Runtime: deps.Browser, Sessions: deps.Sessions}, events: &EventsController{Source: deps.CDC, Live: deps.Events}, } } @@ -101,6 +104,7 @@ func (a *API) Register(root chi.Router) { a.push.Register(r) a.imports.Register(r) a.dev.Register(r) + a.browser.Register(r) // Sibling REST controllers plug in here. }) // Long-lived streams intentionally bypass the REST timeout middleware. diff --git a/backend/internal/httpd/apispec/openapi.yaml b/backend/internal/httpd/apispec/openapi.yaml index 07de1ea912..0e19a7d188 100644 --- a/backend/internal/httpd/apispec/openapi.yaml +++ b/backend/internal/httpd/apispec/openapi.yaml @@ -97,6 +97,99 @@ paths: summary: Refresh the cached local agent adapter catalog tags: - agents + /api/v1/browser/commands: + post: + operationId: executeBrowserCommand + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/BrowserCommandRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/BrowserCommandResponse' + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Bad Request + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Found + "409": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Conflict + "422": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Unprocessable Entity + "501": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Implemented + "503": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Service Unavailable + summary: Execute a target-scoped command in a session's desktop browser + tags: + - browser + /api/v1/browser/status: + get: + operationId: getBrowserStatus + parameters: + - description: AO session identifier. + in: query + name: sessionId + schema: + description: AO session identifier. + type: string + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/BrowserStatusResponse' + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Bad Request + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Found + "501": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Implemented + summary: Check whether the desktop browser runtime is connected for a session + tags: + - browser /api/v1/dev/import-projects: post: operationId: runDevImportProjects @@ -1966,6 +2059,50 @@ components: - id - label type: object + BrowserCommandRequest: + properties: + action: + type: string + args: + additionalProperties: {} + type: object + sessionId: + type: string + required: + - sessionId + - action + type: object + BrowserCommandResponse: + properties: + action: + type: string + requestId: + type: string + result: {} + sessionId: + type: string + required: + - requestId + - sessionId + - action + - result + type: object + BrowserStatusResponse: + properties: + connected: + type: boolean + connectedAt: + format: date-time + type: string + sessionId: + type: string + transport: + type: string + required: + - sessionId + - connected + - transport + type: object CancelReviewResponse: properties: reviewerHandleId: @@ -3404,3 +3541,5 @@ tags: name: dev - description: Connect Mobile LAN bridge control (loopback/desktop only) name: mobile +- description: Target-isolated desktop browser runtime (loopback only) + name: browser diff --git a/backend/internal/httpd/apispec/specgen/build.go b/backend/internal/httpd/apispec/specgen/build.go index 3ccfd0b802..f10805384f 100644 --- a/backend/internal/httpd/apispec/specgen/build.go +++ b/backend/internal/httpd/apispec/specgen/build.go @@ -77,6 +77,8 @@ func Build() ([]byte, error) { "Developer-only maintenance operations"), *(&openapi31.Tag{Name: "mobile"}).WithDescription( "Connect Mobile LAN bridge control (loopback/desktop only)"), + *(&openapi31.Tag{Name: "browser"}).WithDescription( + "Target-isolated desktop browser runtime (loopback only)"), } for _, op := range operations() { @@ -153,6 +155,10 @@ var schemaNames = map[string]string{ "ControllersSessionResponse": "SessionResponse", "ControllersSessionPreviewResponse": "SessionPreviewResponse", "ControllersSetSessionPreviewRequest": "SetSessionPreviewRequest", + "ControllersBrowserStatusQuery": "BrowserStatusQuery", + "ControllersBrowserStatusResponse": "BrowserStatusResponse", + "ControllersBrowserCommandRequest": "BrowserCommandRequest", + "ControllersBrowserCommandResponse": "BrowserCommandResponse", "ControllersRenameSessionRequest": "RenameSessionRequest", "ControllersRenameSessionResponse": "RenameSessionResponse", "ControllersRestoreSessionResponse": "RestoreSessionResponse", @@ -322,9 +328,40 @@ func operations() []operation { ops = append(ops, importOperations()...) ops = append(ops, devOperations()...) ops = append(ops, mobileOperations()...) + ops = append(ops, browserOperations()...) return ops } +func browserOperations() []operation { + return []operation{ + { + method: http.MethodGet, path: "/api/v1/browser/status", id: "getBrowserStatus", tag: "browser", + summary: "Check whether the desktop browser runtime is connected for a session", + pathParams: []any{controllers.BrowserStatusQuery{}}, + resps: []respUnit{ + {http.StatusOK, controllers.BrowserStatusResponse{}}, + {http.StatusBadRequest, envelope.APIError{}}, + {http.StatusNotFound, envelope.APIError{}}, + {http.StatusNotImplemented, envelope.APIError{}}, + }, + }, + { + method: http.MethodPost, path: "/api/v1/browser/commands", id: "executeBrowserCommand", tag: "browser", + summary: "Execute a target-scoped command in a session's desktop browser", + reqBody: controllers.BrowserCommandRequest{}, + resps: []respUnit{ + {http.StatusOK, controllers.BrowserCommandResponse{}}, + {http.StatusBadRequest, envelope.APIError{}}, + {http.StatusNotFound, envelope.APIError{}}, + {http.StatusConflict, envelope.APIError{}}, + {http.StatusUnprocessableEntity, envelope.APIError{}}, + {http.StatusServiceUnavailable, envelope.APIError{}}, + {http.StatusNotImplemented, envelope.APIError{}}, + }, + }, + } +} + func agentOperations() []operation { return []operation{ { diff --git a/backend/internal/httpd/controllers/browser.go b/backend/internal/httpd/controllers/browser.go new file mode 100644 index 0000000000..26050e346c --- /dev/null +++ b/backend/internal/httpd/controllers/browser.go @@ -0,0 +1,130 @@ +package controllers + +import ( + "context" + "errors" + "net/http" + "strings" + + "github.com/go-chi/chi/v5" + + "github.com/aoagents/agent-orchestrator/backend/internal/browserruntime" + "github.com/aoagents/agent-orchestrator/backend/internal/domain" + "github.com/aoagents/agent-orchestrator/backend/internal/httpd/apispec" + "github.com/aoagents/agent-orchestrator/backend/internal/httpd/envelope" +) + +var browserActions = map[string]struct{}{ + "open": {}, + "snapshot": {}, + "click": {}, + "fill": {}, + "wait": {}, + "screenshot": {}, + "console": {}, + "errors": {}, +} + +type BrowserRuntime interface { + Status() browserruntime.Status + Execute(ctx context.Context, sessionID domain.SessionID, action string, args map[string]interface{}) (browserruntime.Result, error) +} + +type BrowserSessionReader interface { + Get(ctx context.Context, id domain.SessionID) (domain.Session, error) +} + +type BrowserController struct { + Runtime BrowserRuntime + Sessions BrowserSessionReader +} + +func (c *BrowserController) Register(r chi.Router) { + r.Get("/browser/status", c.status) + r.Post("/browser/commands", c.execute) +} + +func (c *BrowserController) status(w http.ResponseWriter, r *http.Request) { + if c.Runtime == nil || c.Sessions == nil { + apispec.NotImplemented(w, r, http.MethodGet, "/api/v1/browser/status") + return + } + sessionID := domain.SessionID(strings.TrimSpace(r.URL.Query().Get("sessionId"))) + if sessionID == "" { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "SESSION_ID_REQUIRED", "sessionId is required", nil) + return + } + if _, err := c.Sessions.Get(r.Context(), sessionID); err != nil { + envelope.WriteError(w, r, err) + return + } + status := c.Runtime.Status() + envelope.WriteJSON(w, http.StatusOK, BrowserStatusResponse{ + SessionID: sessionID, + Connected: status.Connected, + ConnectedAt: status.ConnectedAt, + Transport: "electron-webcontents-debugger", + }) +} + +func (c *BrowserController) execute(w http.ResponseWriter, r *http.Request) { + if c.Runtime == nil || c.Sessions == nil { + apispec.NotImplemented(w, r, http.MethodPost, "/api/v1/browser/commands") + return + } + var in BrowserCommandRequest + if err := decodeJSON(r, &in); err != nil { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "INVALID_JSON", "Invalid JSON body", nil) + return + } + in.Action = strings.ToLower(strings.TrimSpace(in.Action)) + if in.SessionID == "" { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "SESSION_ID_REQUIRED", "sessionId is required", nil) + return + } + if _, ok := browserActions[in.Action]; !ok { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "BROWSER_ACTION_UNSUPPORTED", "Unsupported browser action", nil) + return + } + if _, err := c.Sessions.Get(r.Context(), in.SessionID); err != nil { + envelope.WriteError(w, r, err) + return + } + result, err := c.Runtime.Execute(r.Context(), in.SessionID, in.Action, in.Args) + if err != nil { + writeBrowserError(w, r, err) + return + } + envelope.WriteJSON(w, http.StatusOK, BrowserCommandResponse{ + RequestID: result.RequestID, + SessionID: in.SessionID, + Action: in.Action, + Result: result.Value, + }) +} + +func writeBrowserError(w http.ResponseWriter, r *http.Request, err error) { + if errors.Is(err, browserruntime.ErrUnavailable) { + envelope.WriteAPIError(w, r, http.StatusServiceUnavailable, "unavailable", "BROWSER_RUNTIME_UNAVAILABLE", "Desktop browser runtime is not connected", nil) + return + } + var commandErr browserruntime.CommandError + if errors.As(err, &commandErr) { + status := http.StatusUnprocessableEntity + typeName := "unprocessable" + switch commandErr.Code { + case "INVALID_ARGUMENT", "URL_REQUIRED", "REFERENCE_REQUIRED": + status = http.StatusBadRequest + typeName = "bad_request" + case "STALE_REFERENCE": + status = http.StatusConflict + typeName = "conflict" + case "BROWSER_TARGET_UNAVAILABLE": + status = http.StatusServiceUnavailable + typeName = "unavailable" + } + envelope.WriteAPIError(w, r, status, typeName, commandErr.Code, commandErr.Message, nil) + return + } + envelope.WriteError(w, r, err) +} diff --git a/backend/internal/httpd/controllers/browser_test.go b/backend/internal/httpd/controllers/browser_test.go new file mode 100644 index 0000000000..f9e40f46cd --- /dev/null +++ b/backend/internal/httpd/controllers/browser_test.go @@ -0,0 +1,98 @@ +package controllers_test + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/aoagents/agent-orchestrator/backend/internal/browserruntime" + "github.com/aoagents/agent-orchestrator/backend/internal/config" + "github.com/aoagents/agent-orchestrator/backend/internal/domain" + "github.com/aoagents/agent-orchestrator/backend/internal/httpd" +) + +type fakeBrowserRuntime struct { + status browserruntime.Status + action string + args map[string]interface{} + err error +} + +func (f *fakeBrowserRuntime) Status() browserruntime.Status { return f.status } + +func (f *fakeBrowserRuntime) Execute( + _ context.Context, + _ domain.SessionID, + action string, + args map[string]interface{}, +) (browserruntime.Result, error) { + f.action, f.args = action, args + if f.err != nil { + return browserruntime.Result{}, f.err + } + return browserruntime.Result{RequestID: "request-1", Value: map[string]interface{}{"text": "button Save [ref=e1]"}}, nil +} + +func browserServer(t *testing.T, runtime *fakeBrowserRuntime) *httptest.Server { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + srv := httptest.NewServer(httpd.NewRouterWithControl(config.Config{}, log, nil, httpd.APIDeps{ + Sessions: newFakeSessionService(), + Browser: runtime, + }, httpd.ControlDeps{})) + t.Cleanup(srv.Close) + return srv +} + +func TestBrowserStatusAndSnapshot(t *testing.T) { + connectedAt := time.Now().UTC().Truncate(time.Second) + runtime := &fakeBrowserRuntime{status: browserruntime.Status{Connected: true, ConnectedAt: connectedAt}} + srv := browserServer(t, runtime) + + body, status, _ := doRequest(t, srv, http.MethodGet, "/api/v1/browser/status?sessionId=ao-1", "") + if status != http.StatusOK || !containsAll(body, `"connected":true`, `"transport":"electron-webcontents-debugger"`) { + t.Fatalf("status = %d body=%s", status, body) + } + body, status, _ = doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"snapshot","args":{"interactive":true}}`) + if status != http.StatusOK || !containsAll(body, `"requestId":"request-1"`, `"button Save [ref=e1]"`) { + t.Fatalf("command = %d body=%s", status, body) + } + if runtime.action != "snapshot" || runtime.args["interactive"] != true { + t.Fatalf("runtime command = %q %#v", runtime.action, runtime.args) + } +} + +func TestBrowserCommandValidationAndErrors(t *testing.T) { + runtime := &fakeBrowserRuntime{} + srv := browserServer(t, runtime) + + body, status, _ := doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"eval"}`) + if status != http.StatusBadRequest || !containsAll(body, `"code":"BROWSER_ACTION_UNSUPPORTED"`) { + t.Fatalf("unsupported = %d body=%s", status, body) + } + runtime.err = browserruntime.ErrUnavailable + body, status, _ = doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"snapshot"}`) + if status != http.StatusServiceUnavailable || !containsAll(body, `"code":"BROWSER_RUNTIME_UNAVAILABLE"`) { + t.Fatalf("unavailable = %d body=%s", status, body) + } + runtime.err = browserruntime.CommandError{Code: "STALE_REFERENCE", Message: "snapshot again"} + body, status, _ = doRequest(t, srv, http.MethodPost, "/api/v1/browser/commands", `{"sessionId":"ao-1","action":"click"}`) + if status != http.StatusConflict || !containsAll(body, `"code":"STALE_REFERENCE"`) { + t.Fatalf("stale = %d body=%s", status, body) + } +} + +func containsAll(body []byte, parts ...string) bool { + value := string(body) + for _, part := range parts { + if !strings.Contains(value, part) { + return false + } + } + return true +} diff --git a/backend/internal/httpd/controllers/dto.go b/backend/internal/httpd/controllers/dto.go index c9f1fb0f70..9b33b87677 100644 --- a/backend/internal/httpd/controllers/dto.go +++ b/backend/internal/httpd/controllers/dto.go @@ -221,6 +221,37 @@ type SetSessionPreviewRequest struct { URL string `json:"url,omitempty" description:"Preview target URL. When empty, the daemon autodetects a static entry point in the session workspace."` } +// BrowserStatusQuery selects the session whose logical browser is inspected. +type BrowserStatusQuery struct { + SessionID domain.SessionID `query:"sessionId" description:"AO session identifier."` +} + +// BrowserStatusResponse reports whether the desktop-owned browser transport is +// ready. A connected runtime can create the session target while its panel is +// hidden; panel visibility is intentionally not part of this state. +type BrowserStatusResponse struct { + SessionID domain.SessionID `json:"sessionId"` + Connected bool `json:"connected"` + ConnectedAt time.Time `json:"connectedAt,omitempty"` + Transport string `json:"transport"` +} + +// BrowserCommandRequest is the stable daemon-facing command envelope. Action +// arguments remain action-specific JSON so new target-scoped operations do not +// require a new transport or Electron IPC surface. +type BrowserCommandRequest struct { + SessionID domain.SessionID `json:"sessionId"` + Action string `json:"action"` + Args map[string]interface{} `json:"args,omitempty"` +} + +type BrowserCommandResponse struct { + RequestID string `json:"requestId"` + SessionID domain.SessionID `json:"sessionId"` + Action string `json:"action"` + Result interface{} `json:"result"` +} + // RenameSessionResponse is the body of PATCH /api/v1/sessions/{sessionId}. type RenameSessionResponse struct { OK bool `json:"ok"` diff --git a/backend/internal/httpd/lan_listener.go b/backend/internal/httpd/lan_listener.go index 56410b0961..cf5feabf92 100644 --- a/backend/internal/httpd/lan_listener.go +++ b/backend/internal/httpd/lan_listener.go @@ -56,6 +56,7 @@ var lanControlBlockedPrefixes = []string{ "/internal/", "/api/v1/mobile", "/api/v1/dev", + "/api/v1/browser", } // lanControlBlock returns 404 for any request whose path is, or is nested diff --git a/backend/internal/httpd/lan_listener_test.go b/backend/internal/httpd/lan_listener_test.go index 13bb448eec..61859b8fac 100644 --- a/backend/internal/httpd/lan_listener_test.go +++ b/backend/internal/httpd/lan_listener_test.go @@ -44,10 +44,10 @@ func TestLANManagerAuthGatesSharedHandler(t *testing.T) { } // TestLANManagerBlocksLoopbackOnlyControlRoutes proves the LAN listener never -// serves /shutdown, /internal/*, /api/v1/mobile*, or developer maintenance -// routes — even when the request carries a spoofed Host: 127.0.0.1 and valid LAN -// auth, since gating on Host alone (localControlRequest) is what let a LAN -// client reach these routes. +// serves /shutdown, /internal/*, /api/v1/mobile*, /api/v1/dev*, or +// /api/v1/browser* — even when the request carries a spoofed Host: 127.0.0.1 +// and valid LAN auth, since gating on Host alone (localControlRequest) is what +// let a LAN client reach these routes. func TestLANManagerBlocksLoopbackOnlyControlRoutes(t *testing.T) { inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "ok") @@ -66,6 +66,7 @@ func TestLANManagerBlocksLoopbackOnlyControlRoutes(t *testing.T) { "/internal/telemetry/cli-invoked", "/api/v1/mobile/status", "/api/v1/dev/import-projects", + "/api/v1/browser/status", } for _, path := range blocked { req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d%s", port, path), nil) diff --git a/backend/internal/session_manager/manager.go b/backend/internal/session_manager/manager.go index 528a6cf8ff..f1dcc3f435 100644 --- a/backend/internal/session_manager/manager.go +++ b/backend/internal/session_manager/manager.go @@ -2008,8 +2008,13 @@ func (m *Manager) aoSkillPointer() string { dir := skillassets.Dir(m.dataDir) skillFile := filepath.Join(dir, "SKILL.md") commandsGlob := filepath.Join(dir, "commands", "*.md") + browserFile := filepath.Join(dir, "commands", "browser.md") return "\n\n" + "## Using the ao CLI\n\n" + - "When you need to use the `ao` CLI, read `" + skillFile + "` first (and the relevant `" + commandsGlob + "`) for the full command catalog, flags, and examples." + "When you need to use the `ao` CLI, read `" + skillFile + "` first (and the relevant `" + commandsGlob + "`) for the full command catalog, flags, and examples.\n\n" + + "## AO desktop Browser panel\n\n" + + "When the user asks you to inspect, test, click, or type in the page shown in AO's desktop Browser panel, read `" + browserFile + "` and use `ao browser` from this AO session. " + + "Do not use Codex/host in-app browser connectors, `agent.browsers.get(\"iab\")`, or a browser MCP for the AO Browser panel: those are separate browser runtimes and cannot see or control AO's session-owned page. " + + "`ao browser` deliberately operates the same live page the user sees in that panel." } func (m *Manager) workspaceProjectPrompt(ctx context.Context, kind domain.SessionKind, projectID domain.ProjectID) (string, error) { diff --git a/backend/internal/session_manager/manager_test.go b/backend/internal/session_manager/manager_test.go index 3764048d54..6127d204b0 100644 --- a/backend/internal/session_manager/manager_test.go +++ b/backend/internal/session_manager/manager_test.go @@ -2136,7 +2136,10 @@ func TestSpawnOrchestrator_UsesCoordinatorPrompt(t *testing.T) { "`ao session ls --project mer`", "`ao session get `", "Delegate implementation, fixes, tests, and PR ownership to worker sessions", - "skills/using-ao/SKILL.md", + filepath.Join("skills", "using-ao", "SKILL.md"), + "AO desktop Browser panel", + "agent.browsers.get(\"iab\")", + "same live page the user sees", } { if !strings.Contains(systemPrompt, want) { t.Fatalf("system prompt missing %q:\n%s", want, systemPrompt) @@ -2283,9 +2286,12 @@ func TestSystemPrompt_AppendsConfidentialityGuard(t *testing.T) { if !strings.Contains(sp, "role boundaries, delegation policy, CI/review follow-up expectations, PR/MR workflow when applicable, and privacy rules") { t.Fatalf("%s: system prompt missing generic behavior categories:\n%s", tc.name, sp) } - if !strings.Contains(sp, "skills/using-ao/SKILL.md") { + if !strings.Contains(sp, filepath.Join("skills", "using-ao", "SKILL.md")) { t.Fatalf("%s: system prompt missing using-ao skill pointer:\n%s", tc.name, sp) } + if !strings.Contains(sp, "AO desktop Browser panel") || !strings.Contains(sp, "agent.browsers.get(\"iab\")") { + t.Fatalf("%s: system prompt missing AO browser routing guidance:\n%s", tc.name, sp) + } }) } } diff --git a/backend/internal/skillassets/skillassets_test.go b/backend/internal/skillassets/skillassets_test.go index 13ea531538..211bb58dd8 100644 --- a/backend/internal/skillassets/skillassets_test.go +++ b/backend/internal/skillassets/skillassets_test.go @@ -59,6 +59,9 @@ func TestInstall_WritesSkillAndIsIdempotent(t *testing.T) { if _, err := os.Stat(filepath.Join(Dir(dataDir), "commands", "spawn.md")); err != nil { t.Fatalf("commands/spawn.md missing: %v", err) } + if _, err := os.Stat(filepath.Join(Dir(dataDir), "commands", "browser.md")); err != nil { + t.Fatalf("commands/browser.md missing: %v", err) + } // A stale file inside the skill dir must not survive a reinstall (clobber). stale := filepath.Join(Dir(dataDir), "stale.md") diff --git a/backend/internal/skillassets/using-ao/SKILL.md b/backend/internal/skillassets/using-ao/SKILL.md index 758ebbe1e3..4717f3029b 100644 --- a/backend/internal/skillassets/using-ao/SKILL.md +++ b/backend/internal/skillassets/using-ao/SKILL.md @@ -1,7 +1,7 @@ --- name: using-ao -description: "Catalog of the AO (Agent Orchestrator) `ao` CLI: spawning workers, managing sessions and projects, sending messages, previewing pages, and daemon control. Use when using the ao CLI, spawning workers, or managing AO sessions in an AO workspace." -trigger: "Using the ao CLI in an AO workspace: spawning workers, managing sessions/projects, sending messages, previewing pages." +description: "Catalog of the AO (Agent Orchestrator) `ao` CLI: spawning workers, managing sessions and projects, sending messages, controlling the shared browser, previewing pages, and daemon control. Use when using the ao CLI, spawning workers, or managing AO sessions in an AO workspace." +trigger: "Using the ao CLI in an AO workspace: spawning workers, managing sessions/projects, sending messages, controlling or previewing pages." --- # AO CLI Catalog @@ -17,6 +17,7 @@ trigger: "Using the ao CLI in an AO workspace: spawning workers, managing sessio | `review` | Submit a reviewer result for a worker's PR | Completing a code review loop | [commands/review.md](commands/review.md) | | `send` | Send a message to a running agent session | Correcting or directing a live agent | [commands/send.md](commands/send.md) | | `preview` | Open a URL in the desktop browser panel | Demoing a local server or file from inside a session | [commands/preview.md](commands/preview.md) | +| `browser` | Inspect and control the session's shared live browser | Verifying a web app through snapshots, interactions, waits, screenshots, console, and errors | [commands/browser.md](commands/browser.md) | | `start` | Fetch (if needed) and open the AO desktop app | Launching the app | [commands/start.md](commands/start.md) | | `stop` | Stop the AO daemon | Shutting down AO | [commands/stop.md](commands/stop.md) | | `status` | Show daemon status | Verifying the daemon is up and healthy | [commands/status.md](commands/status.md) | @@ -32,5 +33,6 @@ trigger: "Using the ao CLI in an AO workspace: spawning workers, managing sessio - Session and project ids are shown by `ao session ls` and `ao project ls`. - `--agent` is an alias for `--harness` on `ao spawn`. - Every command accepts `-h / --help` for the full flag list. +- For frontend work, prefer `ao browser`: check status, open the app, snapshot, interact, wait for the update, snapshot again, and inspect `ao browser errors` before completing. See [references.md](references.md) for natural-language-to-command mappings. diff --git a/backend/internal/skillassets/using-ao/commands/browser.md b/backend/internal/skillassets/using-ao/commands/browser.md new file mode 100644 index 0000000000..3507130b46 --- /dev/null +++ b/backend/internal/skillassets/using-ao/commands/browser.md @@ -0,0 +1,40 @@ +# ao browser + +Inspect and control the current AO session's target-isolated browser. The desktop app must be open. The agent and user share the same live page, cookies, navigation state, and `WebContentsView`; the runtime remains usable while the Browser panel is hidden. + +`AO_SESSION_ID` selects the target, so run these commands from inside an AO worker session. + +This is the automation interface for AO's visible desktop Browser panel. Do not use Codex/host in-app browser connectors, `agent.browsers.get("iab")`, or a browser MCP for this panel: those belong to separate browser runtimes and will not discover or update AO's session-owned page. + +## Core workflow + +```bash +ao browser status +ao browser open http://localhost:5173 +ao browser snapshot +ao browser click e1 +ao browser fill e2 "hello" +ao browser wait --text "Saved" +ao browser snapshot +ao browser errors +``` + +Element references such as `e1` are short-lived. After navigation or a substantial DOM replacement, take another snapshot. A stale reference fails explicitly and never falls through to another session or page. + +## Commands + +```text +ao browser status [--json] +ao browser open [--json] +ao browser snapshot [--interactive] [--json] +ao browser click [--json] +ao browser fill [--json] +ao browser wait (--text | --selector | --url | --ms ) [--timeout ] [--json] +ao browser screenshot [path] [--json] +ao browser console [--json] +ao browser errors [--json] +``` + +Without `--json`, `screenshot` writes a PNG and refuses to overwrite an existing file. With `--json`, it returns the structured response including base64 image data. + +`ao preview` remains available for the passive URL/static-file workflow. Use `ao browser` when the agent needs to inspect or verify the page. diff --git a/backend/internal/skillassets/using-ao/references.md b/backend/internal/skillassets/using-ao/references.md index ef9fc5507b..fe7515cd44 100644 --- a/backend/internal/skillassets/using-ao/references.md +++ b/backend/internal/skillassets/using-ao/references.md @@ -5,6 +5,10 @@ Natural-language-to-command mappings for common AO tasks. | You want to... | Command | |---|---| | Show me this webpage / open this page | `ao preview ""` | +| Inspect and verify this webpage as the agent | `ao browser open ""`, then `ao browser snapshot` | +| Click or fill a page element | `ao browser snapshot`, then `ao browser click ` or `ao browser fill ""` | +| Check frontend runtime failures | `ao browser errors` and `ao browser console` | +| Capture the page | `ao browser screenshot [path]` | | Spawn a worker on issue N | `ao spawn --project

--issue N --name "<=20 chars>" --prompt "..."` | | Message a running agent | `ao send --session --message "..."` | | Kill a session | `ao session kill ` | diff --git a/docs/STATUS.md b/docs/STATUS.md index 7281d2f987..07a99127d8 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -61,6 +61,11 @@ surface (`npm run sqlc`, `npm run api`). ### Frontend (Electron + React) - Electron + React 19 + TanStack Router/Query + Tailwind + shadcn primitives. +- Target-isolated per-session browser-control spike: a dedicated local + daemon↔Electron bridge drives only the selected session's `WebContentsView` + through Electron's bound debugger transport. `ao browser` supports open, + compact accessibility snapshots and refs, click/fill, waits, screenshots, + console messages, and page errors while the Browser panel is hidden. - Real daemon wiring via the generated `openapi-fetch` typed client (`src/api/schema.ts`); mock data only in `VITE_NO_ELECTRON` web-preview mode. - Electron main handles daemon discovery, launch, and status reporting. diff --git a/docs/architecture.md b/docs/architecture.md index 91fb1ffe1f..0dc4f59ab8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -15,6 +15,7 @@ Agent Orchestrator is a long-running Go daemon that supervises multiple parallel - [Observation Loops](#observation-loops) - [HTTP Layer](#http-layer) - [Terminal Multiplexing](#terminal-multiplexing) +- [Browser Runtime Bridge](#browser-runtime-bridge) --- @@ -832,6 +833,19 @@ sequenceDiagram Mux->>Runtime: Close PTY ``` +## Browser Runtime Bridge + +Browser automation uses a dedicated local socket (`browser.sock` on Unix, +`ao-browser[-dev]` named pipe on Windows) between the daemon and Electron. The +daemon owns command authorization/correlation; Electron owns the actual browser +targets. Commands never use the supervisor liveness socket and never enable an +unauthenticated remote-debugging port. + +Electron attaches its debugger directly to the selected session's +`WebContentsView`, so the protocol transport cannot enumerate or attach to the +AO renderer or a different session. The loopback `/api/v1/browser` surface is +blocked entirely on the opt-in LAN listener. + --- ## Load-Bearing Rules diff --git a/docs/cli/README.md b/docs/cli/README.md index df0f09dcb4..dcbdd5fbe0 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -56,6 +56,7 @@ Every product command resolves to a daemon HTTP route. Run `ao | `ao orchestrator ls` | `GET /api/v1/orchestrators` | | `ao send` | `POST /api/v1/sessions/{id}/send` | | `ao preview [url]` | `POST /api/v1/sessions/{id}/preview` | +| `ao browser ...` | `GET /api/v1/browser/status`, `POST /api/v1/browser/commands` | | `ao hooks ` | `POST /api/v1/sessions/{id}/activity` (hidden) | `ao agent ls` prints the daemon-supported agent catalog with local install/auth @@ -80,6 +81,14 @@ spawn remains the authoritative runtime validation point. Use autodetects an `index.html` in the session workspace; with a URL argument it opens that URL verbatim (`file://`, `http`, `https`). +`ao browser` also resolves its target from `AO_SESSION_ID`, but controls the +session-owned live Electron browser rather than only setting its preview URL. +The initial target-isolated command set is `status`, `open`, `snapshot`, +`click`, `fill`, `wait`, `screenshot`, `console`, and `errors`. The AO desktop +app must be open because Electron owns the `WebContentsView`. References from a +snapshot are invalidated after navigation or DOM replacement; take another +snapshot when a command reports `STALE_REFERENCE`. + `go run .` in `backend/` remains a compatibility wrapper around the daemon. PR and review actions (merge, resolve-comments, review execute/send) are diff --git a/frontend/src/api/schema.ts b/frontend/src/api/schema.ts index 03c95adbed..c30d3f46db 100644 --- a/frontend/src/api/schema.ts +++ b/frontend/src/api/schema.ts @@ -55,6 +55,40 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/browser/commands": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Execute a target-scoped command in a session's desktop browser */ + post: operations["executeBrowserCommand"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/browser/status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Check whether the desktop browser runtime is connected for a session */ + get: operations["getBrowserStatus"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/dev/import-projects": { parameters: { query?: never; @@ -759,6 +793,26 @@ export interface components { id: string; label: string; }; + BrowserCommandRequest: { + action: string; + args?: { + [key: string]: unknown; + }; + sessionId: string; + }; + BrowserCommandResponse: { + action: string; + requestId: string; + result: unknown; + sessionId: string; + }; + BrowserStatusResponse: { + connected: boolean; + /** Format: date-time */ + connectedAt?: string; + sessionId: string; + transport: string; + }; CancelReviewResponse: { reviewerHandleId: string; reviews: components["schemas"]["PRReviewState"][]; @@ -1435,6 +1489,134 @@ export interface operations { }; }; }; + executeBrowserCommand: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BrowserCommandRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BrowserCommandResponse"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Conflict */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Implemented */ + 501: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Service Unavailable */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + }; + }; + getBrowserStatus: { + parameters: { + query?: { + /** @description AO session identifier. */ + sessionId?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BrowserStatusResponse"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Implemented */ + 501: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + }; + }; runDevImportProjects: { parameters: { query?: never; diff --git a/frontend/src/main.ts b/frontend/src/main.ts index 3164f8de4e..ce348c8693 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -51,6 +51,7 @@ import { DEFAULT_POSTHOG_HOST, DEFAULT_POSTHOG_PROJECT_KEY } from "./shared/post import { buildTelemetryBootstrap } from "./shared/telemetry"; import { createBrowserViewHost, type BrowserViewHost } from "./main/browser-view-host"; import { connectSupervisor, type SupervisorLinkHandle } from "./main/supervisor-link"; +import { connectBrowserRuntime, type BrowserRuntimeLinkHandle } from "./main/browser-runtime-link"; import { keepDaemonAlive, shouldLinkOnAttach } from "./main/daemon-owner"; import { readMigrationState, updateMigration, writeAppStateMarker, type MigrationState } from "./main/app-state"; import { isAllowedAppExternalURL, openAllowedAppExternalURL } from "./main/external-open"; @@ -97,6 +98,7 @@ let daemonStartPromise: Promise | null = null; let daemonStartEpoch = 0; let daemonStatus: DaemonStatus = { state: "stopped" }; let browserViewHost: BrowserViewHost | null = null; +let browserRuntimeLink: BrowserRuntimeLinkHandle | null = null; // Held for the app lifetime. Dropping it (on any exit) triggers daemon self-stop. let supervisorLink: SupervisorLinkHandle | null = null; @@ -233,6 +235,9 @@ function applyRuntimeAppIcon(): void { function setDaemonStatus(nextStatus: DaemonStatus): void { daemonStatus = nextStatus; mainWindow?.webContents.send("daemon:status", daemonStatus); + if (nextStatus.state === "ready" && browserViewHost) { + establishBrowserRuntimeLink(); + } } // Role-based menu installed on Windows where the native menu bar is hidden. The @@ -354,6 +359,7 @@ function createWindow(): void { rendererOrigin: RENDERER_ORIGIN, isMac, }); + if (daemonStatus.state === "ready") establishBrowserRuntimeLink(); void mainWindow.loadURL(rendererUrl()); @@ -364,6 +370,8 @@ function createWindow(): void { } mainWindow.on("closed", () => { + browserRuntimeLink?.dispose(); + browserRuntimeLink = null; browserViewHost?.dispose(); browserViewHost = null; mainWindow = null; @@ -568,6 +576,40 @@ function supervisorPipeFromRunFile(rfp: string | null): string { return "\\\\.\\pipe\\ao-supervise-" + dir.replace(/[^a-zA-Z0-9-]/g, "-"); } +function browserRuntimePipeFromRunFile(rfp: string | null): string { + if (!rfp) return "\\\\.\\pipe\\ao-browser"; + const dir = path.basename(path.dirname(rfp)); + if (dir === ".ao" || dir === "." || dir === "") return "\\\\.\\pipe\\ao-browser"; + return "\\\\.\\pipe\\ao-browser-" + dir.replace(/[^a-zA-Z0-9-]/g, "-"); +} + +function establishBrowserRuntimeLink(): void { + if (!browserViewHost || browserRuntimeLink) return; + const rfp = runFilePath(); + const address = + process.platform === "win32" + ? browserRuntimePipeFromRunFile(rfp) + : rfp + ? path.join(path.dirname(rfp), "browser.sock") + : null; + if (!address) { + console.warn("AO: browser runtime link skipped; run-file path unavailable"); + return; + } + browserRuntimeLink = connectBrowserRuntime(address, { + execute: (command) => { + const host = browserViewHost; + if (!host) { + throw Object.assign(new Error("Browser target owner is unavailable"), { + code: "BROWSER_TARGET_UNAVAILABLE", + }); + } + return host.execute(command.sessionId, command.action, command.args); + }, + log: (message) => console.log(`AO: ${message}`), + }); +} + function establishSupervisorLink(): void { const rfp = runFilePath(); const addr = @@ -1030,6 +1072,8 @@ function stopDaemon(): DaemonStatus { // A later daemon:start re-establishes the link via reportBoundPort. supervisorLink?.dispose(); supervisorLink = null; + browserRuntimeLink?.dispose(); + browserRuntimeLink = null; killDaemon(daemonProcess); setDaemonStatus({ state: "stopped" }); return daemonStatus; @@ -1470,6 +1514,8 @@ app.whenReady().then(async () => { // The supervisorLink fd is NOT explicitly closed on quit; the OS closes it when // the process exits for any reason (Cmd+Q, crash, SIGKILL). Sessions survive. app.on("before-quit", () => { + browserRuntimeLink?.dispose(); + browserRuntimeLink = null; browserViewHost?.dispose(); browserViewHost = null; }); diff --git a/frontend/src/main/browser-runtime-link.test.ts b/frontend/src/main/browser-runtime-link.test.ts new file mode 100644 index 0000000000..d2eb1801ac --- /dev/null +++ b/frontend/src/main/browser-runtime-link.test.ts @@ -0,0 +1,97 @@ +import net from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { connectBrowserRuntime, type BrowserRuntimeLinkHandle } from "./browser-runtime-link"; + +const handles: BrowserRuntimeLinkHandle[] = []; +const servers: net.Server[] = []; + +afterEach(async () => { + handles.splice(0).forEach((handle) => handle.dispose()); + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ), + ); +}); + +describe("browser runtime link", () => { + it("handshakes and correlates a command result", async () => { + const execute = vi.fn(async () => ({ text: "button Save [ref=e1]" })); + let serverSocket: net.Socket | null = null; + let inbound = ""; + const messages: unknown[] = []; + const server = net.createServer((socket) => { + serverSocket = socket; + socket.on("data", (chunk) => { + inbound += chunk.toString("utf8"); + for (;;) { + const newline = inbound.indexOf("\n"); + if (newline < 0) return; + messages.push(JSON.parse(inbound.slice(0, newline))); + inbound = inbound.slice(newline + 1); + } + }); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address() as net.AddressInfo; + const handle = connectBrowserRuntime({ host: address.address, port: address.port }, { execute }); + handles.push(handle); + await vi.waitFor(() => expect(handle.connected).toBe(true)); + await vi.waitFor(() => expect(messages).toContainEqual({ type: "hello", version: 1 })); + + serverSocket!.write( + `${JSON.stringify({ type: "command", requestId: "r1", sessionId: "s1", action: "snapshot", args: {} })}\n`, + ); + + await vi.waitFor(() => expect(execute).toHaveBeenCalledWith(expect.objectContaining({ requestId: "r1" }))); + await vi.waitFor(() => + expect(messages).toContainEqual({ + type: "result", + requestId: "r1", + ok: true, + result: { text: "button Save [ref=e1]" }, + }), + ); + }); + + it("returns structured command errors", async () => { + let serverSocket: net.Socket | null = null; + let inbound = ""; + const messages: unknown[] = []; + const server = net.createServer((socket) => { + serverSocket = socket; + socket.on("data", (chunk) => { + inbound += chunk.toString("utf8"); + const lines = inbound.split("\n"); + inbound = lines.pop() ?? ""; + for (const line of lines) if (line) messages.push(JSON.parse(line)); + }); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address() as net.AddressInfo; + const handle = connectBrowserRuntime( + { host: address.address, port: address.port }, + { + execute: async () => { + throw { code: "STALE_REFERENCE", message: "snapshot again" }; + }, + }, + ); + handles.push(handle); + await vi.waitFor(() => expect(handle.connected).toBe(true)); + serverSocket!.write(`${JSON.stringify({ type: "command", requestId: "r2", sessionId: "s1", action: "click" })}\n`); + await vi.waitFor(() => + expect(messages).toContainEqual({ + type: "result", + requestId: "r2", + ok: false, + error: { code: "STALE_REFERENCE", message: "snapshot again" }, + }), + ); + }); +}); diff --git a/frontend/src/main/browser-runtime-link.ts b/frontend/src/main/browser-runtime-link.ts new file mode 100644 index 0000000000..feb766c4ce --- /dev/null +++ b/frontend/src/main/browser-runtime-link.ts @@ -0,0 +1,171 @@ +import net from "node:net"; + +const PROTOCOL_VERSION = 1; +const BACKOFF_INIT_MS = 200; +const BACKOFF_MAX_MS = 2_000; +const MAX_COMMAND_BYTES = 1 << 20; + +export type BrowserRuntimeCommand = { + type: "command"; + requestId: string; + sessionId: string; + action: string; + args?: Record; +}; + +export type BrowserRuntimeCommandError = { + code: string; + message: string; +}; + +export interface BrowserRuntimeLinkHandle { + readonly connected: boolean; + dispose(): void; +} + +type BrowserRuntimeLinkOptions = { + execute: (command: BrowserRuntimeCommand) => Promise; + log?: (message: string) => void; +}; + +export function connectBrowserRuntime( + address: string | net.TcpNetConnectOpts, + options: BrowserRuntimeLinkOptions, +): BrowserRuntimeLinkHandle { + const log = options.log ?? (() => undefined); + let disposed = false; + let connected = false; + let socket: net.Socket | null = null; + let retryTimer: ReturnType | null = null; + let backoff = BACKOFF_INIT_MS; + let buffer = ""; + let commandChain = Promise.resolve(); + + const clearRetry = () => { + if (retryTimer !== null) { + clearTimeout(retryTimer); + retryTimer = null; + } + }; + + const destroySocket = () => { + if (!socket) return; + socket.removeAllListeners(); + socket.destroy(); + socket = null; + }; + + const send = (message: unknown) => { + if (!socket || socket.destroyed) return; + socket.write(`${JSON.stringify(message)}\n`); + }; + + const respond = async (command: BrowserRuntimeCommand) => { + try { + const result = await options.execute(command); + send({ type: "result", requestId: command.requestId, ok: true, result }); + } catch (error) { + const normalized = normalizeCommandError(error); + send({ type: "result", requestId: command.requestId, ok: false, error: normalized }); + } + }; + + const consumeLine = (line: string) => { + if (!line.trim()) return; + let command: BrowserRuntimeCommand; + try { + command = JSON.parse(line) as BrowserRuntimeCommand; + } catch { + return; + } + if ( + command.type !== "command" || + typeof command.requestId !== "string" || + typeof command.sessionId !== "string" || + typeof command.action !== "string" + ) { + return; + } + commandChain = commandChain.then(() => respond(command)); + }; + + const consume = (chunk: Buffer) => { + buffer += chunk.toString("utf8"); + if (Buffer.byteLength(buffer, "utf8") > MAX_COMMAND_BYTES) { + log("browser-runtime-link: oversized command frame; reconnecting"); + socket?.destroy(); + return; + } + for (;;) { + const newline = buffer.indexOf("\n"); + if (newline < 0) return; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + consumeLine(line); + } + }; + + const scheduleReconnect = () => { + if (disposed) return; + clearRetry(); + const delay = backoff; + backoff = Math.min(backoff * 2, BACKOFF_MAX_MS); + retryTimer = setTimeout(connect, delay); + }; + + function connect() { + if (disposed) return; + destroySocket(); + buffer = ""; + const next = typeof address === "string" ? net.connect(address) : net.connect(address); + socket = next; + next.on("connect", () => { + if (disposed) { + next.destroy(); + return; + } + connected = true; + backoff = BACKOFF_INIT_MS; + send({ type: "hello", version: PROTOCOL_VERSION }); + log("browser-runtime-link: connected"); + }); + next.on("data", consume); + next.on("error", (error) => log(`browser-runtime-link: error: ${error.message}`)); + next.on("close", () => { + connected = false; + if (!disposed) scheduleReconnect(); + }); + } + + connect(); + return { + get connected() { + return connected; + }, + dispose() { + disposed = true; + connected = false; + clearRetry(); + destroySocket(); + }, + }; +} + +function normalizeCommandError(error: unknown): BrowserRuntimeCommandError { + if (isCommandError(error)) { + return { code: error.code, message: error.message }; + } + return { + code: "BROWSER_COMMAND_FAILED", + message: error instanceof Error ? error.message : "Browser command failed", + }; +} + +function isCommandError(error: unknown): error is BrowserRuntimeCommandError { + return Boolean( + error && + typeof error === "object" && + typeof (error as BrowserRuntimeCommandError).code === "string" && + typeof (error as BrowserRuntimeCommandError).message === "string", + ); +} diff --git a/frontend/src/main/browser-view-host.test.ts b/frontend/src/main/browser-view-host.test.ts index f32d60d9e8..f0c3f30e07 100644 --- a/frontend/src/main/browser-view-host.test.ts +++ b/frontend/src/main/browser-view-host.test.ts @@ -18,6 +18,14 @@ function setupHost() { let currentURL = ""; let displayHandler: DisplayHandler | null = null; const webContentsListeners = new Map void>(); + const debuggerListeners = new Map void>(); + let debuggerAttached = false; + const debuggerSendCommand = vi.fn(async (method: string): Promise => { + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.resolveNode") return { object: { objectId: "object-1" } }; + if (method === "Runtime.evaluate") return { result: { value: true } }; + return {}; + }); const webContents = { id: 99, mainFrame: { frameToken: "preview-frame" }, @@ -26,7 +34,20 @@ function setupHost() { capturePage: vi.fn(async () => ({ isEmpty: () => false, toJPEG: () => Buffer.from("snapshot"), + toPNG: () => Buffer.from("png-snapshot"), + getSize: () => ({ width: 640, height: 480 }), })), + debugger: { + attach: vi.fn(() => { + debuggerAttached = true; + }), + detach: vi.fn(() => { + debuggerAttached = false; + }), + isAttached: () => debuggerAttached, + on: (event: string, listener: (...args: never[]) => void) => debuggerListeners.set(event, listener), + sendCommand: debuggerSendCommand, + }, clearHistory: () => undefined, getTitle: () => "", getURL: () => currentURL, @@ -126,6 +147,9 @@ function setupHost() { shellSend, view, webContents, + webContentsListeners, + debuggerListeners, + debuggerSendCommand, }; } @@ -230,6 +254,120 @@ describe("browser:capture", () => { }); }); +describe("agent browser runtime", () => { + it("creates one hidden target per session and reuses it when the panel mounts", async () => { + const { host, invoke, view } = setupHost(); + await host.execute("sess-1", "open", { url: "http://localhost:4173" }); + + const state = await invoke("browser:ensure", "sess-1"); + + expect(state.viewId).toBe("0:sess-1"); + expect(view.webContents.loadURL).toHaveBeenCalledTimes(1); + }); + + it("returns compact refs and targets only the referenced WebContents node", async () => { + const { debuggerSendCommand, host } = setupHost(); + debuggerSendCommand.mockImplementation(async (method: string) => { + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { nodeId: "1", role: { value: "document" }, name: { value: "Demo" } }, + { + nodeId: "2", + parentId: "1", + backendDOMNodeId: 42, + role: { value: "button" }, + name: { value: "Save" }, + }, + ], + }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "save-button" } }; + return {}; + }); + + const snapshot = (await host.execute("sess-1", "snapshot", {})) as { text: string }; + await host.execute("sess-1", "click", { ref: "e1" }); + + expect(snapshot.text).toContain('button "Save" [ref=e1]'); + expect(debuggerSendCommand).toHaveBeenCalledWith("DOM.resolveNode", { backendNodeId: 42 }); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Runtime.callFunctionOn", + expect.objectContaining({ objectId: "save-button" }), + ); + }); + + it("fills the same session target mounted in the visible browser panel", async () => { + const { debuggerSendCommand, emit, host, invoke, view } = setupHost(); + debuggerSendCommand.mockImplementation(async (method: string) => { + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { + nodeId: "1", + backendDOMNodeId: 77, + role: { value: "textbox" }, + name: { value: "Profile" }, + }, + ], + }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "profile-input" } }; + return {}; + }); + + await host.execute("sess-1", "snapshot", { interactive: true }); + const panelState = await invoke("browser:ensure", "sess-1"); + emit("browser:setBounds", 1, { + viewId: panelState.viewId, + rect: { x: 20, y: 30, width: 400, height: 300 }, + visible: true, + }); + await host.execute("sess-1", "fill", { ref: "e1", text: "hello i am AO" }); + + expect(panelState.viewId).toBe("0:sess-1"); + expect(view.setVisible).toHaveBeenLastCalledWith(true); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Runtime.callFunctionOn", + expect.objectContaining({ + objectId: "profile-input", + arguments: [{ value: "hello i am AO" }], + }), + ); + }); + + it("invalidates refs after navigation", async () => { + const { debuggerSendCommand, host, webContentsListeners } = setupHost(); + debuggerSendCommand.mockImplementation(async (method: string) => { + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [{ nodeId: "1", backendDOMNodeId: 42, role: { value: "button" }, name: { value: "Save" } }], + }; + } + return {}; + }); + await host.execute("sess-1", "snapshot", {}); + webContentsListeners.get("did-start-loading")?.(); + + await expect(host.execute("sess-1", "click", { ref: "e1" })).rejects.toMatchObject({ + code: "STALE_REFERENCE", + }); + }); + + it("captures a PNG and separates errors from other console messages", async () => { + const { host, webContentsListeners } = setupHost(); + const screenshot = (await host.execute("sess-1", "screenshot")) as { data: string; width: number }; + const consoleListener = webContentsListeners.get("console-message"); + consoleListener?.({} as never, { level: "info", message: "ready" } as never); + consoleListener?.({} as never, { level: "error", message: "boom" } as never); + + const errors = (await host.execute("sess-1", "errors")) as { messages: Array<{ message: string }> }; + expect(screenshot.data).toBe(Buffer.from("png-snapshot").toString("base64")); + expect(screenshot.width).toBe(640); + expect(errors.messages.map((entry) => entry.message)).toEqual(["boom"]); + }); +}); + describe("browser:requestMirror", () => { it("grants the display-media request from the frame that armed the mirror", async () => { const { getDisplayHandler, invoke, rendererFrame, webContents } = setupHost(); diff --git a/frontend/src/main/browser-view-host.ts b/frontend/src/main/browser-view-host.ts index f1d73b58ff..2865986d1b 100644 --- a/frontend/src/main/browser-view-host.ts +++ b/frontend/src/main/browser-view-host.ts @@ -48,6 +48,7 @@ type BrowserWebContents = Pick< | "canGoForward" | "capturePage" | "clearHistory" + | "debugger" | "mainFrame" | "getTitle" | "getURL" @@ -104,6 +105,7 @@ export type BrowserViewHost = { dispose: () => void; destroy: (viewId: string) => void; destroyAll: () => void; + execute: (sessionId: string, action: string, args?: Record) => Promise; // webContents of the most recently focused browser panel (or null); the titlebar menu targets it for Edit/Reload/Zoom/DevTools. getLastFocusedPanelContents: () => WebContents | null; // Drop the remembered panel; call when the shell gains focus for a real reason so a stale panel stops absorbing menu actions. @@ -111,9 +113,22 @@ export type BrowserViewHost = { }; type BrowserEntry = { + sessionId: string; view: BrowserViewLike; state: BrowserNavState; annotationEnabled: boolean; + refGeneration: number; + refs: Map; + consoleMessages: BrowserLogEntry[]; + errors: BrowserLogEntry[]; +}; + +type BrowserLogEntry = { + level: string; + message: string; + source?: string; + line?: number; + timestamp: string; }; const OFFSCREEN_BOUNDS: BrowserRect = { x: -10_000, y: -10_000, width: 0, height: 0 }; @@ -177,6 +192,8 @@ export function scaleBoundsForZoom(rect: BrowserRect, zoomFactor: number): Brows export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserViewHost { const entries = new Map(); + const viewIdsBySessionId = new Map(); + const rendererOwnersByViewId = new Map>(); const viewIdsByWebContentsId = new Map(); const ipcDisposers: Array<() => void> = []; // viewId of the panel that most recently held focus; cleared when it is hidden or destroyed. @@ -218,7 +235,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }); } - const ensure = (viewId: string): BrowserEntry => { + const ensure = (viewId: string, sessionId: string): BrowserEntry => { const existing = entries.get(viewId); if (existing) return existing; @@ -235,11 +252,22 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV options.mainWindow.contentView.addChildView(view); const state: BrowserNavState = emptyNavState(viewId); - const entry = { view, state, annotationEnabled: false }; + const entry: BrowserEntry = { + sessionId, + view, + state, + annotationEnabled: false, + refGeneration: 0, + refs: new Map(), + consoleMessages: [], + errors: [], + }; entries.set(viewId, entry); + viewIdsBySessionId.set(sessionId, viewId); viewIdsByWebContentsId.set(view.webContents.id, viewId); hardenWebContents(view.webContents, options, entry); wireNavEvents(view.webContents, options, entry); + wireAutomationEvents(view.webContents, entry); // The preview is a separate WebContentsView, so renderer-window keydown // listeners never see keys typed here. Forward application shortcuts to the // shell renderer so they still work with the panel focused. @@ -250,6 +278,21 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV return entry; }; + const ensureSession = (sessionId: string, rendererId?: number): BrowserEntry => { + const existingViewId = viewIdsBySessionId.get(sessionId); + const viewId = existingViewId ?? `${rendererId ?? 0}:${sessionId}`; + const entry = ensure(viewId, sessionId); + if (rendererId !== undefined) { + const owners = rendererOwnersByViewId.get(viewId) ?? new Set(); + owners.add(rendererId); + rendererOwnersByViewId.set(viewId, owners); + } + return entry; + }; + + const isRendererOwned = (event: IpcMainInvokeEvent | IpcMainEvent, viewId: string): boolean => + rendererOwnersByViewId.get(viewId)?.has(event.sender.id) ?? false; + const setBounds = ({ viewId, rect, visible, parked }: BrowserBoundsInput, zoomFactor = 1): void => { const entry = entries.get(viewId); if (!entry) return; @@ -276,7 +319,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }; const navigate = async ({ viewId, url }: BrowserNavigateInput): Promise => { - const entry = ensure(viewId); + const entry = entries.get(viewId); + if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); cancelAnnotation(options, entry, "navigation"); const normalized = normalizeBrowserURL(url); if (!isAllowedBrowserURL(normalized.href, options.rendererOrigin)) { @@ -300,7 +344,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV // readNavState normalizes it back to an empty url so the panel shows its // empty state. const clear = async (viewId: string): Promise => { - const entry = ensure(viewId); + const entry = entries.get(viewId); + if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); cancelAnnotation(options, entry, "navigation"); entry.view.setVisible?.(false); entry.view.setBounds(OFFSCREEN_BOUNDS); @@ -326,6 +371,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV const entry = entries.get(viewId); if (!entry) return; entries.delete(viewId); + viewIdsBySessionId.delete(entry.sessionId); + rendererOwnersByViewId.delete(viewId); viewIdsByWebContentsId.delete(entry.view.webContents.id); forgetIfFocused(viewId); // When the window is already gone (dispose fired from mainWindow "closed"), @@ -335,6 +382,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV entry.view.setVisible?.(false); entry.view.setBounds(OFFSCREEN_BOUNDS); options.mainWindow.contentView.removeChildView?.(entry.view); + if (entry.view.webContents.debugger?.isAttached()) { + entry.view.webContents.debugger.detach(); + } entry.view.webContents.close?.(); }; @@ -351,7 +401,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }; const setAnnotationMode = (event: IpcMainInvokeEvent, input: BrowserAnnotationModeInput): void => { - if (!isRendererOwnedViewId(event, input.viewId)) return; + if (!isRendererOwned(event, input.viewId)) return; const entry = entries.get(input.viewId); if (!entry) return; entry.annotationEnabled = input.enabled; @@ -410,24 +460,44 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV ipcDisposers.push(() => options.ipcMain.off(channel, fn)); }; - handle("browser:ensure", (event, sessionId: string) => pushNavState(options, ensure(scopedViewId(event, sessionId)))); - on("browser:setBounds", (event, input: BrowserBoundsInput) => setBounds(input, event.sender.getZoomFactor())); - handle("browser:navigate", (_event, input: BrowserNavigateInput) => navigate(input)); - handle("browser:clear", (_event, viewId: string) => clear(viewId)); - handle("browser:capture", (event, viewId: string) => (isRendererOwnedViewId(event, viewId) ? capture(viewId) : "")); + handle("browser:ensure", (event, sessionId: string) => + pushNavState(options, ensureSession(sessionId, event.sender.id)), + ); + on("browser:setBounds", (event, input: BrowserBoundsInput) => { + if (isRendererOwned(event, input.viewId)) setBounds(input, event.sender.getZoomFactor()); + }); + handle("browser:navigate", (event, input: BrowserNavigateInput) => + isRendererOwned(event, input.viewId) ? navigate(input) : emptyNavState(input.viewId), + ); + handle("browser:clear", (event, viewId: string) => + isRendererOwned(event, viewId) ? clear(viewId) : emptyNavState(viewId), + ); + handle("browser:capture", (event, viewId: string) => (isRendererOwned(event, viewId) ? capture(viewId) : "")); handle("browser:requestMirror", (event, viewId: string) => { - if (!mirrorSupported || !isRendererOwnedViewId(event, viewId) || !entries.has(viewId)) return false; + if (!mirrorSupported || !isRendererOwned(event, viewId) || !entries.has(viewId)) return false; const frame = event.senderFrame; if (!frame) return false; pendingMirror = { viewId, expires: Date.now() + 5000, frame }; return true; }); - handle("browser:goBack", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.goBack(), true)); - handle("browser:goForward", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.goForward(), true)); - handle("browser:reload", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.reload(), true)); - handle("browser:stop", (_event, viewId: string) => invokeNav(viewId, (contents) => contents.stop())); + handle("browser:goBack", (event, viewId: string) => + isRendererOwned(event, viewId) ? invokeNav(viewId, (contents) => contents.goBack(), true) : emptyNavState(viewId), + ); + handle("browser:goForward", (event, viewId: string) => + isRendererOwned(event, viewId) + ? invokeNav(viewId, (contents) => contents.goForward(), true) + : emptyNavState(viewId), + ); + handle("browser:reload", (event, viewId: string) => + isRendererOwned(event, viewId) ? invokeNav(viewId, (contents) => contents.reload(), true) : emptyNavState(viewId), + ); + handle("browser:stop", (event, viewId: string) => + isRendererOwned(event, viewId) ? invokeNav(viewId, (contents) => contents.stop()) : emptyNavState(viewId), + ); handle("browser:annotation:setMode", (event, input: BrowserAnnotationModeInput) => setAnnotationMode(event, input)); - on("browser:destroy", (_event, viewId: string) => destroy(viewId)); + on("browser:destroy", (event, viewId: string) => { + if (isRendererOwned(event, viewId)) destroy(viewId); + }); on("browser:annotation:submit", (event, payload: BrowserAnnotationPageSubmitPayload) => forwardAnnotationSubmit(event, payload), ); @@ -436,6 +506,38 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV ); return { + execute: async (sessionId, action, args = {}) => { + if (!sessionId.trim()) throw browserError("INVALID_ARGUMENT", "sessionId is required"); + const entry = ensureSession(sessionId); + switch (action) { + case "open": { + const url = stringArg(args, "url", "URL_REQUIRED", "url is required"); + const state = await navigate({ viewId: entry.state.viewId, url }); + if (state.error) throw browserError("NAVIGATION_FAILED", state.error); + return state; + } + case "snapshot": + return snapshotEntry(entry, Boolean(args.interactive)); + case "click": + return clickEntry(entry, stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required")); + case "fill": + return fillEntry( + entry, + stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required"), + stringArg(args, "text", "INVALID_ARGUMENT", "text is required", true), + ); + case "wait": + return waitForEntry(entry, args); + case "screenshot": + return screenshotEntry(entry); + case "console": + return { messages: [...entry.consoleMessages] }; + case "errors": + return { messages: [...entry.errors] }; + default: + throw browserError("INVALID_ARGUMENT", `Unsupported browser action: ${action}`); + } + }, dispose: () => { ipcDisposers.splice(0).forEach((dispose) => dispose()); for (const viewId of [...entries.keys()]) { @@ -505,14 +607,6 @@ function emptyNavState(viewId: string): BrowserNavState { }; } -function scopedViewId(event: IpcMainInvokeEvent, sessionId: string): string { - return `${event.sender.id}:${sessionId}`; -} - -function isRendererOwnedViewId(event: IpcMainInvokeEvent, viewId: string): boolean { - return viewId.startsWith(`${event.sender.id}:`); -} - function hardenWebContents(contents: BrowserWebContents, options: BrowserViewHostOptions, entry: BrowserEntry): void { contents.setWindowOpenHandler(({ url }) => { if (isAllowedBrowserURL(url, options.rendererOrigin)) { @@ -542,12 +636,18 @@ function wireNavEvents(contents: BrowserWebContents, options: BrowserViewHostOpt contents.on("did-navigate-in-page", update); contents.on("page-title-updated", update); contents.on("did-start-loading", () => { + invalidateRefs(entry); cancelAnnotation(options, entry, "navigation"); update(); }); contents.on("did-stop-loading", update); contents.on("did-fail-load", (_event, errorCode, errorDescription) => { if (errorCode === -3) return; + pushBrowserLog(entry.errors, { + level: "error", + message: String(errorDescription || `Navigation failed (${errorCode})`), + timestamp: new Date().toISOString(), + }); entry.view.setVisible?.(false); entry.state = { ...readNavState(entry), error: String(errorDescription || "Unable to load page") }; options.mainWindow.webContents.send("browser:navState", entry.state); @@ -586,3 +686,285 @@ function readNavState(entry: BrowserEntry): BrowserNavState { isLoading: webContents.isLoading(), }; } + +type AXValue = { value?: unknown }; +type AXNode = { + nodeId: string; + parentId?: string; + ignored?: boolean; + backendDOMNodeId?: number; + role?: AXValue; + name?: AXValue; + value?: AXValue; + properties?: Array<{ name: string; value?: AXValue }>; +}; + +const INTERACTIVE_ROLES = new Set([ + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "scrollbar", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +]); + +function wireAutomationEvents(contents: BrowserWebContents, entry: BrowserEntry): void { + contents.on("console-message", (...eventArgs: unknown[]) => { + const details = eventArgs.find( + (value) => value && typeof value === "object" && typeof (value as { message?: unknown }).message === "string", + ) as { level?: string; message: string; lineNumber?: number; sourceId?: string } | undefined; + const legacyLevel = typeof eventArgs[1] === "number" ? eventArgs[1] : 1; + const legacyMessage = typeof eventArgs[2] === "string" ? eventArgs[2] : ""; + const level = details?.level ?? ["debug", "info", "warning", "error"][legacyLevel] ?? "info"; + const log: BrowserLogEntry = { + level, + message: details?.message ?? legacyMessage, + source: details?.sourceId ?? (typeof eventArgs[4] === "string" ? eventArgs[4] : undefined), + line: details?.lineNumber ?? (typeof eventArgs[3] === "number" ? eventArgs[3] : undefined), + timestamp: new Date().toISOString(), + }; + pushBrowserLog(entry.consoleMessages, log); + if (level === "error") pushBrowserLog(entry.errors, log); + }); + contents.on("render-process-gone", (_event, details) => { + pushBrowserLog(entry.errors, { + level: "error", + message: `Browser renderer exited: ${details.reason}`, + timestamp: new Date().toISOString(), + }); + }); + const targetDebugger = contents.debugger; + if (!targetDebugger) return; + targetDebugger.on("message", (_event, method, params) => { + if (method === "DOM.documentUpdated") { + invalidateRefs(entry); + return; + } + if (method === "Runtime.exceptionThrown") { + const detail = params as { exceptionDetails?: { text?: string; url?: string; lineNumber?: number } }; + const exception = detail.exceptionDetails; + pushBrowserLog(entry.errors, { + level: "error", + message: exception?.text ?? "Uncaught browser exception", + source: exception?.url, + line: exception?.lineNumber, + timestamp: new Date().toISOString(), + }); + } + }); +} + +function pushBrowserLog(target: BrowserLogEntry[], entry: BrowserLogEntry): void { + target.push(entry); + if (target.length > 200) target.splice(0, target.length - 200); +} + +function invalidateRefs(entry: BrowserEntry): void { + entry.refGeneration += 1; + entry.refs.clear(); +} + +async function ensureDebugger(entry: BrowserEntry): Promise { + const debug = entry.view.webContents.debugger; + if (!debug) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser debugger is unavailable"); + if (!debug.isAttached()) { + try { + debug.attach("1.3"); + } catch (error) { + throw browserError( + "BROWSER_TARGET_UNAVAILABLE", + error instanceof Error ? error.message : "Unable to attach to browser target", + ); + } + } + await debug.sendCommand("Runtime.enable"); + await debug.sendCommand("DOM.enable"); +} + +async function snapshotEntry(entry: BrowserEntry, interactiveOnly: boolean): Promise { + await ensureDebugger(entry); + await entry.view.webContents.debugger.sendCommand("Accessibility.enable"); + const response = (await entry.view.webContents.debugger.sendCommand("Accessibility.getFullAXTree")) as { + nodes?: AXNode[]; + }; + const nodes = response.nodes ?? []; + entry.refGeneration += 1; + entry.refs.clear(); + const generation = entry.refGeneration; + const depths = new Map(); + const lines: string[] = []; + const elements: Array<{ ref: string; role: string; name: string }> = []; + let refIndex = 0; + for (const node of nodes.slice(0, 1_000)) { + if (node.ignored) continue; + const role = stringValue(node.role) || "generic"; + const name = stringValue(node.name); + const value = stringValue(node.value); + const interactive = + INTERACTIVE_ROLES.has(role) || node.properties?.some((property) => property.name === "focusable"); + let ref = ""; + if (interactive && node.backendDOMNodeId) { + ref = `e${++refIndex}`; + entry.refs.set(ref, { backendNodeId: node.backendDOMNodeId, generation }); + elements.push({ ref, role, name }); + } + if (interactiveOnly && !ref) continue; + if (!ref && !name && !value) continue; + const parentDepth = node.parentId ? (depths.get(node.parentId) ?? -1) : -1; + const depth = Math.max(0, parentDepth + 1); + depths.set(node.nodeId, depth); + const label = name ? ` \"${compactText(name)}\"` : ""; + const currentValue = value && value !== name ? ` value=\"${compactText(value)}\"` : ""; + const reference = ref ? ` [ref=${ref}]` : ""; + lines.push(`${" ".repeat(Math.min(depth, 8))}${role}${label}${currentValue}${reference}`); + } + return { + url: entry.view.webContents.getURL(), + title: entry.view.webContents.getTitle(), + generation, + text: lines.join("\n") || "(empty accessibility snapshot)", + elements, + }; +} + +async function clickEntry(entry: BrowserEntry, refName: string): Promise { + const objectId = await resolveRef(entry, refName); + await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: + "function(){ this.scrollIntoView({block:'center',inline:'center'}); this.focus(); this.click(); }", + awaitPromise: true, + }); + return { ref: refName, url: entry.view.webContents.getURL() }; +} + +async function fillEntry(entry: BrowserEntry, refName: string, text: string): Promise { + const objectId = await resolveRef(entry, refName); + await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function(next){ + this.scrollIntoView({block:'center',inline:'center'}); + this.focus(); + const proto = Object.getPrototypeOf(this); + const descriptor = proto && Object.getOwnPropertyDescriptor(proto, 'value'); + if (descriptor && descriptor.set) descriptor.set.call(this, next); else this.value = next; + this.dispatchEvent(new Event('input', {bubbles:true, composed:true})); + this.dispatchEvent(new Event('change', {bubbles:true, composed:true})); + }`, + arguments: [{ value: text }], + awaitPromise: true, + }); + return { ref: refName, value: text, url: entry.view.webContents.getURL() }; +} + +async function resolveRef(entry: BrowserEntry, refName: string): Promise { + await ensureDebugger(entry); + const ref = entry.refs.get(refName); + if (!ref || ref.generation !== entry.refGeneration) { + throw browserError("STALE_REFERENCE", `Element reference ${refName} is stale; run ao browser snapshot again`); + } + try { + const resolved = (await entry.view.webContents.debugger.sendCommand("DOM.resolveNode", { + backendNodeId: ref.backendNodeId, + })) as { object?: { objectId?: string } }; + if (!resolved.object?.objectId) throw new Error("node has no runtime object"); + return resolved.object.objectId; + } catch { + entry.refs.delete(refName); + throw browserError("STALE_REFERENCE", `Element reference ${refName} is stale; run ao browser snapshot again`); + } +} + +async function waitForEntry(entry: BrowserEntry, args: Record): Promise { + const fixedMS = numberArg(args.ms, 0, 60_000); + if (fixedMS > 0) { + await delay(fixedMS); + return { waitedMs: fixedMS, url: entry.view.webContents.getURL() }; + } + const timeoutMS = numberArg(args.timeoutMs, 1, 60_000) || 10_000; + let expression = ""; + let condition = ""; + if (typeof args.text === "string" && args.text) { + expression = `Boolean(document.body && document.body.innerText.includes(${JSON.stringify(args.text)}))`; + condition = `text ${JSON.stringify(args.text)}`; + } else if (typeof args.selector === "string" && args.selector) { + expression = `Boolean(document.querySelector(${JSON.stringify(args.selector)}))`; + condition = `selector ${JSON.stringify(args.selector)}`; + } else if (typeof args.url === "string" && args.url) { + expression = `location.href.includes(${JSON.stringify(args.url)})`; + condition = `URL ${JSON.stringify(args.url)}`; + } else { + throw browserError("INVALID_ARGUMENT", "wait requires text, selector, url, or ms"); + } + await ensureDebugger(entry); + const deadline = Date.now() + timeoutMS; + while (Date.now() <= deadline) { + const evaluated = (await entry.view.webContents.debugger.sendCommand("Runtime.evaluate", { + expression, + returnByValue: true, + })) as { result?: { value?: unknown } }; + if (evaluated.result?.value === true) { + return { condition, url: entry.view.webContents.getURL() }; + } + await delay(100); + } + throw browserError("WAIT_TIMEOUT", `Timed out after ${timeoutMS}ms waiting for ${condition}`); +} + +async function screenshotEntry(entry: BrowserEntry): Promise { + const image = await entry.view.webContents.capturePage(); + if (image.isEmpty()) throw browserError("BROWSER_COMMAND_FAILED", "Browser screenshot is empty"); + const size = image.getSize(); + return { + mimeType: "image/png", + data: image.toPNG().toString("base64"), + width: size.width, + height: size.height, + url: entry.view.webContents.getURL(), + }; +} + +function stringArg( + args: Record, + name: string, + code: string, + message: string, + allowEmpty = false, +): string { + const value = args[name]; + if (typeof value !== "string" || (!allowEmpty && !value.trim())) throw browserError(code, message); + return value; +} + +function numberArg(value: unknown, min: number, max: number): number { + if (typeof value !== "number" || !Number.isFinite(value)) return 0; + return Math.max(min, Math.min(max, Math.round(value))); +} + +function stringValue(value: AXValue | undefined): string { + return typeof value?.value === "string" ? value.value : value?.value == null ? "" : String(value.value); +} + +function compactText(value: string): string { + return value.replace(/\s+/g, " ").replace(/\"/g, '\\"').trim().slice(0, 240); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function browserError(code: string, message: string): Error & { code: string } { + return Object.assign(new Error(message), { code }); +} From af15ef4eebf6ee7612c4a326429bb569b1b38c9e Mon Sep 17 00:00:00 2001 From: aprv10 <1apoorvindia@gmail.com> Date: Thu, 23 Jul 2026 16:53:32 +0530 Subject: [PATCH 03/11] feat(browser): expand core interactions --- backend/internal/cli/browser.go | 100 ++++++ backend/internal/cli/browser_test.go | 43 +++ backend/internal/httpd/controllers/browser.go | 8 + .../httpd/controllers/browser_test.go | 24 ++ .../skillassets/using-ao/commands/browser.md | 15 + docs/STATUS.md | 6 +- docs/cli/README.md | 11 +- frontend/src/main/browser-view-host.test.ts | 87 ++++++ frontend/src/main/browser-view-host.ts | 292 ++++++++++++++++++ 9 files changed, 579 insertions(+), 7 deletions(-) diff --git a/backend/internal/cli/browser.go b/backend/internal/cli/browser.go index f8f04a3379..660d69e39c 100644 --- a/backend/internal/cli/browser.go +++ b/backend/internal/cli/browser.go @@ -107,6 +107,91 @@ func newBrowserCommand(ctx *commandContext) *cobra.Command { }, }) + cmd.AddCommand(&cobra.Command{ + Use: "type ", + Short: "Type text at the current cursor position in a form control", + Args: exactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "type", map[string]any{"ref": args[0], "text": args[1]}, jsonOutput) + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "press ", + Short: "Press a key or modifier chord such as Enter or Control+A", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "press", map[string]any{"key": args[0]}, jsonOutput) + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "hover ", + Short: "Move the pointer over an element", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "hover", map[string]any{"ref": args[0]}, jsonOutput) + }, + }) + + var scrollAmount int + scroll := &cobra.Command{ + Use: "scroll ", + Short: "Scroll the page in one direction", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction( + cmd, + "scroll", + map[string]any{"direction": args[0], "amount": scrollAmount}, + jsonOutput, + ) + }, + } + scroll.Flags().IntVar(&scrollAmount, "amount", 600, "scroll distance in CSS pixels") + cmd.AddCommand(scroll) + + cmd.AddCommand(&cobra.Command{ + Use: "select ", + Short: "Select an option value", + Args: exactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "select", map[string]any{"ref": args[0], "value": args[1]}, jsonOutput) + }, + }) + + for _, checked := range []bool{true, false} { + checked := checked + action := "check" + short := "Check a checkbox or switch" + if !checked { + action = "uncheck" + short = "Uncheck a checkbox or switch" + } + cmd.AddCommand(&cobra.Command{ + Use: action + " ", + Short: short, + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, action, map[string]any{"ref": args[0]}, jsonOutput) + }, + }) + } + + cmd.AddCommand(&cobra.Command{ + Use: "get [ref]", + Short: "Read a page or element property", + Long: "Read url, title, or text from the page, or text, value, or checked from an element reference.", + Args: rangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + actionArgs := map[string]any{"property": args[0]} + if len(args) == 2 { + actionArgs["ref"] = args[1] + } + return ctx.runBrowserAction(cmd, "get", actionArgs, jsonOutput) + }, + }) + var waitText, waitSelector, waitURL string var waitMS, timeoutMS int waitCmd := &cobra.Command{ @@ -189,6 +274,15 @@ func exactArgs(n int) cobra.PositionalArgs { } } +func rangeArgs(min, max int) cobra.PositionalArgs { + return func(cmd *cobra.Command, args []string) error { + if err := cobra.RangeArgs(min, max)(cmd, args); err != nil { + return usageError{err} + } + return nil + } +} + func currentBrowserSessionID() (string, error) { sessionID := strings.TrimSpace(os.Getenv("AO_SESSION_ID")) if sessionID == "" { @@ -252,6 +346,12 @@ func writeBrowserResult(cmd *cobra.Command, action string, result map[string]any } return nil } + if action == "get" { + if value, ok := result["value"]; ok { + _, err := fmt.Fprintln(cmd.OutOrStdout(), value) + return err + } + } if currentURL, ok := result["url"].(string); ok && currentURL != "" { _, err := fmt.Fprintln(cmd.OutOrStdout(), currentURL) return err diff --git a/backend/internal/cli/browser_test.go b/backend/internal/cli/browser_test.go index 1e5d1b7739..29da1eab39 100644 --- a/backend/internal/cli/browser_test.go +++ b/backend/internal/cli/browser_test.go @@ -91,6 +91,46 @@ func TestBrowserClickAndWaitArguments(t *testing.T) { } } +func TestBrowserCoreInteractionArguments(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + + tests := []struct { + name string + args []string + action string + want map[string]any + }{ + {name: "type", args: []string{"type", "e1", "hello"}, action: "type", want: map[string]any{"ref": "e1", "text": "hello"}}, + {name: "press", args: []string{"press", "Control+A"}, action: "press", want: map[string]any{"key": "Control+A"}}, + {name: "hover", args: []string{"hover", "e2"}, action: "hover", want: map[string]any{"ref": "e2"}}, + {name: "scroll", args: []string{"scroll", "down", "--amount", "450"}, action: "scroll", want: map[string]any{"direction": "down", "amount": float64(450)}}, + {name: "select", args: []string{"select", "e3", "large"}, action: "select", want: map[string]any{"ref": "e3", "value": "large"}}, + {name: "check", args: []string{"check", "e4"}, action: "check", want: map[string]any{"ref": "e4"}}, + {name: "uncheck", args: []string{"uncheck", "e4"}, action: "uncheck", want: map[string]any{"ref": "e4"}}, + {name: "get", args: []string{"get", "value", "e5"}, action: "get", want: map[string]any{"property": "value", "ref": "e5"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, _, err := executeCLI(t, deps, append([]string{"browser"}, tt.args...)...); err != nil { + t.Fatal(err) + } + if capture.body.Action != tt.action { + t.Fatalf("action = %q, want %q", capture.body.Action, tt.action) + } + for key, want := range tt.want { + if got := capture.body.Args[key]; got != want { + t.Fatalf("%s arg %q = %#v, want %#v", tt.name, key, got, want) + } + } + }) + } +} + func TestBrowserScreenshotWritesWithoutOverwrite(t *testing.T) { t.Setenv("AO_SESSION_ID", "ao-1") cfg := setConfigEnv(t) @@ -122,4 +162,7 @@ func TestBrowserRequiresSessionAndValidWait(t *testing.T) { if _, _, err := executeCLI(t, Deps{}, "browser", "wait", "--text", "x", "--url", "y"); ExitCode(err) != 2 { t.Fatalf("wait error = %v code=%d", err, ExitCode(err)) } + if _, _, err := executeCLI(t, Deps{}, "browser", "get"); ExitCode(err) != 2 { + t.Fatalf("get error = %v code=%d", err, ExitCode(err)) + } } diff --git a/backend/internal/httpd/controllers/browser.go b/backend/internal/httpd/controllers/browser.go index 26050e346c..96222f1e0d 100644 --- a/backend/internal/httpd/controllers/browser.go +++ b/backend/internal/httpd/controllers/browser.go @@ -19,6 +19,14 @@ var browserActions = map[string]struct{}{ "snapshot": {}, "click": {}, "fill": {}, + "type": {}, + "press": {}, + "hover": {}, + "scroll": {}, + "select": {}, + "check": {}, + "uncheck": {}, + "get": {}, "wait": {}, "screenshot": {}, "console": {}, diff --git a/backend/internal/httpd/controllers/browser_test.go b/backend/internal/httpd/controllers/browser_test.go index f9e40f46cd..aafc703e12 100644 --- a/backend/internal/httpd/controllers/browser_test.go +++ b/backend/internal/httpd/controllers/browser_test.go @@ -67,6 +67,30 @@ func TestBrowserStatusAndSnapshot(t *testing.T) { } } +func TestBrowserCoreInteractionActionsReachRuntime(t *testing.T) { + runtime := &fakeBrowserRuntime{} + srv := browserServer(t, runtime) + actions := []string{"type", "press", "hover", "scroll", "select", "check", "uncheck", "get"} + + for _, action := range actions { + t.Run(action, func(t *testing.T) { + body, status, _ := doRequest( + t, + srv, + http.MethodPost, + "/api/v1/browser/commands", + `{"sessionId":"ao-1","action":"`+action+`","args":{"probe":true}}`, + ) + if status != http.StatusOK { + t.Fatalf("%s = %d body=%s", action, status, body) + } + if runtime.action != action || runtime.args["probe"] != true { + t.Fatalf("runtime command = %q %#v", runtime.action, runtime.args) + } + }) + } +} + func TestBrowserCommandValidationAndErrors(t *testing.T) { runtime := &fakeBrowserRuntime{} srv := browserServer(t, runtime) diff --git a/backend/internal/skillassets/using-ao/commands/browser.md b/backend/internal/skillassets/using-ao/commands/browser.md index 3507130b46..b559e7c94e 100644 --- a/backend/internal/skillassets/using-ao/commands/browser.md +++ b/backend/internal/skillassets/using-ao/commands/browser.md @@ -14,6 +14,8 @@ ao browser open http://localhost:5173 ao browser snapshot ao browser click e1 ao browser fill e2 "hello" +ao browser press Enter +ao browser hover e3 ao browser wait --text "Saved" ao browser snapshot ao browser errors @@ -29,12 +31,25 @@ ao browser open [--json] ao browser snapshot [--interactive] [--json] ao browser click [--json] ao browser fill [--json] +ao browser type [--json] +ao browser press [--json] +ao browser hover [--json] +ao browser scroll [--amount ] [--json] +ao browser select [--json] +ao browser check [--json] +ao browser uncheck [--json] +ao browser get [ref] [--json] ao browser wait (--text | --selector | --url | --ms ) [--timeout ] [--json] ao browser screenshot [path] [--json] ao browser console [--json] ao browser errors [--json] ``` +`fill` replaces the current value, while `type` inserts text at the current +cursor position. `press` accepts named keys and chords such as `Enter`, +`ArrowDown`, and `Control+A`. Page-level `get` supports `url`, `title`, and +`text`; with an element ref it supports `text`, `value`, and `checked`. + Without `--json`, `screenshot` writes a PNG and refuses to overwrite an existing file. With `--json`, it returns the structured response including base64 image data. `ao preview` remains available for the passive URL/static-file workflow. Use `ao browser` when the agent needs to inspect or verify the page. diff --git a/docs/STATUS.md b/docs/STATUS.md index 07a99127d8..30e5bb114c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -64,8 +64,10 @@ surface (`npm run sqlc`, `npm run api`). - Target-isolated per-session browser-control spike: a dedicated local daemon↔Electron bridge drives only the selected session's `WebContentsView` through Electron's bound debugger transport. `ao browser` supports open, - compact accessibility snapshots and refs, click/fill, waits, screenshots, - console messages, and page errors while the Browser panel is hidden. + compact accessibility snapshots and refs, click/fill/type, keyboard input, + hover, scrolling, selection and checked state, property reads, waits, + screenshots, console messages, and page errors while the Browser panel is + hidden. - Real daemon wiring via the generated `openapi-fetch` typed client (`src/api/schema.ts`); mock data only in `VITE_NO_ELECTRON` web-preview mode. - Electron main handles daemon discovery, launch, and status reporting. diff --git a/docs/cli/README.md b/docs/cli/README.md index dcbdd5fbe0..be01d6b5f1 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -83,11 +83,12 @@ opens that URL verbatim (`file://`, `http`, `https`). `ao browser` also resolves its target from `AO_SESSION_ID`, but controls the session-owned live Electron browser rather than only setting its preview URL. -The initial target-isolated command set is `status`, `open`, `snapshot`, -`click`, `fill`, `wait`, `screenshot`, `console`, and `errors`. The AO desktop -app must be open because Electron owns the `WebContentsView`. References from a -snapshot are invalidated after navigation or DOM replacement; take another -snapshot when a command reports `STALE_REFERENCE`. +The target-isolated command set includes `status`, `open`, `snapshot`, `click`, +`fill`, `type`, `press`, `hover`, `scroll`, `select`, `check`, `uncheck`, `get`, +`wait`, `screenshot`, `console`, and `errors`. The AO desktop app must be open +because Electron owns the `WebContentsView`. References from a snapshot are +invalidated after navigation or DOM replacement; take another snapshot when a +command reports `STALE_REFERENCE`. `go run .` in `backend/` remains a compatibility wrapper around the daemon. diff --git a/frontend/src/main/browser-view-host.test.ts b/frontend/src/main/browser-view-host.test.ts index f0c3f30e07..8ad785e9e3 100644 --- a/frontend/src/main/browser-view-host.test.ts +++ b/frontend/src/main/browser-view-host.test.ts @@ -336,6 +336,93 @@ describe("agent browser runtime", () => { ); }); + it("supports keyboard, pointer, form, scroll, and property actions on the session target", async () => { + const { debuggerSendCommand, host } = setupHost(); + debuggerSendCommand.mockImplementation(async (method: string, params?: Record) => { + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { + nodeId: "1", + backendDOMNodeId: 88, + role: { value: "textbox" }, + name: { value: "Search" }, + }, + ], + }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "target-element" } }; + if (method === "DOM.getBoxModel") { + return { model: { border: [10, 20, 30, 20, 30, 40, 10, 40] } }; + } + if (method === "Runtime.evaluate") return { result: { value: { x: 400, y: 300 } } }; + if (method === "Runtime.callFunctionOn") { + const declaration = String(params?.functionDeclaration ?? ""); + if (declaration.includes("HTMLSelectElement")) { + return { result: { value: { supported: true, matched: true, value: "large" } } }; + } + if (declaration.includes("'checked' in this")) { + const desired = (params?.arguments as Array<{ value?: boolean }> | undefined)?.[0]?.value; + return { result: { value: { supported: true, checked: desired } } }; + } + if (declaration.includes("function(property)")) { + return { result: { value: "current value" } }; + } + } + return {}; + }); + + await host.execute("sess-1", "snapshot", { interactive: true }); + await host.execute("sess-1", "type", { ref: "e1", text: "hello" }); + await host.execute("sess-1", "press", { key: "Control+A" }); + await host.execute("sess-1", "hover", { ref: "e1" }); + await host.execute("sess-1", "scroll", { direction: "down", amount: 450 }); + await host.execute("sess-1", "select", { ref: "e1", value: "large" }); + await host.execute("sess-1", "check", { ref: "e1" }); + await host.execute("sess-1", "uncheck", { ref: "e1" }); + const property = (await host.execute("sess-1", "get", { + property: "value", + ref: "e1", + })) as { value: string }; + + expect(property.value).toBe("current value"); + expect(debuggerSendCommand).toHaveBeenCalledWith("Input.insertText", { text: "hello" }); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Input.dispatchKeyEvent", + expect.objectContaining({ type: "rawKeyDown", key: "a", modifiers: 2 }), + ); + expect(debuggerSendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 20, + y: 30, + }); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Input.dispatchMouseEvent", + expect.objectContaining({ type: "mouseWheel", deltaY: 450, x: 400, y: 300 }), + ); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Runtime.callFunctionOn", + expect.objectContaining({ + arguments: [{ value: false }], + functionDeclaration: expect.stringContaining("this.click()"), + }), + ); + }); + + it("rejects unsupported keys, scroll directions, and property names", async () => { + const { host } = setupHost(); + + await expect(host.execute("sess-1", "press", { key: "Hyper+K" })).rejects.toMatchObject({ + code: "INVALID_ARGUMENT", + }); + await expect(host.execute("sess-1", "scroll", { direction: "diagonal" })).rejects.toMatchObject({ + code: "INVALID_ARGUMENT", + }); + await expect(host.execute("sess-1", "get", { property: "html" })).rejects.toMatchObject({ + code: "INVALID_ARGUMENT", + }); + }); + it("invalidates refs after navigation", async () => { const { debuggerSendCommand, host, webContentsListeners } = setupHost(); debuggerSendCommand.mockImplementation(async (method: string) => { diff --git a/frontend/src/main/browser-view-host.ts b/frontend/src/main/browser-view-host.ts index 2865986d1b..95c4edc9b0 100644 --- a/frontend/src/main/browser-view-host.ts +++ b/frontend/src/main/browser-view-host.ts @@ -526,6 +526,46 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required"), stringArg(args, "text", "INVALID_ARGUMENT", "text is required", true), ); + case "type": + return typeEntry( + entry, + stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required"), + stringArg(args, "text", "INVALID_ARGUMENT", "text is required", true), + ); + case "press": + return pressEntry(entry, stringArg(args, "key", "INVALID_ARGUMENT", "key is required")); + case "hover": + return hoverEntry(entry, stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required")); + case "scroll": + return scrollEntry( + entry, + stringArg(args, "direction", "INVALID_ARGUMENT", "direction is required"), + numberArg(args.amount, 1, 5_000) || 600, + ); + case "select": + return selectEntry( + entry, + stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required"), + stringArg(args, "value", "INVALID_ARGUMENT", "value is required", true), + ); + case "check": + return checkEntry( + entry, + stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required"), + true, + ); + case "uncheck": + return checkEntry( + entry, + stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required"), + false, + ); + case "get": + return getEntry( + entry, + stringArg(args, "property", "INVALID_ARGUMENT", "property is required"), + typeof args.ref === "string" && args.ref.trim() ? args.ref : undefined, + ); case "wait": return waitForEntry(entry, args); case "screenshot": @@ -869,6 +909,258 @@ async function fillEntry(entry: BrowserEntry, refName: string, text: string): Pr return { ref: refName, value: text, url: entry.view.webContents.getURL() }; } +async function typeEntry(entry: BrowserEntry, refName: string, text: string): Promise { + const objectId = await resolveRef(entry, refName); + await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: + "function(){ this.scrollIntoView({block:'center',inline:'center'}); this.focus(); }", + }); + await entry.view.webContents.debugger.sendCommand("Input.insertText", { text }); + return { ref: refName, text, url: entry.view.webContents.getURL() }; +} + +type BrowserKey = { + key: string; + code: string; + keyCode: number; + text?: string; + modifiers: number; +}; + +const NAMED_KEYS: Record> = { + enter: { key: "Enter", code: "Enter", keyCode: 13, text: "\r" }, + tab: { key: "Tab", code: "Tab", keyCode: 9, text: "\t" }, + escape: { key: "Escape", code: "Escape", keyCode: 27 }, + esc: { key: "Escape", code: "Escape", keyCode: 27 }, + backspace: { key: "Backspace", code: "Backspace", keyCode: 8 }, + delete: { key: "Delete", code: "Delete", keyCode: 46 }, + home: { key: "Home", code: "Home", keyCode: 36 }, + end: { key: "End", code: "End", keyCode: 35 }, + pageup: { key: "PageUp", code: "PageUp", keyCode: 33 }, + pagedown: { key: "PageDown", code: "PageDown", keyCode: 34 }, + arrowup: { key: "ArrowUp", code: "ArrowUp", keyCode: 38 }, + arrowdown: { key: "ArrowDown", code: "ArrowDown", keyCode: 40 }, + arrowleft: { key: "ArrowLeft", code: "ArrowLeft", keyCode: 37 }, + arrowright: { key: "ArrowRight", code: "ArrowRight", keyCode: 39 }, + space: { key: " ", code: "Space", keyCode: 32, text: " " }, +}; + +function parseBrowserKey(input: string): BrowserKey { + const parts = input + .split("+") + .map((part) => part.trim()) + .filter(Boolean); + if (parts.length === 0) throw browserError("INVALID_ARGUMENT", "key is required"); + let modifiers = 0; + for (const modifier of parts.slice(0, -1)) { + switch (modifier.toLowerCase()) { + case "alt": + modifiers |= 1; + break; + case "control": + case "ctrl": + modifiers |= 2; + break; + case "meta": + case "command": + case "cmd": + modifiers |= 4; + break; + case "shift": + modifiers |= 8; + break; + default: + throw browserError("INVALID_ARGUMENT", `Unsupported key modifier: ${modifier}`); + } + } + const rawKey = parts.at(-1)!; + const named = NAMED_KEYS[rawKey.toLowerCase()]; + if (named) { + return { + ...named, + text: modifiers & (1 | 2 | 4) ? undefined : named.text, + modifiers, + }; + } + if ([...rawKey].length !== 1) { + throw browserError("INVALID_ARGUMENT", `Unsupported key: ${rawKey}`); + } + const rawIsLetter = /^[a-zA-Z]$/.test(rawKey); + const key = rawIsLetter ? (modifiers & 8 ? rawKey.toUpperCase() : rawKey.toLowerCase()) : rawKey; + const upper = key.toUpperCase(); + const isLetter = /^[A-Z]$/.test(upper); + const isDigit = /^\d$/.test(key); + return { + key, + code: isLetter ? `Key${upper}` : isDigit ? `Digit${key}` : "", + keyCode: upper.charCodeAt(0), + text: modifiers & (1 | 2 | 4) ? undefined : key, + modifiers, + }; +} + +async function pressEntry(entry: BrowserEntry, input: string): Promise { + await ensureDebugger(entry); + const key = parseBrowserKey(input); + const params = { + key: key.key, + code: key.code, + windowsVirtualKeyCode: key.keyCode, + nativeVirtualKeyCode: key.keyCode, + modifiers: key.modifiers, + ...(key.text === undefined ? {} : { text: key.text, unmodifiedText: key.text }), + }; + await entry.view.webContents.debugger.sendCommand("Input.dispatchKeyEvent", { + type: key.text === undefined ? "rawKeyDown" : "keyDown", + ...params, + }); + await entry.view.webContents.debugger.sendCommand("Input.dispatchKeyEvent", { + type: "keyUp", + ...params, + text: undefined, + unmodifiedText: undefined, + }); + return { key: input, url: entry.view.webContents.getURL() }; +} + +async function hoverEntry(entry: BrowserEntry, refName: string): Promise { + const objectId = await resolveRef(entry, refName); + const response = (await entry.view.webContents.debugger.sendCommand("DOM.getBoxModel", { objectId })) as { + model?: { border?: number[]; content?: number[] }; + }; + const point = quadCenter(response.model?.border ?? response.model?.content); + if (!point) { + throw browserError("ELEMENT_NOT_VISIBLE", `Element ${refName} has no visible box`); + } + await entry.view.webContents.debugger.sendCommand("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + }); + return { ref: refName, x: point.x, y: point.y, url: entry.view.webContents.getURL() }; +} + +function quadCenter(quad: number[] | undefined): { x: number; y: number } | undefined { + if (!quad || quad.length < 8) return undefined; + const xs = [quad[0], quad[2], quad[4], quad[6]]; + const ys = [quad[1], quad[3], quad[5], quad[7]]; + return { + x: xs.reduce((sum, value) => sum + value, 0) / xs.length, + y: ys.reduce((sum, value) => sum + value, 0) / ys.length, + }; +} + +async function scrollEntry(entry: BrowserEntry, rawDirection: string, amount: number): Promise { + await ensureDebugger(entry); + const direction = rawDirection.toLowerCase(); + const deltas: Record = { + up: { deltaX: 0, deltaY: -amount }, + down: { deltaX: 0, deltaY: amount }, + left: { deltaX: -amount, deltaY: 0 }, + right: { deltaX: amount, deltaY: 0 }, + }; + const delta = deltas[direction]; + if (!delta) { + throw browserError("INVALID_ARGUMENT", "direction must be up, down, left, or right"); + } + const viewport = (await entry.view.webContents.debugger.sendCommand("Runtime.evaluate", { + expression: "({x: Math.max(0, innerWidth / 2), y: Math.max(0, innerHeight / 2)})", + returnByValue: true, + })) as { result?: { value?: { x?: number; y?: number } } }; + await entry.view.webContents.debugger.sendCommand("Input.dispatchMouseEvent", { + type: "mouseWheel", + x: viewport.result?.value?.x ?? 0, + y: viewport.result?.value?.y ?? 0, + ...delta, + }); + return { direction, amount, url: entry.view.webContents.getURL() }; +} + +async function selectEntry(entry: BrowserEntry, refName: string, value: string): Promise { + const objectId = await resolveRef(entry, refName); + const response = (await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function(next){ + if (!(this instanceof HTMLSelectElement)) return {supported:false}; + const values = Array.isArray(next) ? next : [next]; + const matched = Array.from(this.options).some((option) => values.includes(option.value)); + if (!matched) return {supported:true, matched:false, value:this.value}; + for (const option of this.options) option.selected = values.includes(option.value); + this.dispatchEvent(new Event('input', {bubbles:true, composed:true})); + this.dispatchEvent(new Event('change', {bubbles:true, composed:true})); + return {supported:true, matched:true, value:this.value}; + }`, + arguments: [{ value }], + returnByValue: true, + })) as { result?: { value?: { supported?: boolean; matched?: boolean; value?: string } } }; + if (!response.result?.value?.supported) { + throw browserError("INVALID_ELEMENT_STATE", `Element ${refName} is not a select control`); + } + if (!response.result.value.matched) { + throw browserError("INVALID_ARGUMENT", `Select option ${JSON.stringify(value)} does not exist`); + } + return { ref: refName, value: response.result.value.value, url: entry.view.webContents.getURL() }; +} + +async function checkEntry(entry: BrowserEntry, refName: string, checked: boolean): Promise { + const objectId = await resolveRef(entry, refName); + const response = (await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function(next){ + if (!('checked' in this)) return {supported:false}; + if (Boolean(this.checked) !== Boolean(next)) this.click(); + return {supported:true, checked:Boolean(this.checked)}; + }`, + arguments: [{ value: checked }], + returnByValue: true, + })) as { result?: { value?: { supported?: boolean; checked?: boolean } } }; + if (!response.result?.value?.supported) { + throw browserError("INVALID_ELEMENT_STATE", `Element ${refName} is not checkable`); + } + if (response.result.value.checked !== checked) { + throw browserError("ELEMENT_NOT_INTERACTABLE", `Element ${refName} did not change checked state`); + } + return { ref: refName, checked: response.result.value.checked, url: entry.view.webContents.getURL() }; +} + +async function getEntry(entry: BrowserEntry, property: string, refName?: string): Promise { + const normalized = property.toLowerCase(); + if (!refName) { + if (normalized === "url") return { property: normalized, value: entry.view.webContents.getURL() }; + if (normalized === "title") return { property: normalized, value: entry.view.webContents.getTitle() }; + if (normalized !== "text") { + throw browserError("INVALID_ARGUMENT", "page property must be url, title, or text"); + } + await ensureDebugger(entry); + const response = (await entry.view.webContents.debugger.sendCommand("Runtime.evaluate", { + expression: "document.body ? document.body.innerText : ''", + returnByValue: true, + })) as { result?: { value?: unknown } }; + return { property: normalized, value: response.result?.value ?? "" }; + } + if (!["text", "value", "checked"].includes(normalized)) { + throw browserError("INVALID_ARGUMENT", "element property must be text, value, or checked"); + } + const objectId = await resolveRef(entry, refName); + const response = (await entry.view.webContents.debugger.sendCommand("Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function(property){ + if (property === 'text') return this.innerText ?? this.textContent ?? ''; + if (property === 'value') return this.value ?? ''; + if (property === 'checked') return Boolean(this.checked); + }`, + arguments: [{ value: normalized }], + returnByValue: true, + })) as { result?: { value?: unknown } }; + return { + ref: refName, + property: normalized, + value: response.result?.value, + url: entry.view.webContents.getURL(), + }; +} + async function resolveRef(entry: BrowserEntry, refName: string): Promise { await ensureDebugger(entry); const ref = entry.refs.get(refName); From 849183d2a3fad7794aa3702a28a2c760b6607dac Mon Sep 17 00:00:00 2001 From: aprv10 <1apoorvindia@gmail.com> Date: Thu, 23 Jul 2026 17:38:32 +0530 Subject: [PATCH 04/11] feat(browser): add highlighting and stable tabs --- backend/internal/cli/browser.go | 87 +++++ backend/internal/cli/browser_test.go | 24 ++ backend/internal/httpd/controllers/browser.go | 42 ++- .../httpd/controllers/browser_test.go | 6 +- .../skillassets/using-ao/commands/browser.md | 14 + docs/STATUS.md | 3 +- docs/cli/README.md | 10 +- frontend/src/main/browser-view-host.test.ts | 169 +++++++++ frontend/src/main/browser-view-host.ts | 338 ++++++++++++++---- 9 files changed, 609 insertions(+), 84 deletions(-) diff --git a/backend/internal/cli/browser.go b/backend/internal/cli/browser.go index 660d69e39c..98fad34b9e 100644 --- a/backend/internal/cli/browser.go +++ b/backend/internal/cli/browser.go @@ -134,6 +134,72 @@ func newBrowserCommand(ctx *commandContext) *cobra.Command { }, }) + cmd.AddCommand(&cobra.Command{ + Use: "highlight ", + Short: "Visually highlight an element without changing page state", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "highlight", map[string]any{"ref": args[0]}, jsonOutput) + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "unhighlight", + Short: "Remove the current element highlight", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return ctx.runBrowserAction(cmd, "unhighlight", nil, jsonOutput) + }, + }) + + cmd.AddCommand(&cobra.Command{ + Use: "tabs", + Short: "List this session's browser tabs", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return ctx.runBrowserAction(cmd, "tabs", nil, jsonOutput) + }, + }) + + tabCmd := &cobra.Command{ + Use: "tab", + Short: "Create, select, or close a browser tab", + Args: noArgs, + } + tabCmd.AddCommand(&cobra.Command{ + Use: "new [url]", + Short: "Open a new tab, optionally navigating to a URL", + Args: atMostOneArg, + RunE: func(cmd *cobra.Command, args []string) error { + actionArgs := map[string]any{} + if len(args) == 1 { + actionArgs["url"] = args[0] + } + return ctx.runBrowserAction(cmd, "tab-new", actionArgs, jsonOutput) + }, + }) + tabCmd.AddCommand(&cobra.Command{ + Use: "select ", + Short: "Make a browser tab active", + Args: exactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return ctx.runBrowserAction(cmd, "tab-select", map[string]any{"tabId": args[0]}, jsonOutput) + }, + }) + tabCmd.AddCommand(&cobra.Command{ + Use: "close [tab-id]", + Short: "Close a browser tab, defaulting to the active tab", + Args: atMostOneArg, + RunE: func(cmd *cobra.Command, args []string) error { + actionArgs := map[string]any{} + if len(args) == 1 { + actionArgs["tabId"] = args[0] + } + return ctx.runBrowserAction(cmd, "tab-close", actionArgs, jsonOutput) + }, + }) + cmd.AddCommand(tabCmd) + var scrollAmount int scroll := &cobra.Command{ Use: "scroll ", @@ -352,6 +418,27 @@ func writeBrowserResult(cmd *cobra.Command, action string, result map[string]any return err } } + if action == "tabs" { + tabs, _ := result["tabs"].([]any) + if len(tabs) == 0 { + _, err := fmt.Fprintln(cmd.OutOrStdout(), "No browser tabs.") + return err + } + for _, raw := range tabs { + tab, _ := raw.(map[string]any) + id, _ := tab["id"].(string) + title, _ := tab["title"].(string) + currentURL, _ := tab["url"].(string) + marker := " " + if active, _ := tab["active"].(bool); active { + marker = "*" + } + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s %s\t%s\t%s\n", marker, id, title, currentURL); err != nil { + return err + } + } + return nil + } if currentURL, ok := result["url"].(string); ok && currentURL != "" { _, err := fmt.Fprintln(cmd.OutOrStdout(), currentURL) return err diff --git a/backend/internal/cli/browser_test.go b/backend/internal/cli/browser_test.go index 29da1eab39..9c27a1e7a7 100644 --- a/backend/internal/cli/browser_test.go +++ b/backend/internal/cli/browser_test.go @@ -38,6 +38,8 @@ func browserCLIServer(t *testing.T, capture *browserRequestCapture) *httptest.Se result = `{"text":"button Save [ref=e1]"}` case "screenshot": result = `{"data":"cG5n","width":10,"height":20}` + case "tabs": + result = `{"activeTabId":"t2","tabs":[{"id":"t1","title":"First","url":"http://localhost:3000/","active":false},{"id":"t2","title":"Second","url":"http://localhost:4173/","active":true}]}` } _, _ = io.WriteString(w, `{"requestId":"r1","sessionId":"ao-1","action":"`+capture.body.Action+`","result":`+result+`}`) })) @@ -108,6 +110,12 @@ func TestBrowserCoreInteractionArguments(t *testing.T) { {name: "type", args: []string{"type", "e1", "hello"}, action: "type", want: map[string]any{"ref": "e1", "text": "hello"}}, {name: "press", args: []string{"press", "Control+A"}, action: "press", want: map[string]any{"key": "Control+A"}}, {name: "hover", args: []string{"hover", "e2"}, action: "hover", want: map[string]any{"ref": "e2"}}, + {name: "highlight", args: []string{"highlight", "e2"}, action: "highlight", want: map[string]any{"ref": "e2"}}, + {name: "unhighlight", args: []string{"unhighlight"}, action: "unhighlight", want: map[string]any{}}, + {name: "tabs", args: []string{"tabs"}, action: "tabs", want: map[string]any{}}, + {name: "tab new", args: []string{"tab", "new", "localhost:4173"}, action: "tab-new", want: map[string]any{"url": "localhost:4173"}}, + {name: "tab select", args: []string{"tab", "select", "t2"}, action: "tab-select", want: map[string]any{"tabId": "t2"}}, + {name: "tab close", args: []string{"tab", "close", "t1"}, action: "tab-close", want: map[string]any{"tabId": "t1"}}, {name: "scroll", args: []string{"scroll", "down", "--amount", "450"}, action: "scroll", want: map[string]any{"direction": "down", "amount": float64(450)}}, {name: "select", args: []string{"select", "e3", "large"}, action: "select", want: map[string]any{"ref": "e3", "value": "large"}}, {name: "check", args: []string{"check", "e4"}, action: "check", want: map[string]any{"ref": "e4"}}, @@ -131,6 +139,22 @@ func TestBrowserCoreInteractionArguments(t *testing.T) { } } +func TestBrowserTabsPrintStableIDsAndActiveTab(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + + out, errOut, err := executeCLI(t, Deps{ProcessAlive: func(int) bool { return true }}, "browser", "tabs") + if err != nil { + t.Fatalf("tabs err=%v stderr=%s", err, errOut) + } + if !strings.Contains(out, " t1") || !strings.Contains(out, "* t2") { + t.Fatalf("tabs output = %q", out) + } +} + func TestBrowserScreenshotWritesWithoutOverwrite(t *testing.T) { t.Setenv("AO_SESSION_ID", "ao-1") cfg := setConfigEnv(t) diff --git a/backend/internal/httpd/controllers/browser.go b/backend/internal/httpd/controllers/browser.go index 96222f1e0d..3bb8c5eb50 100644 --- a/backend/internal/httpd/controllers/browser.go +++ b/backend/internal/httpd/controllers/browser.go @@ -15,22 +15,28 @@ import ( ) var browserActions = map[string]struct{}{ - "open": {}, - "snapshot": {}, - "click": {}, - "fill": {}, - "type": {}, - "press": {}, - "hover": {}, - "scroll": {}, - "select": {}, - "check": {}, - "uncheck": {}, - "get": {}, - "wait": {}, - "screenshot": {}, - "console": {}, - "errors": {}, + "open": {}, + "snapshot": {}, + "click": {}, + "fill": {}, + "type": {}, + "press": {}, + "hover": {}, + "highlight": {}, + "unhighlight": {}, + "tabs": {}, + "tab-new": {}, + "tab-select": {}, + "tab-close": {}, + "scroll": {}, + "select": {}, + "check": {}, + "uncheck": {}, + "get": {}, + "wait": {}, + "screenshot": {}, + "console": {}, + "errors": {}, } type BrowserRuntime interface { @@ -121,10 +127,10 @@ func writeBrowserError(w http.ResponseWriter, r *http.Request, err error) { status := http.StatusUnprocessableEntity typeName := "unprocessable" switch commandErr.Code { - case "INVALID_ARGUMENT", "URL_REQUIRED", "REFERENCE_REQUIRED": + case "INVALID_ARGUMENT", "URL_REQUIRED", "REFERENCE_REQUIRED", "TAB_ID_REQUIRED": status = http.StatusBadRequest typeName = "bad_request" - case "STALE_REFERENCE": + case "STALE_REFERENCE", "TAB_NOT_FOUND": status = http.StatusConflict typeName = "conflict" case "BROWSER_TARGET_UNAVAILABLE": diff --git a/backend/internal/httpd/controllers/browser_test.go b/backend/internal/httpd/controllers/browser_test.go index aafc703e12..c90afc2936 100644 --- a/backend/internal/httpd/controllers/browser_test.go +++ b/backend/internal/httpd/controllers/browser_test.go @@ -70,7 +70,11 @@ func TestBrowserStatusAndSnapshot(t *testing.T) { func TestBrowserCoreInteractionActionsReachRuntime(t *testing.T) { runtime := &fakeBrowserRuntime{} srv := browserServer(t, runtime) - actions := []string{"type", "press", "hover", "scroll", "select", "check", "uncheck", "get"} + actions := []string{ + "type", "press", "hover", "highlight", "unhighlight", + "tabs", "tab-new", "tab-select", "tab-close", + "scroll", "select", "check", "uncheck", "get", + } for _, action := range actions { t.Run(action, func(t *testing.T) { diff --git a/backend/internal/skillassets/using-ao/commands/browser.md b/backend/internal/skillassets/using-ao/commands/browser.md index b559e7c94e..f8471da987 100644 --- a/backend/internal/skillassets/using-ao/commands/browser.md +++ b/backend/internal/skillassets/using-ao/commands/browser.md @@ -34,6 +34,12 @@ ao browser fill [--json] ao browser type [--json] ao browser press [--json] ao browser hover [--json] +ao browser highlight [--json] +ao browser unhighlight [--json] +ao browser tabs [--json] +ao browser tab new [url] [--json] +ao browser tab select [--json] +ao browser tab close [tab-id] [--json] ao browser scroll [--amount ] [--json] ao browser select [--json] ao browser check [--json] @@ -49,6 +55,14 @@ ao browser errors [--json] cursor position. `press` accepts named keys and chords such as `Enter`, `ArrowDown`, and `Control+A`. Page-level `get` supports `url`, `title`, and `text`; with an element ref it supports `text`, `value`, and `checked`. +`highlight` draws a non-mutating overlay around a snapshot ref until +`unhighlight`, navigation, or target replacement. +`tabs` reports stable logical IDs such as `t1` and marks the active tab. +`tab new` creates and selects a tab, `tab select` changes the target of all +following browser commands, and `tab close` defaults to the active tab. +Allowed page popups are captured as new AO tabs instead of opening a separate +OS browser. Take a new snapshot after switching tabs because element refs are +invalidated at the tab boundary. Without `--json`, `screenshot` writes a PNG and refuses to overwrite an existing file. With `--json`, it returns the structured response including base64 image data. diff --git a/docs/STATUS.md b/docs/STATUS.md index 30e5bb114c..550ac931d9 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -65,7 +65,8 @@ surface (`npm run sqlc`, `npm run api`). daemon↔Electron bridge drives only the selected session's `WebContentsView` through Electron's bound debugger transport. `ao browser` supports open, compact accessibility snapshots and refs, click/fill/type, keyboard input, - hover, scrolling, selection and checked state, property reads, waits, + hover and non-mutating element highlighting, scrolling, selection and checked + state, property reads, stable logical tabs and captured popups, waits, screenshots, console messages, and page errors while the Browser panel is hidden. - Real daemon wiring via the generated `openapi-fetch` typed client diff --git a/docs/cli/README.md b/docs/cli/README.md index be01d6b5f1..cba4bcaf26 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -85,9 +85,13 @@ opens that URL verbatim (`file://`, `http`, `https`). session-owned live Electron browser rather than only setting its preview URL. The target-isolated command set includes `status`, `open`, `snapshot`, `click`, `fill`, `type`, `press`, `hover`, `scroll`, `select`, `check`, `uncheck`, `get`, -`wait`, `screenshot`, `console`, and `errors`. The AO desktop app must be open -because Electron owns the `WebContentsView`. References from a snapshot are -invalidated after navigation or DOM replacement; take another snapshot when a +`highlight`, `unhighlight`, `tabs`, `tab new`, `tab select`, `tab close`, +`wait`, `screenshot`, `console`, and `errors`. Logical tab IDs remain stable +for the session, and allowed popups become AO browser tabs rather than separate +OS-browser windows. The AO desktop app must be open because Electron owns the +`WebContentsView`. +References from a snapshot are invalidated after navigation or DOM replacement; +they are also invalidated when changing tabs. Take another snapshot when a command reports `STALE_REFERENCE`. `go run .` in `backend/` remains a compatibility wrapper around the daemon. diff --git a/frontend/src/main/browser-view-host.test.ts b/frontend/src/main/browser-view-host.test.ts index 8ad785e9e3..1e17c4348d 100644 --- a/frontend/src/main/browser-view-host.test.ts +++ b/frontend/src/main/browser-view-host.test.ts @@ -153,6 +153,109 @@ function setupHost() { }; } +function setupTabHost() { + const views: Array<{ + webContents: { + id: number; + getURL: () => string; + loadURL: ReturnType; + openWindow: (url: string) => void; + close: ReturnType; + }; + setBounds: ReturnType; + setVisible: ReturnType; + }> = []; + let nextID = 100; + const makeView = () => { + let currentURL = ""; + let windowOpenHandler: ((details: { url: string }) => { action: string }) | undefined; + const listeners = new Map void>(); + let debuggerAttached = false; + const webContents = { + id: nextID++, + mainFrame: {}, + canGoBack: () => false, + canGoForward: () => false, + capturePage: vi.fn(async () => ({ + isEmpty: () => false, + toJPEG: () => Buffer.from("snapshot"), + toPNG: () => Buffer.from("snapshot"), + getSize: () => ({ width: 640, height: 480 }), + })), + clearHistory: () => undefined, + debugger: { + attach: () => { + debuggerAttached = true; + }, + detach: () => { + debuggerAttached = false; + }, + isAttached: () => debuggerAttached, + on: () => undefined, + sendCommand: async (method: string) => { + if (method === "Runtime.evaluate") return { result: { value: true } }; + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { + nodeId: "1", + backendDOMNodeId: 42, + role: { value: "button" }, + name: { value: "Open" }, + }, + ], + }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "button" } }; + return {}; + }, + }, + getTitle: () => (currentURL ? `Title ${currentURL}` : ""), + getURL: () => currentURL, + goBack: () => undefined, + goForward: () => undefined, + isLoading: () => false, + loadURL: vi.fn(async (url: string) => { + currentURL = url; + }), + on: (event: string, listener: (...args: never[]) => void) => listeners.set(event, listener), + reload: () => undefined, + send: () => undefined, + setWindowOpenHandler: (handler: (details: { url: string }) => { action: string }) => { + windowOpenHandler = handler; + }, + stop: () => undefined, + close: vi.fn(), + openWindow: (url: string) => { + windowOpenHandler?.({ url }); + }, + }; + const view = { webContents, setBounds: vi.fn(), setVisible: vi.fn() }; + views.push(view); + return view; + }; + const host = createBrowserViewHost({ + mainWindow: { + contentView: { addChildView: () => undefined, removeChildView: () => undefined }, + getContentBounds: () => ({ x: 0, y: 0, width: 800, height: 600 }), + webContents: { id: 1, focus: () => undefined, send: () => undefined }, + } as never, + ipcMain: { + handle: () => undefined, + on: () => undefined, + removeHandler: () => undefined, + off: () => undefined, + } as never, + shell: { openExternal: async () => undefined }, + WebContentsView: function () { + return makeView(); + } as never, + annotatePreloadPath: "/preload.js", + rendererOrigin: "http://localhost:5173", + }); + return { host, views }; +} + describe("new-session shortcut forwarding", () => { it("focuses the shell before forwarding a matching preview chord", async () => { const { emitBeforeInput, invoke, shellFocus, shellSend } = setupHost(); @@ -376,6 +479,8 @@ describe("agent browser runtime", () => { await host.execute("sess-1", "type", { ref: "e1", text: "hello" }); await host.execute("sess-1", "press", { key: "Control+A" }); await host.execute("sess-1", "hover", { ref: "e1" }); + await host.execute("sess-1", "highlight", { ref: "e1" }); + await host.execute("sess-1", "unhighlight"); await host.execute("sess-1", "scroll", { direction: "down", amount: 450 }); await host.execute("sess-1", "select", { ref: "e1", value: "large" }); await host.execute("sess-1", "check", { ref: "e1" }); @@ -396,6 +501,16 @@ describe("agent browser runtime", () => { x: 20, y: 30, }); + expect(debuggerSendCommand).toHaveBeenCalledWith( + "Overlay.highlightNode", + expect.objectContaining({ + objectId: "target-element", + highlightConfig: expect.objectContaining({ + borderColor: { r: 37, g: 99, b: 235, a: 1 }, + }), + }), + ); + expect(debuggerSendCommand).toHaveBeenCalledWith("Overlay.hideHighlight"); expect(debuggerSendCommand).toHaveBeenCalledWith( "Input.dispatchMouseEvent", expect.objectContaining({ type: "mouseWheel", deltaY: 450, x: 400, y: 300 }), @@ -453,6 +568,60 @@ describe("agent browser runtime", () => { expect(screenshot.width).toBe(640); expect(errors.messages.map((entry) => entry.message)).toEqual(["boom"]); }); + + it("keeps stable logical tab IDs, separate targets, and the selected tab active", async () => { + const { host, views } = setupTabHost(); + await host.execute("sess-1", "open", { url: "http://localhost:3000" }); + await host.execute("sess-1", "snapshot"); + const created = (await host.execute("sess-1", "tab-new", { + url: "http://localhost:4173", + })) as { id: string }; + + const listed = (await host.execute("sess-1", "tabs")) as { + activeTabId: string; + tabs: Array<{ id: string; url: string; active: boolean }>; + }; + expect(created.id).toBe("t2"); + expect(listed.activeTabId).toBe("t2"); + expect(listed.tabs).toEqual([ + expect.objectContaining({ id: "t1", url: "http://localhost:3000/", active: false }), + expect.objectContaining({ id: "t2", url: "http://localhost:4173/", active: true }), + ]); + expect(views).toHaveLength(2); + + await host.execute("sess-1", "tab-select", { tabId: "t1" }); + const current = (await host.execute("sess-1", "get", { property: "url" })) as { value: string }; + expect(current.value).toBe("http://localhost:3000/"); + expect(views[1].setVisible).toHaveBeenLastCalledWith(false); + await expect(host.execute("sess-1", "click", { ref: "e1" })).rejects.toMatchObject({ + code: "STALE_REFERENCE", + }); + await host.execute("sess-1", "tab-close", { tabId: "t2" }); + const replacement = (await host.execute("sess-1", "tab-new")) as { id: string }; + expect(replacement.id).toBe("t3"); + }); + + it("captures allowed popups as new tabs and protects the final tab", async () => { + const { host, views } = setupTabHost(); + await host.execute("sess-1", "open", { url: "http://localhost:3000" }); + + views[0].webContents.openWindow("http://localhost:3000/popup"); + await Promise.resolve(); + + const listed = (await host.execute("sess-1", "tabs")) as { + activeTabId: string; + tabs: Array<{ id: string; url: string }>; + }; + expect(listed.activeTabId).toBe("t2"); + expect(listed.tabs[1]).toEqual( + expect.objectContaining({ id: "t2", url: "http://localhost:3000/popup" }), + ); + + await host.execute("sess-1", "tab-close"); + await expect(host.execute("sess-1", "tab-close")).rejects.toMatchObject({ + code: "CANNOT_CLOSE_LAST_TAB", + }); + }); }); describe("browser:requestMirror", () => { diff --git a/frontend/src/main/browser-view-host.ts b/frontend/src/main/browser-view-host.ts index 95c4edc9b0..ecbb26171b 100644 --- a/frontend/src/main/browser-view-host.ts +++ b/frontend/src/main/browser-view-host.ts @@ -114,6 +114,7 @@ export type BrowserViewHost = { type BrowserEntry = { sessionId: string; + tabId: string; view: BrowserViewLike; state: BrowserNavState; annotationEnabled: boolean; @@ -123,6 +124,17 @@ type BrowserEntry = { errors: BrowserLogEntry[]; }; +type BrowserSessionEntry = { + sessionId: string; + viewId: string; + tabs: Map; + activeTabId: string; + nextTabNumber: number; + bounds: BrowserRect; + visible: boolean; + parked: boolean; +}; + type BrowserLogEntry = { level: string; message: string; @@ -191,10 +203,10 @@ export function scaleBoundsForZoom(rect: BrowserRect, zoomFactor: number): Brows } export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserViewHost { - const entries = new Map(); + const entries = new Map(); const viewIdsBySessionId = new Map(); const rendererOwnersByViewId = new Map>(); - const viewIdsByWebContentsId = new Map(); + const tabsByWebContentsId = new Map(); const ipcDisposers: Array<() => void> = []; // viewId of the panel that most recently held focus; cleared when it is hidden or destroyed. let lastFocusedViewId: string | null = null; @@ -212,13 +224,13 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV displayMediaSession!.setDisplayMediaRequestHandler((request, callback) => { const pending = pendingMirror; pendingMirror = null; - const entry = + const session = pending && pending.expires > Date.now() && sameFrame(pending.frame, request.frame) ? entries.get(pending.viewId) : undefined; try { - if (entry) { - callback({ video: entry.view.webContents.mainFrame }); + if (session) { + callback({ video: activeEntry(session).view.webContents.mainFrame }); } else { callback({}); } @@ -235,10 +247,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }); } - const ensure = (viewId: string, sessionId: string): BrowserEntry => { - const existing = entries.get(viewId); - if (existing) return existing; - + const createTab = (session: BrowserSessionEntry, activate: boolean): BrowserEntry => { const view = new options.WebContentsView({ webPreferences: { contextIsolation: true, @@ -251,9 +260,11 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV view.setVisible?.(false); options.mainWindow.contentView.addChildView(view); - const state: BrowserNavState = emptyNavState(viewId); + const tabId = `t${session.nextTabNumber++}`; + const state: BrowserNavState = emptyNavState(session.viewId); const entry: BrowserEntry = { - sessionId, + sessionId: session.sessionId, + tabId, view, state, annotationEnabled: false, @@ -262,65 +273,150 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV consoleMessages: [], errors: [], }; - entries.set(viewId, entry); - viewIdsBySessionId.set(sessionId, viewId); - viewIdsByWebContentsId.set(view.webContents.id, viewId); - hardenWebContents(view.webContents, options, entry); - wireNavEvents(view.webContents, options, entry); + session.tabs.set(tabId, entry); + tabsByWebContentsId.set(view.webContents.id, entry); + hardenWebContents(view.webContents, options, entry, (url) => { + void openTab(session, url, true).catch((error) => { + pushBrowserLog(entry.errors, { + level: "error", + message: error instanceof Error ? error.message : "Unable to open browser popup", + timestamp: new Date().toISOString(), + }); + }); + }); + wireNavEvents( + view.webContents, + options, + entry, + () => entries.get(session.viewId)?.activeTabId === entry.tabId, + () => applySessionBounds(session, entry), + ); wireAutomationEvents(view.webContents, entry); // The preview is a separate WebContentsView, so renderer-window keydown // listeners never see keys typed here. Forward application shortcuts to the // shell renderer so they still work with the panel focused. attachAppShortcuts(view.webContents, Boolean(options.isMac), options.mainWindow.webContents, true); view.webContents.on("focus", () => { - lastFocusedViewId = viewId; + lastFocusedViewId = session.viewId; }); + if (activate) activateTab(session, tabId); return entry; }; - const ensureSession = (sessionId: string, rendererId?: number): BrowserEntry => { + const ensureSession = (sessionId: string, rendererId?: number): BrowserSessionEntry => { const existingViewId = viewIdsBySessionId.get(sessionId); const viewId = existingViewId ?? `${rendererId ?? 0}:${sessionId}`; - const entry = ensure(viewId, sessionId); + let session = entries.get(viewId); + if (!session) { + session = { + sessionId, + viewId, + tabs: new Map(), + activeTabId: "", + nextTabNumber: 1, + bounds: OFFSCREEN_BOUNDS, + visible: false, + parked: false, + }; + entries.set(viewId, session); + viewIdsBySessionId.set(sessionId, viewId); + createTab(session, true); + } if (rendererId !== undefined) { const owners = rendererOwnersByViewId.get(viewId) ?? new Set(); owners.add(rendererId); rendererOwnersByViewId.set(viewId, owners); } + return session; + }; + + const openTab = async (session: BrowserSessionEntry, url: string | undefined, activate: boolean): Promise => { + let normalizedURL: string | undefined; + if (url) { + const normalized = normalizeBrowserURL(url); + if (!isAllowedBrowserURL(normalized.href, options.rendererOrigin)) { + throw browserError("NAVIGATION_FAILED", "Unsupported browser URL"); + } + normalizedURL = normalized.href; + } + const entry = createTab(session, activate); + if (normalizedURL) { + const state = await navigateEntry(entry, normalizedURL); + if (state.error) throw browserError("NAVIGATION_FAILED", state.error); + } return entry; }; + function activateTab(session: BrowserSessionEntry, tabId: string): BrowserEntry { + const next = session.tabs.get(tabId); + if (!next) throw browserError("TAB_NOT_FOUND", `Browser tab ${tabId} does not exist`); + const previous = session.tabs.get(session.activeTabId); + if (previous && previous !== next) { + invalidateRefs(previous); + previous.view.setVisible?.(false); + previous.view.setBounds(OFFSCREEN_BOUNDS); + } + session.activeTabId = tabId; + invalidateRefs(next); + applySessionBounds(session, next); + pushNavState(options, next); + return next; + } + + function applySessionBounds(session: BrowserSessionEntry, entry: BrowserEntry): void { + if (!session.visible) { + entry.view.setVisible?.(false); + entry.view.setBounds(OFFSCREEN_BOUNDS); + return; + } + entry.view.setBounds(session.bounds); + entry.view.setVisible?.(session.parked || (session.bounds.width > 0 && session.bounds.height > 0)); + } + const isRendererOwned = (event: IpcMainInvokeEvent | IpcMainEvent, viewId: string): boolean => rendererOwnersByViewId.get(viewId)?.has(event.sender.id) ?? false; const setBounds = ({ viewId, rect, visible, parked }: BrowserBoundsInput, zoomFactor = 1): void => { - const entry = entries.get(viewId); - if (!entry) return; + const session = entries.get(viewId); + if (!session) return; + const entry = activeEntry(session); if (parked) { const scaled = scaleBoundsForZoom(rect, zoomFactor); const width = Math.max(1, Math.round(scaled.width)); const height = Math.max(1, Math.round(scaled.height)); - entry.view.setBounds({ x: OFFSCREEN_BOUNDS.x, y: 0, width, height }); - entry.view.setVisible?.(true); + session.bounds = { x: OFFSCREEN_BOUNDS.x, y: 0, width, height }; + session.visible = true; + session.parked = true; + applySessionBounds(session, entry); return; } if (!visible) { - entry.view.setVisible?.(false); - entry.view.setBounds(OFFSCREEN_BOUNDS); + session.bounds = OFFSCREEN_BOUNDS; + session.visible = false; + session.parked = false; + applySessionBounds(session, entry); forgetIfFocused(viewId); return; } // The renderer measures the slot in page-zoomed CSS pixels, while // WebContentsView bounds are window coordinates. Convert before clamping so // Cmd+/Cmd- page zoom does not detach the native view from its React slot. - const bounds = clampBoundsToWindow(scaleBoundsForZoom(rect, zoomFactor), options.mainWindow.getContentBounds()); - entry.view.setBounds(bounds); - entry.view.setVisible?.(bounds.width > 0 && bounds.height > 0); + session.bounds = clampBoundsToWindow( + scaleBoundsForZoom(rect, zoomFactor), + options.mainWindow.getContentBounds(), + ); + session.visible = true; + session.parked = false; + applySessionBounds(session, entry); }; const navigate = async ({ viewId, url }: BrowserNavigateInput): Promise => { - const entry = entries.get(viewId); - if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); + const session = entries.get(viewId); + if (!session) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); + return navigateEntry(activeEntry(session), url); + }; + + const navigateEntry = async (entry: BrowserEntry, url: string): Promise => { cancelAnnotation(options, entry, "navigation"); const normalized = normalizeBrowserURL(url); if (!isAllowedBrowserURL(normalized.href, options.rendererOrigin)) { @@ -335,7 +431,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV options.mainWindow.webContents.send("browser:navState", entry.state); return entry.state; } - entry.view.setVisible?.(true); + const session = entries.get(entry.state.viewId); + if (session?.activeTabId === entry.tabId) applySessionBounds(session, entry); return pushNavState(options, entry); }; @@ -344,11 +441,14 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV // readNavState normalizes it back to an empty url so the panel shows its // empty state. const clear = async (viewId: string): Promise => { - const entry = entries.get(viewId); - if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); + const session = entries.get(viewId); + if (!session) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Browser target is unavailable"); + const entry = activeEntry(session); cancelAnnotation(options, entry, "navigation"); - entry.view.setVisible?.(false); - entry.view.setBounds(OFFSCREEN_BOUNDS); + session.visible = false; + session.parked = false; + session.bounds = OFFSCREEN_BOUNDS; + applySessionBounds(session, entry); forgetIfFocused(viewId); await entry.view.webContents.loadURL("about:blank"); entry.view.webContents.clearHistory(); @@ -356,8 +456,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }; const capture = async (viewId: string): Promise => { - const entry = entries.get(viewId); - if (!entry) return ""; + const session = entries.get(viewId); + if (!session) return ""; + const entry = activeEntry(session); try { const image = await entry.view.webContents.capturePage(); if (image.isEmpty()) return ""; @@ -368,17 +469,26 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }; const destroy = (viewId: string): void => { - const entry = entries.get(viewId); - if (!entry) return; + const session = entries.get(viewId); + if (!session) return; entries.delete(viewId); - viewIdsBySessionId.delete(entry.sessionId); + viewIdsBySessionId.delete(session.sessionId); rendererOwnersByViewId.delete(viewId); - viewIdsByWebContentsId.delete(entry.view.webContents.id); forgetIfFocused(viewId); // When the window is already gone (dispose fired from mainWindow "closed"), // Electron has torn down contentView and the child WebContentsViews. Touching // them throws "Object has been destroyed", so just drop our reference. - if (options.mainWindow.isDestroyed?.()) return; + if (options.mainWindow.isDestroyed?.()) { + for (const entry of session.tabs.values()) tabsByWebContentsId.delete(entry.view.webContents.id); + return; + } + for (const entry of session.tabs.values()) { + tabsByWebContentsId.delete(entry.view.webContents.id); + destroyTabView(entry); + } + }; + + const destroyTabView = (entry: BrowserEntry): void => { entry.view.setVisible?.(false); entry.view.setBounds(OFFSCREEN_BOUNDS); options.mainWindow.contentView.removeChildView?.(entry.view); @@ -393,8 +503,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV action: (contents: BrowserWebContents) => void, cancelForNavigation = false, ): BrowserNavState => { - const entry = entries.get(viewId); - if (!entry) return emptyNavState(viewId); + const session = entries.get(viewId); + if (!session) return emptyNavState(viewId); + const entry = activeEntry(session); if (cancelForNavigation) cancelAnnotation(options, entry, "navigation"); action(entry.view.webContents); return pushNavState(options, entry); @@ -402,8 +513,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV const setAnnotationMode = (event: IpcMainInvokeEvent, input: BrowserAnnotationModeInput): void => { if (!isRendererOwned(event, input.viewId)) return; - const entry = entries.get(input.viewId); - if (!entry) return; + const session = entries.get(input.viewId); + if (!session) return; + const entry = activeEntry(session); entry.annotationEnabled = input.enabled; entry.view.webContents.send("browser:annotation:setMode", { enabled: input.enabled }); }; @@ -412,8 +524,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV event: IpcMainEvent, payload: BrowserAnnotationPageSubmitPayload | undefined, ): void => { - const viewId = viewIdsByWebContentsId.get(event.sender.id); - const entry = viewId ? entries.get(viewId) : undefined; + const entry = tabsByWebContentsId.get(event.sender.id); + const viewId = entry?.state.viewId; if ( !viewId || !entry || @@ -437,8 +549,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV event: IpcMainEvent, payload: BrowserAnnotationPageCancelPayload | undefined, ): void => { - const viewId = viewIdsByWebContentsId.get(event.sender.id); - const entry = viewId ? entries.get(viewId) : undefined; + const entry = tabsByWebContentsId.get(event.sender.id); + const viewId = entry?.state.viewId; if (!viewId || !entry) return; entry.annotationEnabled = false; const forwarded: BrowserAnnotationCancelPayload = { @@ -461,7 +573,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }; handle("browser:ensure", (event, sessionId: string) => - pushNavState(options, ensureSession(sessionId, event.sender.id)), + pushNavState(options, activeEntry(ensureSession(sessionId, event.sender.id))), ); on("browser:setBounds", (event, input: BrowserBoundsInput) => { if (isRendererOwned(event, input.viewId)) setBounds(input, event.sender.getZoomFactor()); @@ -508,7 +620,8 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV return { execute: async (sessionId, action, args = {}) => { if (!sessionId.trim()) throw browserError("INVALID_ARGUMENT", "sessionId is required"); - const entry = ensureSession(sessionId); + const session = ensureSession(sessionId); + const entry = activeEntry(session); switch (action) { case "open": { const url = stringArg(args, "url", "URL_REQUIRED", "url is required"); @@ -536,6 +649,42 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV return pressEntry(entry, stringArg(args, "key", "INVALID_ARGUMENT", "key is required")); case "hover": return hoverEntry(entry, stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required")); + case "highlight": + return highlightEntry(entry, stringArg(args, "ref", "REFERENCE_REQUIRED", "ref is required")); + case "unhighlight": + return unhighlightEntry(entry); + case "tabs": + return listTabs(session); + case "tab-new": { + const url = typeof args.url === "string" && args.url.trim() ? args.url : undefined; + const tab = await openTab(session, url, true); + return tabResult(tab, true); + } + case "tab-select": { + const tab = activateTab( + session, + stringArg(args, "tabId", "TAB_ID_REQUIRED", "tabId is required"), + ); + return tabResult(tab, true); + } + case "tab-close": { + if (session.tabs.size === 1) { + throw browserError("CANNOT_CLOSE_LAST_TAB", "The only browser tab cannot be closed"); + } + const tabId = + typeof args.tabId === "string" && args.tabId.trim() ? args.tabId.trim() : session.activeTabId; + const tab = session.tabs.get(tabId); + if (!tab) throw browserError("TAB_NOT_FOUND", `Browser tab ${tabId} does not exist`); + const wasActive = tabId === session.activeTabId; + session.tabs.delete(tabId); + tabsByWebContentsId.delete(tab.view.webContents.id); + destroyTabView(tab); + if (wasActive) { + const nextTabId = [...session.tabs.keys()].at(-1)!; + activateTab(session, nextTabId); + } + return { closedTabId: tabId, ...listTabs(session) }; + } case "scroll": return scrollEntry( entry, @@ -592,8 +741,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV }, getLastFocusedPanelContents: () => { if (lastFocusedViewId === null) return null; - const entry = entries.get(lastFocusedViewId); - if (!entry) return null; + const session = entries.get(lastFocusedViewId); + if (!session) return null; + const entry = activeEntry(session); // Stored narrowed as BrowserWebContents but is a full WebContents at runtime. const contents = entry.view.webContents as unknown as WebContents; return contents.isDestroyed() ? null : contents; @@ -647,10 +797,45 @@ function emptyNavState(viewId: string): BrowserNavState { }; } -function hardenWebContents(contents: BrowserWebContents, options: BrowserViewHostOptions, entry: BrowserEntry): void { +function activeEntry(session: BrowserSessionEntry): BrowserEntry { + const entry = session.tabs.get(session.activeTabId); + if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Active browser tab is unavailable"); + return entry; +} + +function tabResult(entry: BrowserEntry, active: boolean): { + id: string; + url: string; + title: string; + active: boolean; +} { + return { + id: entry.tabId, + url: entry.view.webContents.getURL(), + title: entry.view.webContents.getTitle(), + active, + }; +} + +function listTabs(session: BrowserSessionEntry): { + activeTabId: string; + tabs: Array<{ id: string; url: string; title: string; active: boolean }>; +} { + return { + activeTabId: session.activeTabId, + tabs: [...session.tabs.values()].map((entry) => tabResult(entry, entry.tabId === session.activeTabId)), + }; +} + +function hardenWebContents( + contents: BrowserWebContents, + options: BrowserViewHostOptions, + entry: BrowserEntry, + onPopup: (url: string) => void, +): void { contents.setWindowOpenHandler(({ url }) => { if (isAllowedBrowserURL(url, options.rendererOrigin)) { - void options.shell.openExternal(url); + onPopup(url); } return { action: "deny" }; }); @@ -665,12 +850,18 @@ function hardenWebContents(contents: BrowserWebContents, options: BrowserViewHos contents.on("will-redirect", blockUnsafeNavigation); } -function wireNavEvents(contents: BrowserWebContents, options: BrowserViewHostOptions, entry: BrowserEntry): void { +function wireNavEvents( + contents: BrowserWebContents, + options: BrowserViewHostOptions, + entry: BrowserEntry, + isActive: () => boolean, + syncActiveBounds: () => void, +): void { const update = () => { - pushNavState(options, entry); + if (isActive()) pushNavState(options, entry); }; contents.on("did-navigate", () => { - entry.view.setVisible?.(true); + if (isActive()) syncActiveBounds(); update(); }); contents.on("did-navigate-in-page", update); @@ -688,9 +879,9 @@ function wireNavEvents(contents: BrowserWebContents, options: BrowserViewHostOpt message: String(errorDescription || `Navigation failed (${errorCode})`), timestamp: new Date().toISOString(), }); - entry.view.setVisible?.(false); + if (isActive()) entry.view.setVisible?.(false); entry.state = { ...readNavState(entry), error: String(errorDescription || "Unable to load page") }; - options.mainWindow.webContents.send("browser:navState", entry.state); + if (isActive()) options.mainWindow.webContents.send("browser:navState", entry.state); }); } @@ -1041,6 +1232,31 @@ async function hoverEntry(entry: BrowserEntry, refName: string): Promise { + const objectId = await resolveRef(entry, refName); + await entry.view.webContents.debugger.sendCommand("Overlay.enable"); + await entry.view.webContents.debugger.sendCommand("Overlay.highlightNode", { + objectId, + highlightConfig: { + showInfo: false, + showStyles: false, + showRulers: false, + contentColor: { r: 59, g: 130, b: 246, a: 0.18 }, + borderColor: { r: 37, g: 99, b: 235, a: 1 }, + paddingColor: { r: 96, g: 165, b: 250, a: 0.12 }, + marginColor: { r: 147, g: 197, b: 253, a: 0.08 }, + }, + }); + return { ref: refName, url: entry.view.webContents.getURL() }; +} + +async function unhighlightEntry(entry: BrowserEntry): Promise { + await ensureDebugger(entry); + await entry.view.webContents.debugger.sendCommand("Overlay.enable"); + await entry.view.webContents.debugger.sendCommand("Overlay.hideHighlight"); + return { url: entry.view.webContents.getURL() }; +} + function quadCenter(quad: number[] | undefined): { x: number; y: number } | undefined { if (!quad || quad.length < 8) return undefined; const xs = [quad[0], quad[2], quad[4], quad[6]]; From 3fed1d7ac04a79d154283d20598e0128205a0b09 Mon Sep 17 00:00:00 2001 From: aprv10 <1apoorvindia@gmail.com> Date: Fri, 24 Jul 2026 01:38:44 +0530 Subject: [PATCH 05/11] feat(browser): add reliable wait conditions --- backend/internal/cli/browser.go | 37 +++++++++-- backend/internal/cli/browser_test.go | 34 ++++++++++ .../skillassets/using-ao/commands/browser.md | 6 +- docs/STATUS.md | 4 +- docs/cli/README.md | 3 + frontend/src/main/browser-view-host.test.ts | 44 +++++++++++++ frontend/src/main/browser-view-host.ts | 66 +++++++++++++++++-- 7 files changed, 180 insertions(+), 14 deletions(-) diff --git a/backend/internal/cli/browser.go b/backend/internal/cli/browser.go index 98fad34b9e..b00b678861 100644 --- a/backend/internal/cli/browser.go +++ b/backend/internal/cli/browser.go @@ -258,30 +258,53 @@ func newBrowserCommand(ctx *commandContext) *cobra.Command { }, }) - var waitText, waitSelector, waitURL string - var waitMS, timeoutMS int + var waitText, waitTextGone, waitSelector, waitSelectorGone, waitURL string + var waitMS, waitStableMS, timeoutMS int + var waitLoad bool waitCmd := &cobra.Command{ Use: "wait", - Short: "Wait for time, text, selector, or URL state", + Short: "Wait for page load, DOM stability, time, text, selector, or URL state", Args: noArgs, RunE: func(cmd *cobra.Command, _ []string) error { selected := 0 - for _, active := range []bool{waitText != "", waitSelector != "", waitURL != "", waitMS > 0} { + for _, active := range []bool{ + waitText != "", + waitTextGone != "", + waitSelector != "", + waitSelectorGone != "", + waitURL != "", + waitLoad, + waitStableMS > 0, + waitMS > 0, + } { if active { selected++ } } if selected != 1 { - return usageError{errors.New("choose exactly one of --text, --selector, --url, or --ms")} + return usageError{errors.New( + "choose exactly one of --text, --text-gone, --selector, --selector-gone, --url, --load, --dom-stable, or --ms", + )} + } + if waitStableMS > timeoutMS { + return usageError{errors.New("--timeout must be at least as long as --dom-stable")} } args := map[string]any{"timeoutMs": timeoutMS} switch { case waitText != "": args["text"] = waitText + case waitTextGone != "": + args["textGone"] = waitTextGone case waitSelector != "": args["selector"] = waitSelector + case waitSelectorGone != "": + args["selectorGone"] = waitSelectorGone case waitURL != "": args["url"] = waitURL + case waitLoad: + args["load"] = true + case waitStableMS > 0: + args["stableMs"] = waitStableMS default: args["ms"] = waitMS } @@ -289,8 +312,12 @@ func newBrowserCommand(ctx *commandContext) *cobra.Command { }, } waitCmd.Flags().StringVar(&waitText, "text", "", "wait until visible page text contains this value") + waitCmd.Flags().StringVar(&waitTextGone, "text-gone", "", "wait until visible page text no longer contains this value") waitCmd.Flags().StringVar(&waitSelector, "selector", "", "wait until this CSS selector exists") + waitCmd.Flags().StringVar(&waitSelectorGone, "selector-gone", "", "wait until this CSS selector no longer exists") waitCmd.Flags().StringVar(&waitURL, "url", "", "wait until the current URL contains this value") + waitCmd.Flags().BoolVar(&waitLoad, "load", false, "wait until the current page finishes loading") + waitCmd.Flags().IntVar(&waitStableMS, "dom-stable", 0, "wait until the DOM has not mutated for this many milliseconds") waitCmd.Flags().IntVar(&waitMS, "ms", 0, "wait for a fixed number of milliseconds") waitCmd.Flags().IntVar(&timeoutMS, "timeout", 10_000, "condition timeout in milliseconds") cmd.AddCommand(waitCmd) diff --git a/backend/internal/cli/browser_test.go b/backend/internal/cli/browser_test.go index 9c27a1e7a7..d69a0338bd 100644 --- a/backend/internal/cli/browser_test.go +++ b/backend/internal/cli/browser_test.go @@ -93,6 +93,37 @@ func TestBrowserClickAndWaitArguments(t *testing.T) { } } +func TestBrowserExpandedWaitArguments(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + + tests := []struct { + name string + args []string + key string + want any + }{ + {name: "text disappears", args: []string{"--text-gone", "Saving..."}, key: "textGone", want: "Saving..."}, + {name: "selector disappears", args: []string{"--selector-gone", ".spinner"}, key: "selectorGone", want: ".spinner"}, + {name: "page load", args: []string{"--load"}, key: "load", want: true}, + {name: "DOM stability", args: []string{"--dom-stable", "750"}, key: "stableMs", want: float64(750)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, _, err := executeCLI(t, deps, append([]string{"browser", "wait"}, tt.args...)...); err != nil { + t.Fatal(err) + } + if capture.body.Action != "wait" || capture.body.Args[tt.key] != tt.want { + t.Fatalf("wait command = %#v", capture.body) + } + }) + } +} + func TestBrowserCoreInteractionArguments(t *testing.T) { t.Setenv("AO_SESSION_ID", "ao-1") cfg := setConfigEnv(t) @@ -186,6 +217,9 @@ func TestBrowserRequiresSessionAndValidWait(t *testing.T) { if _, _, err := executeCLI(t, Deps{}, "browser", "wait", "--text", "x", "--url", "y"); ExitCode(err) != 2 { t.Fatalf("wait error = %v code=%d", err, ExitCode(err)) } + if _, _, err := executeCLI(t, Deps{}, "browser", "wait", "--dom-stable", "5000", "--timeout", "1000"); ExitCode(err) != 2 { + t.Fatalf("dom-stable timeout error = %v code=%d", err, ExitCode(err)) + } if _, _, err := executeCLI(t, Deps{}, "browser", "get"); ExitCode(err) != 2 { t.Fatalf("get error = %v code=%d", err, ExitCode(err)) } diff --git a/backend/internal/skillassets/using-ao/commands/browser.md b/backend/internal/skillassets/using-ao/commands/browser.md index f8471da987..ad6bc5d92e 100644 --- a/backend/internal/skillassets/using-ao/commands/browser.md +++ b/backend/internal/skillassets/using-ao/commands/browser.md @@ -45,7 +45,7 @@ ao browser select [--json] ao browser check [--json] ao browser uncheck [--json] ao browser get [ref] [--json] -ao browser wait (--text | --selector | --url | --ms ) [--timeout ] [--json] +ao browser wait (--text | --text-gone | --selector | --selector-gone | --url | --load | --dom-stable | --ms ) [--timeout ] [--json] ao browser screenshot [path] [--json] ao browser console [--json] ao browser errors [--json] @@ -63,6 +63,10 @@ following browser commands, and `tab close` defaults to the active tab. Allowed page popups are captured as new AO tabs instead of opening a separate OS browser. Take a new snapshot after switching tabs because element refs are invalidated at the tab boundary. +Use `wait --load` after navigation, `--text-gone` or `--selector-gone` for +transient UI, and `--dom-stable ` after HMR or a dynamic render. Conditional +waits retry through brief execution-context replacement during navigation and +fail with `WAIT_TIMEOUT` when `--timeout` expires. Without `--json`, `screenshot` writes a PNG and refuses to overwrite an existing file. With `--json`, it returns the structured response including base64 image data. diff --git a/docs/STATUS.md b/docs/STATUS.md index 550ac931d9..27de49caef 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -67,8 +67,8 @@ surface (`npm run sqlc`, `npm run api`). compact accessibility snapshots and refs, click/fill/type, keyboard input, hover and non-mutating element highlighting, scrolling, selection and checked state, property reads, stable logical tabs and captured popups, waits, - screenshots, console messages, and page errors while the Browser panel is - hidden. + including load/disappearance/DOM-stability conditions, screenshots, console + messages, and page errors while the Browser panel is hidden. - Real daemon wiring via the generated `openapi-fetch` typed client (`src/api/schema.ts`); mock data only in `VITE_NO_ELECTRON` web-preview mode. - Electron main handles daemon discovery, launch, and status reporting. diff --git a/docs/cli/README.md b/docs/cli/README.md index cba4bcaf26..2da76b3aa0 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -93,6 +93,9 @@ OS-browser windows. The AO desktop app must be open because Electron owns the References from a snapshot are invalidated after navigation or DOM replacement; they are also invalidated when changing tabs. Take another snapshot when a command reports `STALE_REFERENCE`. +Browser waits cover load completion, text or selector appearance and +disappearance, URL matching, fixed delays, and a configurable DOM-stability +window for HMR-driven verification. `go run .` in `backend/` remains a compatibility wrapper around the daemon. diff --git a/frontend/src/main/browser-view-host.test.ts b/frontend/src/main/browser-view-host.test.ts index 1e17c4348d..b4f6d2852a 100644 --- a/frontend/src/main/browser-view-host.test.ts +++ b/frontend/src/main/browser-view-host.test.ts @@ -538,6 +538,50 @@ describe("agent browser runtime", () => { }); }); + it("waits for load completion, disappearance, and DOM stability", async () => { + const { debuggerSendCommand, host } = setupHost(); + const expressions: string[] = []; + debuggerSendCommand.mockImplementation(async (method: string, params?: Record) => { + if (method !== "Runtime.evaluate") return {}; + const expression = String(params?.expression ?? ""); + expressions.push(expression); + if (expression.includes("__ao_browser_dom_stability__")) { + return { result: { value: 500 } }; + } + return { result: { value: true } }; + }); + + await host.execute("sess-1", "wait", { load: true, timeoutMs: 500 }); + await host.execute("sess-1", "wait", { textGone: "Saving...", timeoutMs: 500 }); + await host.execute("sess-1", "wait", { selectorGone: ".spinner", timeoutMs: 500 }); + await host.execute("sess-1", "wait", { stableMs: 250, timeoutMs: 500 }); + + expect(expressions).toEqual( + expect.arrayContaining([ + "document.readyState === 'complete'", + expect.stringContaining("!document.body.innerText.includes"), + expect.stringContaining("!document.querySelector"), + expect.stringContaining("__ao_browser_dom_stability__"), + ]), + ); + }); + + it("retries a wait when navigation briefly replaces the execution context", async () => { + const { debuggerSendCommand, host } = setupHost(); + let attempts = 0; + debuggerSendCommand.mockImplementation(async (method: string) => { + if (method !== "Runtime.evaluate") return {}; + attempts++; + if (attempts === 1) throw new Error("Execution context was destroyed"); + return { result: { value: true } }; + }); + + await expect(host.execute("sess-1", "wait", { text: "Ready", timeoutMs: 500 })).resolves.toMatchObject({ + condition: 'text "Ready"', + }); + expect(attempts).toBe(2); + }); + it("invalidates refs after navigation", async () => { const { debuggerSendCommand, host, webContentsListeners } = setupHost(); debuggerSendCommand.mockImplementation(async (method: string) => { diff --git a/frontend/src/main/browser-view-host.ts b/frontend/src/main/browser-view-host.ts index ecbb26171b..b7dbca53c7 100644 --- a/frontend/src/main/browser-view-host.ts +++ b/frontend/src/main/browser-view-host.ts @@ -1402,28 +1402,82 @@ async function waitForEntry(entry: BrowserEntry, args: Record): return { waitedMs: fixedMS, url: entry.view.webContents.getURL() }; } const timeoutMS = numberArg(args.timeoutMs, 1, 60_000) || 10_000; + const stableMS = numberArg(args.stableMs, 1, 10_000); let expression = ""; let condition = ""; + let valueSatisfies = (value: unknown): boolean => value === true; if (typeof args.text === "string" && args.text) { expression = `Boolean(document.body && document.body.innerText.includes(${JSON.stringify(args.text)}))`; condition = `text ${JSON.stringify(args.text)}`; + } else if (typeof args.textGone === "string" && args.textGone) { + expression = `Boolean(!document.body || !document.body.innerText.includes(${JSON.stringify(args.textGone)}))`; + condition = `text ${JSON.stringify(args.textGone)} to disappear`; } else if (typeof args.selector === "string" && args.selector) { expression = `Boolean(document.querySelector(${JSON.stringify(args.selector)}))`; condition = `selector ${JSON.stringify(args.selector)}`; + } else if (typeof args.selectorGone === "string" && args.selectorGone) { + expression = `Boolean(!document.querySelector(${JSON.stringify(args.selectorGone)}))`; + condition = `selector ${JSON.stringify(args.selectorGone)} to disappear`; } else if (typeof args.url === "string" && args.url) { expression = `location.href.includes(${JSON.stringify(args.url)})`; condition = `URL ${JSON.stringify(args.url)}`; + } else if (args.load === true) { + expression = "document.readyState === 'complete'"; + condition = "page load completion"; + } else if (stableMS > 0) { + expression = `(() => { + const key = "__ao_browser_dom_stability__"; + let state = globalThis[key]; + if (!state || state.document !== document) { + state = {document, lastMutation: performance.now()}; + state.observer = new MutationObserver(() => { state.lastMutation = performance.now(); }); + state.observer.observe(document, { + subtree: true, + childList: true, + attributes: true, + characterData: true, + }); + globalThis[key] = state; + } + return performance.now() - state.lastMutation; + })()`; + condition = `DOM stability for ${stableMS}ms`; + valueSatisfies = (value) => typeof value === "number" && value >= stableMS; } else { - throw browserError("INVALID_ARGUMENT", "wait requires text, selector, url, or ms"); + throw browserError( + "INVALID_ARGUMENT", + "wait requires text, textGone, selector, selectorGone, url, load, stableMs, or ms", + ); } await ensureDebugger(entry); const deadline = Date.now() + timeoutMS; while (Date.now() <= deadline) { - const evaluated = (await entry.view.webContents.debugger.sendCommand("Runtime.evaluate", { - expression, - returnByValue: true, - })) as { result?: { value?: unknown } }; - if (evaluated.result?.value === true) { + if (args.load === true && entry.view.webContents.isLoading()) { + await delay(100); + continue; + } + let evaluated: { + result?: { value?: unknown }; + exceptionDetails?: { text?: string }; + }; + try { + evaluated = (await entry.view.webContents.debugger.sendCommand("Runtime.evaluate", { + expression, + returnByValue: true, + })) as typeof evaluated; + } catch { + // Navigations and HMR can briefly replace the execution context. Retry + // until the requested condition or timeout rather than failing early. + await delay(100); + continue; + } + if (evaluated.exceptionDetails) { + throw browserError( + "INVALID_ARGUMENT", + evaluated.exceptionDetails.text ?? `Unable to evaluate wait condition ${condition}`, + ); + } + if (valueSatisfies(evaluated.result?.value)) { return { condition, url: entry.view.webContents.getURL() }; } await delay(100); From 7c29cb242f315e1b6971eefede0bf992f8517f5e Mon Sep 17 00:00:00 2001 From: aprv10 <1apoorvindia@gmail.com> Date: Fri, 24 Jul 2026 01:56:16 +0530 Subject: [PATCH 06/11] feat(browser): isolate worker profiles --- .../skillassets/using-ao/commands/browser.md | 2 +- docs/STATUS.md | 4 +++- docs/cli/README.md | 3 +++ frontend/src/main/browser-view-host.test.ts | 23 +++++++++++++++++-- frontend/src/main/browser-view-host.ts | 7 ++++++ 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/backend/internal/skillassets/using-ao/commands/browser.md b/backend/internal/skillassets/using-ao/commands/browser.md index ad6bc5d92e..cf830284a2 100644 --- a/backend/internal/skillassets/using-ao/commands/browser.md +++ b/backend/internal/skillassets/using-ao/commands/browser.md @@ -1,6 +1,6 @@ # ao browser -Inspect and control the current AO session's target-isolated browser. The desktop app must be open. The agent and user share the same live page, cookies, navigation state, and `WebContentsView`; the runtime remains usable while the Browser panel is hidden. +Inspect and control the current AO session's target-isolated browser. The desktop app must be open. The agent and user share the same live page, cookies, navigation state, and `WebContentsView`; the runtime remains usable while the Browser panel is hidden. Tabs in this worker share an ephemeral browser profile, while other AO workers use isolated profiles. `AO_SESSION_ID` selects the target, so run these commands from inside an AO worker session. diff --git a/docs/STATUS.md b/docs/STATUS.md index 27de49caef..2b92678124 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -68,7 +68,9 @@ surface (`npm run sqlc`, `npm run api`). hover and non-mutating element highlighting, scrolling, selection and checked state, property reads, stable logical tabs and captured popups, waits, including load/disappearance/DOM-stability conditions, screenshots, console - messages, and page errors while the Browser panel is hidden. + messages, and page errors while the Browser panel is hidden. Tabs within one + worker share an ephemeral Electron profile; different workers have isolated + cookies and web storage. - Real daemon wiring via the generated `openapi-fetch` typed client (`src/api/schema.ts`); mock data only in `VITE_NO_ELECTRON` web-preview mode. - Electron main handles daemon discovery, launch, and status reporting. diff --git a/docs/cli/README.md b/docs/cli/README.md index 2da76b3aa0..d1854fcb66 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -96,6 +96,9 @@ command reports `STALE_REFERENCE`. Browser waits cover load completion, text or selector appearance and disappearance, URL matching, fixed delays, and a configurable DOM-stability window for HMR-driven verification. +Browser tabs in the same worker share a memory-only Electron profile. Different +workers receive distinct partitions, so cookies, authentication, local storage, +and session storage do not leak between their browser runtimes. `go run .` in `backend/` remains a compatibility wrapper around the daemon. diff --git a/frontend/src/main/browser-view-host.test.ts b/frontend/src/main/browser-view-host.test.ts index b4f6d2852a..f57c342398 100644 --- a/frontend/src/main/browser-view-host.test.ts +++ b/frontend/src/main/browser-view-host.test.ts @@ -154,6 +154,7 @@ function setupHost() { } function setupTabHost() { + const constructorOptions: Array<{ webPreferences: { partition?: string } }> = []; const views: Array<{ webContents: { id: number; @@ -247,13 +248,14 @@ function setupTabHost() { off: () => undefined, } as never, shell: { openExternal: async () => undefined }, - WebContentsView: function () { + WebContentsView: function (options: { webPreferences: { partition?: string } }) { + constructorOptions.push(options); return makeView(); } as never, annotatePreloadPath: "/preload.js", rendererOrigin: "http://localhost:5173", }); - return { host, views }; + return { constructorOptions, host, views }; } describe("new-session shortcut forwarding", () => { @@ -645,6 +647,23 @@ describe("agent browser runtime", () => { expect(replacement.id).toBe("t3"); }); + it("shares one ephemeral profile across a worker's tabs and isolates other workers", async () => { + const { constructorOptions, host } = setupTabHost(); + await host.execute("sess-1", "tabs"); + await host.execute("sess-1", "tab-new"); + await host.execute("sess-2", "tabs"); + + const firstPartition = constructorOptions[0].webPreferences.partition; + expect(firstPartition).toMatch(/^ao-browser-/); + expect(firstPartition).not.toMatch(/^persist:/); + expect(constructorOptions[1].webPreferences.partition).toBe(firstPartition); + expect(constructorOptions[2].webPreferences.partition).not.toBe(firstPartition); + + host.destroy("0:sess-1"); + await host.execute("sess-1", "tabs"); + expect(constructorOptions[3].webPreferences.partition).not.toBe(firstPartition); + }); + it("captures allowed popups as new tabs and protects the final tab", async () => { const { host, views } = setupTabHost(); await host.execute("sess-1", "open", { url: "http://localhost:3000" }); diff --git a/frontend/src/main/browser-view-host.ts b/frontend/src/main/browser-view-host.ts index b7dbca53c7..70acbf2c54 100644 --- a/frontend/src/main/browser-view-host.ts +++ b/frontend/src/main/browser-view-host.ts @@ -8,6 +8,7 @@ import type { WebContents, WebFrameMain, } from "electron"; +import { randomUUID } from "node:crypto"; import type { BrowserAnnotationCancelPayload, BrowserAnnotationModeInput, @@ -127,6 +128,7 @@ type BrowserEntry = { type BrowserSessionEntry = { sessionId: string; viewId: string; + profilePartition: string; tabs: Map; activeTabId: string; nextTabNumber: number; @@ -252,6 +254,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV webPreferences: { contextIsolation: true, nodeIntegration: false, + partition: session.profilePartition, preload: options.annotatePreloadPath, sandbox: true, }, @@ -311,6 +314,10 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV session = { sessionId, viewId, + // A non-persist: Electron partition is memory-only. Every tab in + // this worker shares it, while a fresh worker runtime receives a + // different partition even if a session ID is ever reused. + profilePartition: `ao-browser-${randomUUID()}`, tabs: new Map(), activeTabId: "", nextTabNumber: 1, From ffa2cb66db74eb8d681350f130cd07b43780e01c Mon Sep 17 00:00:00 2001 From: aprv10 <1apoorvindia@gmail.com> Date: Fri, 24 Jul 2026 14:52:01 +0530 Subject: [PATCH 07/11] feat(browser): complete session-owned preview runtime --- backend/internal/cli/browser.go | 116 +++ backend/internal/cli/browser_test.go | 40 + backend/internal/cli/preview.go | 151 +++- backend/internal/cli/preview_test.go | 100 ++- backend/internal/daemon/daemon.go | 18 +- backend/internal/daemon/lifecycle_wiring.go | 3 +- backend/internal/daemon/wiring_test.go | 6 +- backend/internal/httpd/api.go | 6 +- backend/internal/httpd/apispec/openapi.yaml | 196 +++++ .../internal/httpd/apispec/specgen/build.go | 41 + backend/internal/httpd/controllers/browser.go | 49 +- .../httpd/controllers/browser_test.go | 1 + backend/internal/httpd/controllers/dto.go | 21 + .../internal/httpd/controllers/sessions.go | 142 +++- .../httpd/controllers/sessions_test.go | 207 ++++- backend/internal/ports/outbound.go | 7 + backend/internal/preview/poller.go | 36 +- backend/internal/preview/poller_test.go | 32 +- backend/internal/previewserver/manager.go | 754 ++++++++++++++++++ .../internal/previewserver/manager_test.go | 171 ++++ .../internal/previewserver/process_unix.go | 26 + .../internal/previewserver/process_windows.go | 72 ++ .../previewserver/process_windows_test.go | 26 + backend/internal/session_manager/manager.go | 22 +- .../internal/session_manager/manager_test.go | 29 + .../internal/skillassets/skillassets_test.go | 54 ++ .../internal/skillassets/using-ao/SKILL.md | 12 +- .../skillassets/using-ao/commands/browser.md | 24 +- .../skillassets/using-ao/commands/preview.md | 93 ++- .../skillassets/using-ao/references.md | 6 + docs/STATUS.md | 13 +- docs/architecture.md | 8 + docs/cli/README.md | 31 +- frontend/src/api/schema.ts | 236 ++++++ frontend/src/main/browser-view-host.test.ts | 235 +++++- frontend/src/main/browser-view-host.ts | 491 +++++++++++- frontend/src/preload.ts | 26 +- .../renderer/components/BrowserPanel.test.tsx | 87 +- .../src/renderer/components/BrowserPanel.tsx | 120 ++- .../renderer/components/SessionView.test.tsx | 8 + .../renderer/hooks/useBrowserView.test.tsx | 112 +++ frontend/src/renderer/hooks/useBrowserView.ts | 101 ++- frontend/src/renderer/lib/api-client.test.ts | 3 + frontend/src/renderer/lib/api-client.ts | 1 + frontend/src/renderer/lib/bridge.ts | 5 + frontend/src/renderer/test/setup.ts | 5 + 46 files changed, 3808 insertions(+), 135 deletions(-) create mode 100644 backend/internal/previewserver/manager.go create mode 100644 backend/internal/previewserver/manager_test.go create mode 100644 backend/internal/previewserver/process_unix.go create mode 100644 backend/internal/previewserver/process_windows.go create mode 100644 backend/internal/previewserver/process_windows_test.go diff --git a/backend/internal/cli/browser.go b/backend/internal/cli/browser.go index b00b678861..edd85434f1 100644 --- a/backend/internal/cli/browser.go +++ b/backend/internal/cli/browser.go @@ -344,6 +344,51 @@ func newBrowserCommand(ctx *commandContext) *cobra.Command { }, }) + var networkDuration int + networkCmd := &cobra.Command{ + Use: "network", + Short: "Temporarily capture sanitized network request metadata", + Args: noArgs, + } + networkStart := &cobra.Command{ + Use: "start", + Short: "Start bounded metadata-only capture on the active tab", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if networkDuration < 1 || networkDuration > 300 { + return usageError{errors.New("--duration must be between 1 and 300 seconds")} + } + return ctx.runBrowserAction( + cmd, + "network-start", + map[string]any{"durationSeconds": networkDuration}, + jsonOutput, + ) + }, + } + networkStart.Flags().IntVar(&networkDuration, "duration", 60, "capture duration in seconds (maximum 300)") + networkCmd.AddCommand(networkStart) + for _, subcommand := range []struct { + name string + short string + }{ + {name: "status", short: "Show capture state without enabling it"}, + {name: "list", short: "List captured sanitized request metadata"}, + {name: "stop", short: "Stop capture and list the retained requests"}, + {name: "clear", short: "Clear retained requests without changing capture state"}, + } { + subcommand := subcommand + networkCmd.AddCommand(&cobra.Command{ + Use: subcommand.name, + Short: subcommand.short, + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return ctx.runBrowserAction(cmd, "network-"+subcommand.name, nil, jsonOutput) + }, + }) + } + cmd.AddCommand(networkCmd) + for _, action := range []string{"console", "errors"} { action := action cmd.AddCommand(&cobra.Command{ @@ -466,6 +511,9 @@ func writeBrowserResult(cmd *cobra.Command, action string, result map[string]any } return nil } + if strings.HasPrefix(action, "network-") { + return writeBrowserNetworkResult(cmd, action, result) + } if currentURL, ok := result["url"].(string); ok && currentURL != "" { _, err := fmt.Fprintln(cmd.OutOrStdout(), currentURL) return err @@ -474,6 +522,74 @@ func writeBrowserResult(cmd *cobra.Command, action string, result map[string]any return err } +func writeBrowserNetworkResult(cmd *cobra.Command, action string, result map[string]any) error { + if action == "network-clear" { + _, err := fmt.Fprintln(cmd.OutOrStdout(), "Browser network capture cleared.") + return err + } + active, _ := result["active"].(bool) + state := "inactive" + if active { + state = "active" + } + tabID, _ := result["tabId"].(string) + count := numberString(result["requestCount"]) + maxEntries := numberString(result["maxEntries"]) + if count == "" { + count = "0" + } + if maxEntries == "" { + maxEntries = "200" + } + if action == "network-start" || action == "network-status" { + _, err := fmt.Fprintf( + cmd.OutOrStdout(), + "Browser network capture: %s (tab %s, %s/%s requests, metadata only)\n", + state, + tabID, + count, + maxEntries, + ) + return err + } + + requests, _ := result["requests"].([]any) + if len(requests) == 0 { + _, err := fmt.Fprintln(cmd.OutOrStdout(), "No browser network requests captured.") + return err + } + for _, raw := range requests { + request, _ := raw.(map[string]any) + method, _ := request["method"].(string) + currentURL, _ := request["url"].(string) + resourceType, _ := request["resourceType"].(string) + status := numberString(request["status"]) + if failed, _ := request["failed"].(bool); failed { + status = "FAILED" + } else if status == "" { + status = "PENDING" + } + duration := numberString(request["durationMs"]) + if duration != "" { + duration += "ms" + } else { + duration = "-" + } + if _, err := fmt.Fprintf( + cmd.OutOrStdout(), + "%s %s %s %s %s\n", + method, + status, + resourceType, + duration, + currentURL, + ); err != nil { + return err + } + } + return nil +} + func writeBrowserScreenshot(cmd *cobra.Command, result map[string]any, target string) error { encoded, _ := result["data"].(string) if encoded == "" { diff --git a/backend/internal/cli/browser_test.go b/backend/internal/cli/browser_test.go index d69a0338bd..7c7a3f29c0 100644 --- a/backend/internal/cli/browser_test.go +++ b/backend/internal/cli/browser_test.go @@ -40,6 +40,12 @@ func browserCLIServer(t *testing.T, capture *browserRequestCapture) *httptest.Se result = `{"data":"cG5n","width":10,"height":20}` case "tabs": result = `{"activeTabId":"t2","tabs":[{"id":"t1","title":"First","url":"http://localhost:3000/","active":false},{"id":"t2","title":"Second","url":"http://localhost:4173/","active":true}]}` + case "network-start", "network-status": + result = `{"active":true,"metadataOnly":true,"tabId":"t1","requestCount":1,"maxEntries":200}` + case "network-list", "network-stop": + result = `{"active":false,"metadataOnly":true,"tabId":"t1","requestCount":1,"maxEntries":200,"requests":[{"method":"GET","url":"https://api.example.test/items?token=%5Bredacted%5D","resourceType":"xhr","status":200,"durationMs":42}]}` + case "network-clear": + result = `{"active":true,"metadataOnly":true,"tabId":"t1","requestCount":0,"maxEntries":200}` } _, _ = io.WriteString(w, `{"requestId":"r1","sessionId":"ao-1","action":"`+capture.body.Action+`","result":`+result+`}`) })) @@ -186,6 +192,40 @@ func TestBrowserTabsPrintStableIDsAndActiveTab(t *testing.T) { } } +func TestBrowserNetworkCommandsAreExplicitAndReadable(t *testing.T) { + t.Setenv("AO_SESSION_ID", "ao-1") + cfg := setConfigEnv(t) + capture := &browserRequestCapture{} + srv := browserCLIServer(t, capture) + writeRunFileFor(t, cfg, srv) + deps := Deps{ProcessAlive: func(int) bool { return true }} + + out, errOut, err := executeCLI(t, deps, "browser", "network", "start", "--duration", "45") + if err != nil { + t.Fatalf("network start err=%v stderr=%s", err, errOut) + } + if capture.body.Action != "network-start" || capture.body.Args["durationSeconds"] != float64(45) { + t.Fatalf("network start = %#v", capture.body) + } + if !strings.Contains(out, "active") || !strings.Contains(out, "metadata only") { + t.Fatalf("network start output = %q", out) + } + + out, errOut, err = executeCLI(t, deps, "browser", "network", "list") + if err != nil { + t.Fatalf("network list err=%v stderr=%s", err, errOut) + } + if capture.body.Action != "network-list" || + !strings.Contains(out, "GET 200 xhr 42ms") || + !strings.Contains(out, "token=%5Bredacted%5D") { + t.Fatalf("network list command=%#v output=%q", capture.body, out) + } + + if _, _, err := executeCLI(t, deps, "browser", "network", "start", "--duration", "301"); ExitCode(err) != 2 { + t.Fatalf("network duration error = %v code=%d", err, ExitCode(err)) + } +} + func TestBrowserScreenshotWritesWithoutOverwrite(t *testing.T) { t.Setenv("AO_SESSION_ID", "ao-1") cfg := setConfigEnv(t) diff --git a/backend/internal/cli/preview.go b/backend/internal/cli/preview.go index 02fadd1036..340943e30b 100644 --- a/backend/internal/cli/preview.go +++ b/backend/internal/cli/preview.go @@ -3,21 +3,40 @@ package cli import ( "context" "errors" + "fmt" + "io" "net/url" "os" "strings" + "time" "github.com/spf13/cobra" ) // previewAPIRequest mirrors the daemon's body for // POST /api/v1/sessions/{id}/preview. An empty Url asks the daemon to -// autodetect an index.html in the workspace. The CLI keeps its own copy so it +// autodetect a static entry in the workspace. The CLI keeps its own copy so it // need not import httpd. type previewAPIRequest struct { Url string `json:"url"` } +type previewServerStartRequest struct { + Configuration string `json:"configuration,omitempty"` +} + +type previewServerStatusDTO struct { + SessionID string `json:"sessionId"` + State string `json:"state"` + Configuration string `json:"configuration,omitempty"` + TargetKind string `json:"targetKind,omitempty"` + URL string `json:"url,omitempty"` + Port int `json:"port,omitempty"` + StartedAt time.Time `json:"startedAt,omitempty"` + Error string `json:"error,omitempty"` + Logs []string `json:"logs"` +} + func newPreviewCommand(ctx *commandContext) *cobra.Command { cmd := &cobra.Command{ Use: "preview [url]", @@ -25,11 +44,16 @@ func newPreviewCommand(ctx *commandContext) *cobra.Command { Long: "Open a URL in the desktop browser panel for the current session.\n\n" + "With no argument it opens the workspace's static entry point, falling\n" + "back to this session's existing preview target when no entry point exists.\n" + - "A local file can be opened by its absolute file:// URL\n" + - "(e.g. file:///home/me/proj/index.html). Use `ao preview clear` to empty the panel.", + "A workspace-relative Markdown or HTML path opens through AO's isolated\n" + + "file preview. Use `ao preview start` for a configured dev server and\n" + + "`ao preview clear` to empty the panel.", Example: ` ao preview - ao preview file://$(pwd)/index.html + ao preview README.md ao preview http://localhost:5173 + ao preview start + ao preview start web + ao preview status + ao preview stop ao preview clear`, Args: atMostOneArg, RunE: func(cmd *cobra.Command, args []string) error { @@ -48,6 +72,57 @@ func newPreviewCommand(ctx *commandContext) *cobra.Command { return ctx.clearPreview(cmd.Context()) }, }) + var startJSON bool + startCmd := &cobra.Command{ + Use: "start [configuration]", + Short: "Start a session-owned dev server from .ao/launch.json and open its preview", + Args: atMostOneArg, + RunE: func(cmd *cobra.Command, args []string) error { + configuration := "" + if len(args) == 1 { + configuration = args[0] + } + status, err := ctx.startPreviewServer(cmd.Context(), configuration) + if err != nil { + return err + } + return writePreviewServerStatus(cmd.OutOrStdout(), status, startJSON) + }, + } + startCmd.Flags().BoolVar(&startJSON, "json", false, "print JSON") + cmd.AddCommand(startCmd) + + var statusJSON bool + statusCmd := &cobra.Command{ + Use: "status", + Short: "Show this session's managed preview server status and recent logs", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + status, err := ctx.previewServerStatus(cmd.Context()) + if err != nil { + return err + } + return writePreviewServerStatus(cmd.OutOrStdout(), status, statusJSON) + }, + } + statusCmd.Flags().BoolVar(&statusJSON, "json", false, "print JSON") + cmd.AddCommand(statusCmd) + + var stopJSON bool + stopCmd := &cobra.Command{ + Use: "stop", + Short: "Stop this session's managed preview server", + Args: noArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + status, err := ctx.stopPreviewServer(cmd.Context()) + if err != nil { + return err + } + return writePreviewServerStatus(cmd.OutOrStdout(), status, stopJSON) + }, + } + stopCmd.Flags().BoolVar(&stopJSON, "json", false, "print JSON") + cmd.AddCommand(stopCmd) return cmd } @@ -78,3 +153,71 @@ func sessionPreviewPath() (string, error) { // well-formed regardless. return "sessions/" + url.PathEscape(sessionID) + "/preview", nil } + +func previewServerPath() (string, error) { + path, err := sessionPreviewPath() + if err != nil { + return "", err + } + return path + "/server", nil +} + +func (c *commandContext) startPreviewServer( + ctx context.Context, + configuration string, +) (previewServerStatusDTO, error) { + path, err := previewServerPath() + if err != nil { + return previewServerStatusDTO{}, err + } + var out previewServerStatusDTO + err = c.postJSON(ctx, path, previewServerStartRequest{Configuration: configuration}, &out) + return out, err +} + +func (c *commandContext) previewServerStatus(ctx context.Context) (previewServerStatusDTO, error) { + path, err := previewServerPath() + if err != nil { + return previewServerStatusDTO{}, err + } + var out previewServerStatusDTO + err = c.getJSON(ctx, path, &out) + return out, err +} + +func (c *commandContext) stopPreviewServer(ctx context.Context) (previewServerStatusDTO, error) { + path, err := previewServerPath() + if err != nil { + return previewServerStatusDTO{}, err + } + var out previewServerStatusDTO + err = c.deleteJSON(ctx, path, &out) + return out, err +} + +func writePreviewServerStatus(out io.Writer, status previewServerStatusDTO, jsonOutput bool) error { + if jsonOutput { + return writeJSON(out, status) + } + summary := status.State + if status.Configuration != "" { + summary += " " + status.Configuration + } + if status.URL != "" { + summary += " " + status.URL + } + if _, err := fmt.Fprintln(out, summary); err != nil { + return err + } + if status.Error != "" { + if _, err := fmt.Fprintln(out, "Error:", status.Error); err != nil { + return err + } + } + for _, line := range status.Logs { + if _, err := fmt.Fprintln(out, line); err != nil { + return err + } + } + return nil +} diff --git a/backend/internal/cli/preview_test.go b/backend/internal/cli/preview_test.go index 4f3ced2957..787469e408 100644 --- a/backend/internal/cli/preview_test.go +++ b/backend/internal/cli/preview_test.go @@ -48,6 +48,30 @@ func previewServer(t *testing.T, status int, respBody string) (*httptest.Server, return srv, capture } +func previewLifecycleServer(t *testing.T, status int, respBody string) (*httptest.Server, *previewCapture) { + t.Helper() + capture := &previewCapture{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/sessions/aa-47/preview/server" { + http.NotFound(w, r) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + capture.called = true + capture.body = string(body) + capture.path = r.URL.Path + capture.method = r.Method + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, respBody) + })) + t.Cleanup(srv.Close) + return srv, capture +} + func TestPreview_WithURLArg(t *testing.T) { t.Setenv("AO_SESSION_ID", "aa-47") cfg := setConfigEnv(t) @@ -111,6 +135,73 @@ func TestPreviewClear_DeletesSessionPreview(t *testing.T) { } } +func TestPreviewStartUsesNamedConfigurationAndPrintsReadyURL(t *testing.T) { + t.Setenv("AO_SESSION_ID", "aa-47") + cfg := setConfigEnv(t) + srv, capture := previewLifecycleServer( + t, + http.StatusOK, + `{"sessionId":"aa-47","state":"ready","configuration":"web","targetKind":"app","url":"http://127.0.0.1:4173/","port":4173,"logs":[]}`, + ) + writeRunFileFor(t, cfg, srv) + + out, errOut, err := executeCLI(t, Deps{ProcessAlive: func(int) bool { return true }}, "preview", "start", "web") + if err != nil { + t.Fatalf("unexpected error: %v\nstderr=%s", err, errOut) + } + if capture.method != http.MethodPost || capture.path != "/api/v1/sessions/aa-47/preview/server" { + t.Fatalf("request = %s %s", capture.method, capture.path) + } + if capture.body != `{"configuration":"web"}` { + t.Fatalf("body = %q", capture.body) + } + if !strings.Contains(out, "ready web http://127.0.0.1:4173/") { + t.Fatalf("output = %q", out) + } +} + +func TestPreviewStatusAndStopUseManagedServerRoute(t *testing.T) { + for _, test := range []struct { + name string + args []string + method string + }{ + {name: "status", args: []string{"preview", "status", "--json"}, method: http.MethodGet}, + {name: "stop", args: []string{"preview", "stop", "--json"}, method: http.MethodDelete}, + } { + t.Run(test.name, func(t *testing.T) { + t.Setenv("AO_SESSION_ID", "aa-47") + cfg := setConfigEnv(t) + srv, capture := previewLifecycleServer( + t, + http.StatusOK, + `{"sessionId":"aa-47","state":"stopped","logs":[]}`, + ) + writeRunFileFor(t, cfg, srv) + + out, errOut, err := executeCLI(t, Deps{ProcessAlive: func(int) bool { return true }}, test.args...) + if err != nil { + t.Fatalf("unexpected error: %v\nstderr=%s", err, errOut) + } + if capture.method != test.method { + t.Fatalf("method = %q, want %q", capture.method, test.method) + } + if !strings.Contains(out, `"state": "stopped"`) { + t.Fatalf("JSON output = %q", out) + } + }) + } +} + +func TestPreviewStartMissingSessionIDIsUsageError(t *testing.T) { + t.Setenv("AO_SESSION_ID", "") + setConfigEnv(t) + _, _, err := executeCLI(t, Deps{}, "preview", "start") + if got := ExitCode(err); got != 2 { + t.Fatalf("exit code = %d, want 2; err=%v", got, err) + } +} + func TestPreviewClear_MissingSessionIDIsUsageError(t *testing.T) { t.Setenv("AO_SESSION_ID", "") cfg := setConfigEnv(t) @@ -181,12 +272,11 @@ func TestPreview_HelpIncludesExamples(t *testing.T) { if !strings.Contains(out, "EXAMPLES") && !strings.Contains(out, "Examples") { t.Errorf("help output missing Examples section:\n%s", out) } - // file:// URL example (not a relative path). - if !strings.Contains(out, "file://$(pwd)/index.html") { - t.Errorf("help output missing file:// example:\n%s", out) + if !strings.Contains(out, "README.md") { + t.Errorf("help output missing Markdown example:\n%s", out) } - if strings.Contains(out, "./dist/index.html") { - t.Errorf("help output still references relative ./dist/index.html:\n%s", out) + if !strings.Contains(out, "ao preview start") { + t.Errorf("help output missing managed server example:\n%s", out) } } diff --git a/backend/internal/daemon/daemon.go b/backend/internal/daemon/daemon.go index c261058eb7..7b0a15576f 100644 --- a/backend/internal/daemon/daemon.go +++ b/backend/internal/daemon/daemon.go @@ -24,6 +24,7 @@ import ( "github.com/aoagents/agent-orchestrator/backend/internal/notify" "github.com/aoagents/agent-orchestrator/backend/internal/ports" "github.com/aoagents/agent-orchestrator/backend/internal/preview" + "github.com/aoagents/agent-orchestrator/backend/internal/previewserver" "github.com/aoagents/agent-orchestrator/backend/internal/push" "github.com/aoagents/agent-orchestrator/backend/internal/runfile" agentsvc "github.com/aoagents/agent-orchestrator/backend/internal/service/agent" @@ -104,6 +105,7 @@ func Run() error { // is handed to httpd, which mounts it at /mux. Raw PTY bytes never flow // through the CDC change_log -- only session-state events do. runtimeAdapter := runtimeselect.New(log) + managedPreview := previewserver.New(log) termMgr := terminal.NewManager(runtimeAdapter, cdcPipe.Broadcaster, log) defer termMgr.Close() @@ -141,7 +143,7 @@ func Run() error { // selected runtime, a gitworktree workspace, the per-session agent resolver // (AO_AGENT validated here for compatibility), and the agent messenger, then mount it // on the API. - sessionSvc, reviewSvc, sessMgr, err := startSession(cfg, runtimeAdapter, store, lcStack.LCM, messenger, telemetrySink, agents, log) + sessionSvc, reviewSvc, sessMgr, err := startSession(cfg, runtimeAdapter, store, lcStack.LCM, messenger, telemetrySink, agents, managedPreview, log) if err != nil { stop() lcStack.Stop() @@ -213,12 +215,13 @@ func Run() error { return sqlite.OpenReadOnly(ctx, dataDir) }, }), - CDC: store, - Events: cdcPipe.Broadcaster, - Activity: lcStack.LCM, - Telemetry: telemetrySink, - Mobile: mc, - Browser: browserBroker, + CDC: store, + Events: cdcPipe.Broadcaster, + Activity: lcStack.LCM, + Telemetry: telemetrySink, + Mobile: mc, + Browser: browserBroker, + PreviewServer: managedPreview, }) if err != nil { stop() @@ -297,6 +300,7 @@ func Run() error { // via defer) avoids the LIFO trap where a Stop() that blocks on ctx-cancel // runs before the cancel: a non-signal exit path would hang otherwise. stop() + managedPreview.Close() <-previewDone lcStack.Stop() lanStopCtx, lanCancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout) diff --git a/backend/internal/daemon/lifecycle_wiring.go b/backend/internal/daemon/lifecycle_wiring.go index bf33c396a7..d0261076c3 100644 --- a/backend/internal/daemon/lifecycle_wiring.go +++ b/backend/internal/daemon/lifecycle_wiring.go @@ -132,7 +132,7 @@ type sessionLifecycle interface { // store + LCM, the per-session agent resolver, and the agent messenger. The // returned service is mounted at httpd APIDeps.Sessions. It also returns the // manager so the caller can wire Reconcile into the boot sequence. -func startSession(cfg config.Config, runtime runtimeselect.Runtime, store *sqlite.Store, lcm *lifecycle.Manager, messenger ports.AgentMessenger, telemetry ports.EventSink, agents ports.AgentResolver, log *slog.Logger) (*sessionsvc.Service, reviewsvc.Manager, sessionLifecycle, error) { +func startSession(cfg config.Config, runtime runtimeselect.Runtime, store *sqlite.Store, lcm *lifecycle.Manager, messenger ports.AgentMessenger, telemetry ports.EventSink, agents ports.AgentResolver, previewLifecycle ports.SessionPreviewLifecycle, log *slog.Logger) (*sessionsvc.Service, reviewsvc.Manager, sessionLifecycle, error) { ws, err := gitworktree.New(gitworktree.Options{ // Per-session worktrees live under the data dir, so a single AO_DATA_DIR // override moves all durable per-user state together. @@ -152,6 +152,7 @@ func startSession(cfg config.Config, runtime runtimeselect.Runtime, store *sqlit Store: store, Messenger: messenger, Lifecycle: lcm, + Preview: previewLifecycle, DataDir: cfg.DataDir, Logger: log, }) diff --git a/backend/internal/daemon/wiring_test.go b/backend/internal/daemon/wiring_test.go index 2a62c90163..d59dd36579 100644 --- a/backend/internal/daemon/wiring_test.go +++ b/backend/internal/daemon/wiring_test.go @@ -183,7 +183,7 @@ func TestWiring_StartSessionBuildsSessionService(t *testing.T) { if err != nil { t.Fatalf("buildAgentResolver: %v", err) } - svc, reviewSvc, lc, err := startSession(cfg, rt, store, lcm, messenger, telemetryadapter.NoopSink{}, agents, log) + svc, reviewSvc, lc, err := startSession(cfg, rt, store, lcm, messenger, telemetryadapter.NoopSink{}, agents, nil, log) if err != nil { t.Fatalf("startSession: %v", err) } @@ -228,7 +228,7 @@ func TestStartSession_SpawnDoesNotPanicWhenNoTrackerToken(t *testing.T) { if agentsErr != nil { t.Fatalf("buildAgentResolver: %v", agentsErr) } - svc, _, _, err := startSession(cfg, rt, store, lcm, messenger, telemetryadapter.NoopSink{}, agents, log) + svc, _, _, err := startSession(cfg, rt, store, lcm, messenger, telemetryadapter.NoopSink{}, agents, nil, log) if err != nil { t.Fatalf("startSession: %v", err) } @@ -261,7 +261,7 @@ func TestStartTrackerIntake_RunsEvenWithoutEnabledProjects(t *testing.T) { if agentsErr != nil { t.Fatalf("buildAgentResolver: %v", agentsErr) } - svc, _, _, err := startSession(cfg, rt, store, lcm, messenger, telemetryadapter.NoopSink{}, agents, log) + svc, _, _, err := startSession(cfg, rt, store, lcm, messenger, telemetryadapter.NoopSink{}, agents, nil, log) if err != nil { t.Fatalf("startSession: %v", err) } diff --git a/backend/internal/httpd/api.go b/backend/internal/httpd/api.go index 69a46e7558..f406aa31aa 100644 --- a/backend/internal/httpd/api.go +++ b/backend/internal/httpd/api.go @@ -35,6 +35,7 @@ type APIDeps struct { Telemetry ports.EventSink Mobile *controllers.MobileController Browser controllers.BrowserRuntime + PreviewServer controllers.ManagedPreviewServer } // API owns one controller per resource and is the single Register call the @@ -67,8 +68,9 @@ func NewAPI(cfg config.Config, deps APIDeps) *API { Mgr: deps.Projects, }, sessions: &controllers.SessionsController{ - Svc: deps.Sessions, - Activity: deps.Activity, + Svc: deps.Sessions, + Activity: deps.Activity, + PreviewServer: deps.PreviewServer, }, prs: &controllers.PRsController{Svc: deps.PRs}, reviews: &controllers.ReviewsController{Svc: deps.Reviews}, diff --git a/backend/internal/httpd/apispec/openapi.yaml b/backend/internal/httpd/apispec/openapi.yaml index 0e19a7d188..f400ac7f75 100644 --- a/backend/internal/httpd/apispec/openapi.yaml +++ b/backend/internal/httpd/apispec/openapi.yaml @@ -1573,6 +1573,158 @@ paths: summary: Serve a static browser preview file from a session workspace tags: - sessions + /api/v1/sessions/{sessionId}/preview/server: + delete: + operationId: stopSessionPreviewServer + parameters: + - description: Session identifier, e.g. project-1. + in: path + name: sessionId + required: true + schema: + description: Session identifier, e.g. project-1. + type: string + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/PreviewServerStatusResponse' + description: OK + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Found + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Internal Server Error + "501": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Implemented + summary: Stop the managed preview server for a session + tags: + - sessions + get: + operationId: getSessionPreviewServer + parameters: + - description: Session identifier, e.g. project-1. + in: path + name: sessionId + required: true + schema: + description: Session identifier, e.g. project-1. + type: string + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/PreviewServerStatusResponse' + description: OK + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Found + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Internal Server Error + "501": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Implemented + summary: Get the managed preview server status for a session + tags: + - sessions + post: + operationId: startSessionPreviewServer + parameters: + - description: Session identifier, e.g. project-1. + in: path + name: sessionId + required: true + schema: + description: Session identifier, e.g. project-1. + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/StartPreviewServerRequest' + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/PreviewServerStatusResponse' + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Bad Request + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Found + "408": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Request Timeout + "409": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Conflict + "422": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Unprocessable Entity + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Internal Server Error + "501": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Not Implemented + "504": + content: + application/json: + schema: + $ref: '#/components/schemas/APIError' + description: Gateway Timeout + summary: Start a session-owned server from .ao/launch.json and open its application + preview + tags: + - sessions /api/v1/sessions/{sessionId}/restore: post: operationId: restoreSession @@ -2645,6 +2797,43 @@ components: - targetSha - status type: object + PreviewServerStatusResponse: + properties: + configuration: + type: string + error: + type: string + logs: + items: + type: string + type: array + port: + type: integer + sessionId: + type: string + startedAt: + format: date-time + type: string + state: + enum: + - stopped + - starting + - ready + - stopping + - failed + type: string + targetKind: + enum: + - app + - api + type: string + url: + type: string + required: + - sessionId + - state + - logs + type: object ProbeAgentResponse: properties: agent: @@ -3356,6 +3545,13 @@ components: required: - projectId type: object + StartPreviewServerRequest: + properties: + configuration: + description: Named preview configuration. Optional when exactly one configuration + exists. + type: string + type: object SubmitReviewInput: properties: body: diff --git a/backend/internal/httpd/apispec/specgen/build.go b/backend/internal/httpd/apispec/specgen/build.go index f10805384f..416aa01164 100644 --- a/backend/internal/httpd/apispec/specgen/build.go +++ b/backend/internal/httpd/apispec/specgen/build.go @@ -155,6 +155,8 @@ var schemaNames = map[string]string{ "ControllersSessionResponse": "SessionResponse", "ControllersSessionPreviewResponse": "SessionPreviewResponse", "ControllersSetSessionPreviewRequest": "SetSessionPreviewRequest", + "ControllersStartPreviewServerRequest": "StartPreviewServerRequest", + "ControllersPreviewServerStatusResponse": "PreviewServerStatusResponse", "ControllersBrowserStatusQuery": "BrowserStatusQuery", "ControllersBrowserStatusResponse": "BrowserStatusResponse", "ControllersBrowserCommandRequest": "BrowserCommandRequest", @@ -774,6 +776,45 @@ func sessionOperations() []operation { {http.StatusNotImplemented, envelope.APIError{}}, }, }, + { + method: http.MethodGet, path: "/api/v1/sessions/{sessionId}/preview/server", id: "getSessionPreviewServer", tag: "sessions", + summary: "Get the managed preview server status for a session", + pathParams: []any{controllers.SessionIDParam{}}, + resps: []respUnit{ + {http.StatusOK, controllers.PreviewServerStatusResponse{}}, + {http.StatusNotFound, envelope.APIError{}}, + {http.StatusInternalServerError, envelope.APIError{}}, + {http.StatusNotImplemented, envelope.APIError{}}, + }, + }, + { + method: http.MethodPost, path: "/api/v1/sessions/{sessionId}/preview/server", id: "startSessionPreviewServer", tag: "sessions", + summary: "Start a session-owned server from .ao/launch.json and open its application preview", + pathParams: []any{controllers.SessionIDParam{}}, + reqBody: controllers.StartPreviewServerRequest{}, + resps: []respUnit{ + {http.StatusOK, controllers.PreviewServerStatusResponse{}}, + {http.StatusBadRequest, envelope.APIError{}}, + {http.StatusNotFound, envelope.APIError{}}, + {http.StatusConflict, envelope.APIError{}}, + {http.StatusRequestTimeout, envelope.APIError{}}, + {http.StatusUnprocessableEntity, envelope.APIError{}}, + {http.StatusGatewayTimeout, envelope.APIError{}}, + {http.StatusInternalServerError, envelope.APIError{}}, + {http.StatusNotImplemented, envelope.APIError{}}, + }, + }, + { + method: http.MethodDelete, path: "/api/v1/sessions/{sessionId}/preview/server", id: "stopSessionPreviewServer", tag: "sessions", + summary: "Stop the managed preview server for a session", + pathParams: []any{controllers.SessionIDParam{}}, + resps: []respUnit{ + {http.StatusOK, controllers.PreviewServerStatusResponse{}}, + {http.StatusNotFound, envelope.APIError{}}, + {http.StatusInternalServerError, envelope.APIError{}}, + {http.StatusNotImplemented, envelope.APIError{}}, + }, + }, { method: http.MethodGet, path: "/api/v1/sessions/{sessionId}/preview/files/*", id: "getSessionPreviewFile", tag: "sessions", summary: "Serve a static browser preview file from a session workspace", diff --git a/backend/internal/httpd/controllers/browser.go b/backend/internal/httpd/controllers/browser.go index 3bb8c5eb50..015db71870 100644 --- a/backend/internal/httpd/controllers/browser.go +++ b/backend/internal/httpd/controllers/browser.go @@ -15,28 +15,33 @@ import ( ) var browserActions = map[string]struct{}{ - "open": {}, - "snapshot": {}, - "click": {}, - "fill": {}, - "type": {}, - "press": {}, - "hover": {}, - "highlight": {}, - "unhighlight": {}, - "tabs": {}, - "tab-new": {}, - "tab-select": {}, - "tab-close": {}, - "scroll": {}, - "select": {}, - "check": {}, - "uncheck": {}, - "get": {}, - "wait": {}, - "screenshot": {}, - "console": {}, - "errors": {}, + "open": {}, + "snapshot": {}, + "click": {}, + "fill": {}, + "type": {}, + "press": {}, + "hover": {}, + "highlight": {}, + "unhighlight": {}, + "tabs": {}, + "tab-new": {}, + "tab-select": {}, + "tab-close": {}, + "scroll": {}, + "select": {}, + "check": {}, + "uncheck": {}, + "get": {}, + "wait": {}, + "screenshot": {}, + "network-start": {}, + "network-status": {}, + "network-list": {}, + "network-stop": {}, + "network-clear": {}, + "console": {}, + "errors": {}, } type BrowserRuntime interface { diff --git a/backend/internal/httpd/controllers/browser_test.go b/backend/internal/httpd/controllers/browser_test.go index c90afc2936..72ef272da2 100644 --- a/backend/internal/httpd/controllers/browser_test.go +++ b/backend/internal/httpd/controllers/browser_test.go @@ -74,6 +74,7 @@ func TestBrowserCoreInteractionActionsReachRuntime(t *testing.T) { "type", "press", "hover", "highlight", "unhighlight", "tabs", "tab-new", "tab-select", "tab-close", "scroll", "select", "check", "uncheck", "get", + "network-start", "network-status", "network-list", "network-stop", "network-clear", } for _, action := range actions { diff --git a/backend/internal/httpd/controllers/dto.go b/backend/internal/httpd/controllers/dto.go index 9b33b87677..8334d05e19 100644 --- a/backend/internal/httpd/controllers/dto.go +++ b/backend/internal/httpd/controllers/dto.go @@ -221,6 +221,27 @@ type SetSessionPreviewRequest struct { URL string `json:"url,omitempty" description:"Preview target URL. When empty, the daemon autodetects a static entry point in the session workspace."` } +// StartPreviewServerRequest selects one named entry from .ao/launch.json. The +// name may be omitted when the file contains exactly one configuration. +type StartPreviewServerRequest struct { + Configuration string `json:"configuration,omitempty" description:"Named preview configuration. Optional when exactly one configuration exists."` +} + +// PreviewServerStatusResponse reports the deterministic server AO owns for one +// session. Logs are bounded to the latest lines and never contain global +// process or port discovery. +type PreviewServerStatusResponse struct { + SessionID domain.SessionID `json:"sessionId"` + State string `json:"state" enum:"stopped,starting,ready,stopping,failed"` + Configuration string `json:"configuration,omitempty"` + TargetKind string `json:"targetKind,omitempty" enum:"app,api"` + URL string `json:"url,omitempty"` + Port int `json:"port,omitempty"` + StartedAt time.Time `json:"startedAt,omitempty"` + Error string `json:"error,omitempty"` + Logs []string `json:"logs"` +} + // BrowserStatusQuery selects the session whose logical browser is inspected. type BrowserStatusQuery struct { SessionID domain.SessionID `query:"sessionId" description:"AO session identifier."` diff --git a/backend/internal/httpd/controllers/sessions.go b/backend/internal/httpd/controllers/sessions.go index 5bdf597f27..3bdd1a080f 100644 --- a/backend/internal/httpd/controllers/sessions.go +++ b/backend/internal/httpd/controllers/sessions.go @@ -20,6 +20,7 @@ import ( "github.com/aoagents/agent-orchestrator/backend/internal/httpd/envelope" "github.com/aoagents/agent-orchestrator/backend/internal/ports" previewutil "github.com/aoagents/agent-orchestrator/backend/internal/preview" + "github.com/aoagents/agent-orchestrator/backend/internal/previewserver" sessionsvc "github.com/aoagents/agent-orchestrator/backend/internal/service/session" ) @@ -59,11 +60,20 @@ type ActivityRecorder interface { ApplyActivitySignal(ctx context.Context, id domain.SessionID, s ports.ActivitySignal) error } +// ManagedPreviewServer is the deterministic server lifecycle attached to a +// worker. It is separate from static file rendering and browser automation. +type ManagedPreviewServer interface { + Start(ctx context.Context, sessionID domain.SessionID, workspacePath, configurationName string) (previewserver.Status, error) + Stop(ctx context.Context, sessionID domain.SessionID) (previewserver.Status, error) + Status(sessionID domain.SessionID) previewserver.Status +} + // SessionsController owns the session routes. Nil keeps routes registered but // returns OpenAPI-backed 501s. type SessionsController struct { - Svc SessionService - Activity ActivityRecorder + Svc SessionService + Activity ActivityRecorder + PreviewServer ManagedPreviewServer } // Register mounts the session routes on the supplied router. @@ -75,6 +85,9 @@ func (c *SessionsController) Register(r chi.Router) { r.Get("/sessions/{sessionId}/preview", c.preview) r.Post("/sessions/{sessionId}/preview", c.setPreview) r.Delete("/sessions/{sessionId}/preview", c.clearPreview) + r.Get("/sessions/{sessionId}/preview/server", c.previewServerStatus) + r.Post("/sessions/{sessionId}/preview/server", c.startPreviewServer) + r.Delete("/sessions/{sessionId}/preview/server", c.stopPreviewServer) r.Get("/sessions/{sessionId}/preview/files/*", c.previewFile) r.Get("/sessions/{sessionId}/workspace/files", c.listWorkspaceFiles) r.Get("/sessions/{sessionId}/workspace/file", c.getWorkspaceFile) @@ -405,6 +418,128 @@ func (c *SessionsController) clearPreview(w http.ResponseWriter, r *http.Request envelope.WriteJSON(w, http.StatusOK, SessionResponse{Session: sessionView(updated)}) } +func (c *SessionsController) previewServerStatus(w http.ResponseWriter, r *http.Request) { + if c.Svc == nil || c.PreviewServer == nil { + apispec.NotImplemented(w, r, http.MethodGet, "/api/v1/sessions/{sessionId}/preview/server") + return + } + if _, err := c.Svc.Get(r.Context(), sessionID(r)); err != nil { + envelope.WriteError(w, r, err) + return + } + envelope.WriteJSON(w, http.StatusOK, previewServerStatusResponse(c.PreviewServer.Status(sessionID(r)))) +} + +func (c *SessionsController) startPreviewServer(w http.ResponseWriter, r *http.Request) { + if c.Svc == nil || c.PreviewServer == nil { + apispec.NotImplemented(w, r, http.MethodPost, "/api/v1/sessions/{sessionId}/preview/server") + return + } + var in StartPreviewServerRequest + if err := decodeJSON(r, &in); err != nil { + envelope.WriteAPIError(w, r, http.StatusBadRequest, "bad_request", "INVALID_JSON", "Invalid JSON body", nil) + return + } + sess, err := c.Svc.Get(r.Context(), sessionID(r)) + if err != nil { + envelope.WriteError(w, r, err) + return + } + if sess.IsTerminated { + envelope.WriteAPIError(w, r, http.StatusConflict, "conflict", "SESSION_TERMINATED", "Session is terminated", nil) + return + } + status, err := c.PreviewServer.Start( + r.Context(), + sessionID(r), + sess.Metadata.WorkspacePath, + strings.TrimSpace(in.Configuration), + ) + if err != nil { + writePreviewServerError(w, r, err) + return + } + if status.TargetKind == previewserver.TargetApp { + if _, err := c.Svc.SetPreview(r.Context(), sessionID(r), status.URL); err != nil { + _, _ = c.PreviewServer.Stop(context.Background(), sessionID(r)) + envelope.WriteError(w, r, err) + return + } + } + envelope.WriteJSON(w, http.StatusOK, previewServerStatusResponse(status)) +} + +func (c *SessionsController) stopPreviewServer(w http.ResponseWriter, r *http.Request) { + if c.Svc == nil || c.PreviewServer == nil { + apispec.NotImplemented(w, r, http.MethodDelete, "/api/v1/sessions/{sessionId}/preview/server") + return + } + sess, err := c.Svc.Get(r.Context(), sessionID(r)) + if err != nil { + envelope.WriteError(w, r, err) + return + } + previous := c.PreviewServer.Status(sessionID(r)) + status, err := c.PreviewServer.Stop(r.Context(), sessionID(r)) + if err != nil { + writePreviewServerError(w, r, err) + return + } + if previous.URL != "" && sess.Metadata.PreviewURL == previous.URL { + if _, err := c.Svc.SetPreview(r.Context(), sessionID(r), ""); err != nil { + envelope.WriteError(w, r, err) + return + } + } + envelope.WriteJSON(w, http.StatusOK, previewServerStatusResponse(status)) +} + +func previewServerStatusResponse(status previewserver.Status) PreviewServerStatusResponse { + logs := status.Logs + if logs == nil { + logs = []string{} + } + return PreviewServerStatusResponse{ + SessionID: status.SessionID, + State: string(status.State), + Configuration: status.Configuration, + TargetKind: string(status.TargetKind), + URL: status.URL, + Port: status.Port, + StartedAt: status.StartedAt, + Error: status.Error, + Logs: logs, + } +} + +func writePreviewServerError(w http.ResponseWriter, r *http.Request, err error) { + var serviceErr previewserver.Error + if !errors.As(err, &serviceErr) { + envelope.WriteError(w, r, err) + return + } + status := http.StatusUnprocessableEntity + typeName := "unprocessable" + switch serviceErr.Code { + case "PREVIEW_CONFIG_NOT_FOUND", "PREVIEW_CONFIGURATION_NOT_FOUND": + status = http.StatusNotFound + typeName = "not_found" + case "PREVIEW_CONFIGURATION_REQUIRED": + status = http.StatusBadRequest + typeName = "bad_request" + case "PREVIEW_NOT_READY": + status = http.StatusGatewayTimeout + typeName = "timeout" + case "PREVIEW_START_CANCELED": + status = http.StatusRequestTimeout + typeName = "timeout" + case "PREVIEW_START_FAILED", "PREVIEW_EXITED", "PREVIEW_STOP_FAILED": + status = http.StatusInternalServerError + typeName = "internal_error" + } + envelope.WriteAPIError(w, r, status, typeName, serviceErr.Code, serviceErr.Message, nil) +} + func (c *SessionsController) listPRs(w http.ResponseWriter, r *http.Request) { if c.Svc == nil { apispec.NotImplemented(w, r, "GET", "/api/v1/sessions/{sessionId}/pr") @@ -484,6 +619,9 @@ func (c *SessionsController) kill(w http.ResponseWriter, r *http.Request) { apispec.NotImplemented(w, r, "POST", "/api/v1/sessions/{sessionId}/kill") return } + if c.PreviewServer != nil { + _, _ = c.PreviewServer.Stop(context.Background(), sessionID(r)) + } freed, err := c.Svc.Kill(r.Context(), sessionID(r)) if err != nil { envelope.WriteError(w, r, err) diff --git a/backend/internal/httpd/controllers/sessions_test.go b/backend/internal/httpd/controllers/sessions_test.go index ab65d3ce8c..bcd74ebafd 100644 --- a/backend/internal/httpd/controllers/sessions_test.go +++ b/backend/internal/httpd/controllers/sessions_test.go @@ -20,6 +20,7 @@ import ( "github.com/aoagents/agent-orchestrator/backend/internal/httpd/apierr" "github.com/aoagents/agent-orchestrator/backend/internal/ports" previewutil "github.com/aoagents/agent-orchestrator/backend/internal/preview" + "github.com/aoagents/agent-orchestrator/backend/internal/previewserver" sessionsvc "github.com/aoagents/agent-orchestrator/backend/internal/service/session" ) @@ -37,6 +38,51 @@ type fakeSessionService struct { workspaceErr error } +type fakeManagedPreviewServer struct { + status previewserver.Status + startErr error + startName string + startWorkspace string + stopCalls int +} + +func (f *fakeManagedPreviewServer) Start( + _ context.Context, + sessionID domain.SessionID, + workspacePath string, + configurationName string, +) (previewserver.Status, error) { + f.startName = configurationName + f.startWorkspace = workspacePath + if f.startErr != nil { + return previewserver.Status{}, f.startErr + } + f.status.SessionID = sessionID + return f.status, nil +} + +func (f *fakeManagedPreviewServer) Stop( + _ context.Context, + sessionID domain.SessionID, +) (previewserver.Status, error) { + f.stopCalls++ + f.status.SessionID = sessionID + f.status.State = previewserver.StateStopped + return f.status, nil +} + +func (f *fakeManagedPreviewServer) Status(sessionID domain.SessionID) previewserver.Status { + status := f.status + status.SessionID = sessionID + if status.State == "" { + status.State = previewserver.StateStopped + } + if status.Logs == nil { + status.Logs = []string{} + } + return status +} + func newFakeSessionService() *fakeSessionService { now := time.Now().UTC() s := domain.Session{SessionRecord: domain.SessionRecord{ID: "ao-1", ProjectID: "ao", Kind: domain.KindWorker, Activity: domain.Activity{State: domain.ActivityIdle, LastActivityAt: now}, CreatedAt: now, UpdatedAt: now}, Status: domain.StatusIdle, TerminalHandleID: "ao-1/terminal_0"} @@ -247,9 +293,21 @@ func (f *fakeSessionService) GetWorkspaceFile(_ context.Context, id domain.Sessi } func newSessionTestServer(t *testing.T, svc *fakeSessionService) *httptest.Server { + return newSessionTestServerWithPreview(t, svc, nil) +} + +func newSessionTestServerWithPreview( + t *testing.T, + svc *fakeSessionService, + managed *fakeManagedPreviewServer, +) *httptest.Server { t.Helper() log := slog.New(slog.NewTextHandler(io.Discard, nil)) - srv := httptest.NewServer(httpd.NewRouterWithControl(config.Config{}, log, nil, httpd.APIDeps{Sessions: svc}, httpd.ControlDeps{})) + deps := httpd.APIDeps{Sessions: svc} + if managed != nil { + deps.PreviewServer = managed + } + srv := httptest.NewServer(httpd.NewRouterWithControl(config.Config{}, log, nil, deps, httpd.ControlDeps{})) t.Cleanup(srv.Close) return srv } @@ -631,6 +689,59 @@ func TestSessionsAPI_SetPreviewLocalRelativePathResolvesToPreviewOrigin(t *testi } } +func TestSessionsAPI_SetPreviewServesBrowserDisplayableArtifacts(t *testing.T) { + tests := []struct { + name string + path string + contents []byte + contentType string + }{ + {name: "PDF", path: "artifacts/report.pdf", contents: []byte("%PDF-1.4\n%%EOF\n"), contentType: "application/pdf"}, + {name: "PNG", path: "artifacts/mockup.png", contents: []byte("\x89PNG\r\n\x1a\n"), contentType: "image/png"}, + {name: "SVG", path: "artifacts/diagram.svg", contents: []byte(``), contentType: "image/svg+xml"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + svc := newFakeSessionService() + workspace := t.TempDir() + artifact := filepath.Join(workspace, filepath.FromSlash(tc.path)) + if err := os.MkdirAll(filepath.Dir(artifact), 0o755); err != nil { + t.Fatalf("mkdir artifact dir: %v", err) + } + if err := os.WriteFile(artifact, tc.contents, 0o644); err != nil { + t.Fatalf("write artifact: %v", err) + } + session := svc.sessions["ao-1"] + session.Metadata = domain.SessionMetadata{WorkspacePath: workspace} + svc.sessions["ao-1"] = session + srv := newSessionTestServer(t, svc) + + request := `{"url":` + strconv.Quote(tc.path) + `}` + body, status, _ := doRequest(t, srv, http.MethodPost, "/api/v1/sessions/ao-1/preview", request) + if status != http.StatusOK { + t.Fatalf("set artifact preview = %d body=%s", status, body) + } + var response struct { + Session struct { + PreviewURL string `json:"previewUrl"` + } `json:"session"` + } + mustJSON(t, body, &response) + if !strings.HasSuffix(response.Session.PreviewURL, "/"+tc.path) { + t.Fatalf("preview URL = %q, want suffix /%s", response.Session.PreviewURL, tc.path) + } + + served, servedStatus, headers := doPreviewOriginRequest(t, srv, response.Session.PreviewURL, "/") + if servedStatus != http.StatusOK { + t.Fatalf("serve artifact = %d body=%q", servedStatus, served) + } + if got := headers.Get("Content-Type"); !strings.HasPrefix(got, tc.contentType) { + t.Fatalf("Content-Type = %q, want %q", got, tc.contentType) + } + }) + } +} + func TestSessionsAPI_PreviewOriginResolvesRootRelativeAssetsFromEntryDirectory(t *testing.T) { svc := newFakeSessionService() workspace := t.TempDir() @@ -951,6 +1062,100 @@ func TestSessionsAPI_ClearPreviewNotFound(t *testing.T) { assertErrorCode(t, body, status, http.StatusNotFound, "SESSION_NOT_FOUND") } +func TestSessionsAPI_ManagedPreviewStartsExactApplicationAndPersistsTarget(t *testing.T) { + svc := newFakeSessionService() + session := svc.sessions["ao-1"] + session.Metadata.WorkspacePath = t.TempDir() + svc.sessions["ao-1"] = session + managed := &fakeManagedPreviewServer{status: previewserver.Status{ + State: previewserver.StateReady, + Configuration: "web", + TargetKind: previewserver.TargetApp, + URL: "http://127.0.0.1:43123/", + Port: 43123, + Logs: []string{"ready"}, + }} + srv := newSessionTestServerWithPreview(t, svc, managed) + + body, status, _ := doRequest( + t, + srv, + http.MethodPost, + "/api/v1/sessions/ao-1/preview/server", + `{"configuration":"web"}`, + ) + if status != http.StatusOK || !containsAll(body, `"state":"ready"`, `"configuration":"web"`, `"targetKind":"app"`) { + t.Fatalf("start managed preview = %d body=%s", status, body) + } + if managed.startName != "web" || managed.startWorkspace != session.Metadata.WorkspacePath { + t.Fatalf("start args = name %q workspace %q", managed.startName, managed.startWorkspace) + } + if got := svc.sessions["ao-1"].Metadata.PreviewURL; got != managed.status.URL { + t.Fatalf("persisted preview URL = %q, want %q", got, managed.status.URL) + } +} + +func TestSessionsAPI_APIManagedPreviewDoesNotTakeOverBrowser(t *testing.T) { + svc := newFakeSessionService() + session := svc.sessions["ao-1"] + session.Metadata.WorkspacePath = t.TempDir() + session.Metadata.PreviewURL = "http://127.0.0.1:4173/" + svc.sessions["ao-1"] = session + managed := &fakeManagedPreviewServer{status: previewserver.Status{ + State: previewserver.StateReady, + Configuration: "api", + TargetKind: previewserver.TargetAPI, + URL: "http://127.0.0.1:8080/health", + Port: 8080, + Logs: []string{}, + }} + srv := newSessionTestServerWithPreview(t, svc, managed) + + body, status, _ := doRequest(t, srv, http.MethodPost, "/api/v1/sessions/ao-1/preview/server", `{}`) + if status != http.StatusOK { + t.Fatalf("start API preview = %d body=%s", status, body) + } + if got := svc.sessions["ao-1"].Metadata.PreviewURL; got != "http://127.0.0.1:4173/" { + t.Fatalf("API server replaced browser target with %q", got) + } +} + +func TestSessionsAPI_StopManagedPreviewPreservesExplicitFileTarget(t *testing.T) { + svc := newFakeSessionService() + session := svc.sessions["ao-1"] + session.Metadata.PreviewURL = "http://ao-1.preview.localhost:3001/README.md" + svc.sessions["ao-1"] = session + managed := &fakeManagedPreviewServer{status: previewserver.Status{ + State: previewserver.StateReady, + TargetKind: previewserver.TargetApp, + URL: "http://127.0.0.1:4173/", + Logs: []string{}, + }} + srv := newSessionTestServerWithPreview(t, svc, managed) + + body, status, _ := doRequest(t, srv, http.MethodDelete, "/api/v1/sessions/ao-1/preview/server", "") + if status != http.StatusOK || !containsAll(body, `"state":"stopped"`) { + t.Fatalf("stop managed preview = %d body=%s", status, body) + } + if got := svc.sessions["ao-1"].Metadata.PreviewURL; got != session.Metadata.PreviewURL { + t.Fatalf("explicit file target was cleared: %q", got) + } +} + +func TestSessionsAPI_KillStopsManagedPreview(t *testing.T) { + svc := newFakeSessionService() + managed := &fakeManagedPreviewServer{} + srv := newSessionTestServerWithPreview(t, svc, managed) + + body, status, _ := doRequest(t, srv, http.MethodPost, "/api/v1/sessions/ao-1/kill", "") + if status != http.StatusOK { + t.Fatalf("kill = %d body=%s", status, body) + } + if managed.stopCalls != 1 { + t.Fatalf("managed preview stop calls = %d, want 1", managed.stopCalls) + } +} + func TestSessionsAPI_ListWorkspaceFiles(t *testing.T) { svc := newFakeSessionService() svc.workspaceFiles = sessionsvc.WorkspaceFiles{ diff --git a/backend/internal/ports/outbound.go b/backend/internal/ports/outbound.go index d1bb53db57..af84f9809e 100644 --- a/backend/internal/ports/outbound.go +++ b/backend/internal/ports/outbound.go @@ -103,6 +103,13 @@ type RuntimeHandle struct { ID string } +// SessionPreviewLifecycle is the narrow teardown hook for any managed preview +// process attached to a session. Session Manager invokes it before destroying +// the agent runtime/workspace so non-HTTP lifecycle paths cannot leak servers. +type SessionPreviewLifecycle interface { + StopSession(ctx context.Context, id domain.SessionID) error +} + // Stream is one live terminal attach: PTY-like bytes plus resize. Returned // already-open by a Runtime's Attach. tmux backs it with a local PTY around // their attach CLI; conpty backs it with a loopback connection to the pty-host. diff --git a/backend/internal/preview/poller.go b/backend/internal/preview/poller.go index 782620dafe..f6119389e5 100644 --- a/backend/internal/preview/poller.go +++ b/backend/internal/preview/poller.go @@ -27,8 +27,10 @@ type PollerConfig struct { Logger *slog.Logger } -// Poller watches active worker workspaces for static frontend entrypoints and -// persists preview URL refreshes through the normal session service path. +// Poller watches explicitly selected workspace previews and persists refreshes +// through the normal session service path. It never chooses a preview for a +// fresh worker: selection belongs to `ao preview`, a managed server start, or +// deliberate user navigation. type Poller struct { source sessionPreviewSource setter previewSetter @@ -113,25 +115,33 @@ func (p *Poller) Poll(ctx context.Context) error { continue } storedEntry, workspaceOwned := StoredWorkspaceEntry(sess.Metadata.PreviewURL, sess.ID) - entry, ok := Entry{}, false - if workspaceOwned { - entry, ok = EntryAtPath(sess.Metadata.WorkspacePath, storedEntry) - } - if !ok { - entry, ok = DiscoverEntry(sess.Metadata.WorkspacePath) + previous, seenBefore := p.seen[sess.ID] + restoringCleared := false + if !workspaceOwned { + // Only restore an entry after the poller itself cleared it because + // the selected file temporarily disappeared. A blank fresh session + // or an explicit user clear must remain blank. + if !seenBefore || !previous.cleared || previous.path == "" { + continue + } + storedEntry = previous.path + workspaceOwned = true + restoringCleared = true } + entry, ok := EntryAtPath(sess.Metadata.WorkspacePath, storedEntry) if !ok { if workspaceOwned { - if _, err := p.setter.SetPreview(ctx, sess.ID, ""); err != nil { - p.logger.Error("preview poller: failed to clear stale preview", - "session", sess.ID, "err", err) + if !restoringCleared { + if _, err := p.setter.SetPreview(ctx, sess.ID, ""); err != nil { + p.logger.Error("preview poller: failed to clear stale preview", + "session", sess.ID, "err", err) + } } - p.seen[sess.ID] = entryState{cleared: true} + p.seen[sess.ID] = entryState{path: storedEntry, cleared: true} } continue } state := stateFor(entry) - previous, seenBefore := p.seen[sess.ID] if seenBefore && previous == state { continue } diff --git a/backend/internal/preview/poller_test.go b/backend/internal/preview/poller_test.go index 68455ecf9c..bbf82e32b4 100644 --- a/backend/internal/preview/poller_test.go +++ b/backend/internal/preview/poller_test.go @@ -38,7 +38,7 @@ func (f *fakePreviewSessions) SetPreview(_ context.Context, id domain.SessionID, return domain.Session{}, nil } -func TestPollerSetsPreviewWhenActiveWorkerEntryAppears(t *testing.T) { +func TestPollerDoesNotOpenUntargetedWorkerEntry(t *testing.T) { workspace := t.TempDir() writeFile(t, filepath.Join(workspace, "index.html"), "

hello
") svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "")}} @@ -48,16 +48,13 @@ func TestPollerSetsPreviewWhenActiveWorkerEntryAppears(t *testing.T) { t.Fatalf("Poll: %v", err) } - assertSets(t, svc.sets, previewSet{ - id: "ao-1", - url: mustFileURL(t, "http://127.0.0.1:3001", "ao-1", "index.html"), - }) + assertSets(t, svc.sets) } -func TestPollerUsesFirstExistingEntrypoint(t *testing.T) { +func TestPollerNormalizesExplicitWorkspaceEntrypoint(t *testing.T) { workspace := t.TempDir() writeFile(t, filepath.Join(workspace, "dist", "index.html"), "
dist
") - svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "")}} + svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "dist/index.html")}} poller := NewPoller(svc, svc, "http://127.0.0.1:3001", PollerConfig{Logger: discardLogger()}) if err := poller.Poll(context.Background()); err != nil { @@ -70,11 +67,11 @@ func TestPollerUsesFirstExistingEntrypoint(t *testing.T) { }) } -func TestPollerPreservesEntrypointPriority(t *testing.T) { +func TestPollerDoesNotReplaceExplicitEntrypointWithAnotherStaticFile(t *testing.T) { workspace := t.TempDir() writeFile(t, filepath.Join(workspace, "public", "index.html"), "
public
") writeFile(t, filepath.Join(workspace, "dist", "index.html"), "
dist
") - svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "")}} + svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "dist/index.html")}} poller := NewPoller(svc, svc, "http://127.0.0.1:3001", PollerConfig{Logger: discardLogger()}) if err := poller.Poll(context.Background()); err != nil { @@ -83,7 +80,7 @@ func TestPollerPreservesEntrypointPriority(t *testing.T) { assertSets(t, svc.sets, previewSet{ id: "ao-1", - url: mustFileURL(t, "http://127.0.0.1:3001", "ao-1", "public/index.html"), + url: mustFileURL(t, "http://127.0.0.1:3001", "ao-1", "dist/index.html"), }) } @@ -91,7 +88,7 @@ func TestPollerRefreshesOnlyWhenEntrypointChanges(t *testing.T) { workspace := t.TempDir() entry := filepath.Join(workspace, "index.html") writeFile(t, entry, "
v1
") - svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "")}} + svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "index.html")}} poller := NewPoller(svc, svc, "http://127.0.0.1:3001", PollerConfig{Logger: discardLogger()}) if err := poller.Poll(context.Background()); err != nil { @@ -122,10 +119,10 @@ func TestPollerRediscoverEntryAfterDeleteAndRecreate(t *testing.T) { workspace := t.TempDir() entry := filepath.Join(workspace, "index.html") writeFile(t, entry, "
v1
") - svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "")}} + svc := &fakePreviewSessions{sessions: []domain.SessionRecord{workerSession("ao-1", workspace, "index.html")}} poller := NewPoller(svc, svc, "http://127.0.0.1:3001", PollerConfig{Logger: discardLogger()}) - // First poll discovers the entry and sets the preview. + // First poll normalizes the explicitly selected workspace entry. if err := poller.Poll(context.Background()); err != nil { t.Fatalf("first Poll: %v", err) } @@ -147,9 +144,16 @@ func TestPollerRediscoverEntryAfterDeleteAndRecreate(t *testing.T) { } // Recreate the entry — poller must re-discover. + if err := poller.Poll(context.Background()); err != nil { + t.Fatalf("third Poll (still deleted): %v", err) + } + if len(svc.sets) != 2 { + t.Fatalf("sets while entry remains deleted = %#v, want no repeated clear", svc.sets) + } + writeFile(t, entry, "
v2
") if err := poller.Poll(context.Background()); err != nil { - t.Fatalf("third Poll (recreate): %v", err) + t.Fatalf("fourth Poll (recreate): %v", err) } if len(svc.sets) != 3 { t.Fatalf("sets after recreate = %#v, want 3 sets (discover + clear + rediscover)", svc.sets) diff --git a/backend/internal/previewserver/manager.go b/backend/internal/previewserver/manager.go new file mode 100644 index 0000000000..b444a2f1da --- /dev/null +++ b/backend/internal/previewserver/manager.go @@ -0,0 +1,754 @@ +// Package previewserver owns deterministic, session-scoped development server +// processes used by the desktop preview. It deliberately does not scan global +// localhost ports: a server is started from an explicit project configuration, +// so its worker, command, working directory, and URL are known. +package previewserver + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/aoagents/agent-orchestrator/backend/internal/domain" +) + +const ( + ConfigPath = ".ao/launch.json" + defaultReadyTimeout = 30 * time.Second + maxReadyTimeout = 55 * time.Second + probeInterval = 150 * time.Millisecond + probeTimeout = time.Second + maxRedirects = 5 + statusLogLines = 40 + maxBufferedLogLines = 200 + maxBufferedPartialBytes = 16 * 1024 +) + +type State string + +const ( + StateStopped State = "stopped" + StateStarting State = "starting" + StateReady State = "ready" + StateStopping State = "stopping" + StateFailed State = "failed" +) + +type TargetKind string + +const ( + TargetApp TargetKind = "app" + TargetAPI TargetKind = "api" +) + +// Error is a stable service error that the HTTP controller can map without +// parsing platform-specific process errors. +type Error struct { + Code string + Message string +} + +func (e Error) Error() string { return e.Message } + +// Status is the public state for one session's managed preview server. +type Status struct { + SessionID domain.SessionID `json:"sessionId"` + State State `json:"state"` + Configuration string `json:"configuration,omitempty"` + TargetKind TargetKind `json:"targetKind,omitempty"` + URL string `json:"url,omitempty"` + Port int `json:"port,omitempty"` + StartedAt time.Time `json:"startedAt,omitempty"` + Error string `json:"error,omitempty"` + Logs []string `json:"logs"` +} + +type launchFile struct { + Version int `json:"version"` + Configurations []Configuration `json:"configurations"` +} + +// Configuration is one named server entry in .ao/launch.json. ${PORT} is +// expanded in runtimeArgs, url, and env values after AO chooses the port. +type Configuration struct { + Name string `json:"name"` + RuntimeExecutable string `json:"runtimeExecutable"` + RuntimeArgs []string `json:"runtimeArgs,omitempty"` + Cwd string `json:"cwd,omitempty"` + Port int `json:"port,omitempty"` + AutoPort bool `json:"autoPort,omitempty"` + URL string `json:"url,omitempty"` + Env map[string]string `json:"env,omitempty"` + TargetKind TargetKind `json:"targetKind,omitempty"` + ReadyTimeoutMillis int `json:"readyTimeoutMs,omitempty"` +} + +type serverRun struct { + status Status + cmd *exec.Cmd + done chan struct{} + logs *lineBuffer + stopping bool +} + +// Manager supervises at most one managed preview server per AO session. +type Manager struct { + log *slog.Logger + client *http.Client + + mu sync.Mutex + runs map[domain.SessionID]*serverRun + + operationsMu sync.Mutex + operations map[domain.SessionID]*sync.Mutex +} + +func New(log *slog.Logger) *Manager { + if log == nil { + log = slog.Default() + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + manager := &Manager{ + log: log, + runs: make(map[domain.SessionID]*serverRun), + operations: make(map[domain.SessionID]*sync.Mutex), + } + manager.client = &http.Client{ + Transport: transport, + Timeout: probeTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= maxRedirects { + return errors.New("too many preview readiness redirects") + } + if !isLoopbackHost(req.URL.Hostname()) { + return errors.New("preview readiness redirected away from loopback") + } + return nil + }, + } + return manager +} + +// Start loads a named configuration, replaces any existing managed server for +// the session, and waits until the configured loopback URL responds. +func (m *Manager) Start( + ctx context.Context, + sessionID domain.SessionID, + workspacePath string, + configurationName string, +) (Status, error) { + operation := m.operationLock(sessionID) + operation.Lock() + operationLocked := true + defer func() { + if operationLocked { + operation.Unlock() + } + }() + + cfg, err := loadConfiguration(workspacePath, configurationName) + if err != nil { + return stoppedStatus(sessionID), err + } + if _, err := m.stop(context.Background(), sessionID); err != nil { + return stoppedStatus(sessionID), err + } + + port, err := selectPort(cfg.Port, cfg.AutoPort) + if err != nil { + return stoppedStatus(sessionID), serviceError("PREVIEW_PORT_UNAVAILABLE", err.Error()) + } + workingDir, err := resolveWorkingDirectory(workspacePath, cfg.Cwd) + if err != nil { + return stoppedStatus(sessionID), err + } + targetURL, err := resolveTargetURL(cfg.URL, port) + if err != nil { + return stoppedStatus(sessionID), err + } + executable := interpolatePort(cfg.RuntimeExecutable, port) + if strings.ContainsAny(executable, `/\`) && !filepath.IsAbs(executable) { + executable = filepath.Join(workingDir, executable) + } + args := make([]string, len(cfg.RuntimeArgs)) + for i, arg := range cfg.RuntimeArgs { + args[i] = interpolatePort(arg, port) + } + + cmd := previewCommand(executable, args...) + cmd.Dir = workingDir + cmd.Env = previewEnvironment(os.Environ(), cfg.Env, sessionID, port) + logs := newLineBuffer(maxBufferedLogLines) + cmd.Stdout = logs + cmd.Stderr = logs + + run := &serverRun{ + status: Status{ + SessionID: sessionID, + State: StateStarting, + Configuration: cfg.Name, + TargetKind: cfg.TargetKind, + URL: targetURL, + Port: port, + StartedAt: time.Now().UTC(), + Logs: []string{}, + }, + cmd: cmd, + done: make(chan struct{}), + logs: logs, + } + if err := cmd.Start(); err != nil { + run.cmd = nil + run.status.State = StateFailed + run.status.Error = fmt.Sprintf("start preview server: %v", err) + m.mu.Lock() + m.runs[sessionID] = run + m.mu.Unlock() + return m.statusFor(run), serviceError("PREVIEW_START_FAILED", run.status.Error) + } + + m.mu.Lock() + m.runs[sessionID] = run + m.mu.Unlock() + go m.waitForExit(sessionID, run) + operation.Unlock() + operationLocked = false + + timeout := readyTimeout(cfg.ReadyTimeoutMillis) + deadline := time.Now().Add(timeout) + for { + if err := m.probe(ctx, targetURL); err == nil { + m.mu.Lock() + ready := false + if m.runs[sessionID] == run && run.cmd != nil && !run.stopping { + run.status.State = StateReady + run.status.Error = "" + ready = true + } + status := m.statusForLocked(run) + m.mu.Unlock() + if !ready { + return status, serviceError("PREVIEW_START_CANCELED", "preview server was stopped before becoming ready") + } + m.log.Info("preview server ready", "session", sessionID, "configuration", cfg.Name, "url", targetURL) + return status, nil + } + + select { + case <-ctx.Done(): + message := "preview start canceled: " + ctx.Err().Error() + return m.failAndStop(sessionID, run, "PREVIEW_START_CANCELED", message) + case <-run.done: + status := m.statusFor(run) + if status.Error == "" { + status.Error = "preview server exited before becoming ready" + } + return status, serviceError("PREVIEW_EXITED", status.Error) + default: + } + if time.Now().After(deadline) { + message := fmt.Sprintf("preview server was not ready at %s within %s", targetURL, timeout) + return m.failAndStop(sessionID, run, "PREVIEW_NOT_READY", message) + } + timer := time.NewTimer(probeInterval) + select { + case <-ctx.Done(): + timer.Stop() + message := "preview start canceled: " + ctx.Err().Error() + return m.failAndStop(sessionID, run, "PREVIEW_START_CANCELED", message) + case <-run.done: + timer.Stop() + status := m.statusFor(run) + if status.Error == "" { + status.Error = "preview server exited before becoming ready" + } + return status, serviceError("PREVIEW_EXITED", status.Error) + case <-timer.C: + } + } +} + +// Stop terminates the exact process tree AO launched for the session. +func (m *Manager) Stop(ctx context.Context, sessionID domain.SessionID) (Status, error) { + operation := m.operationLock(sessionID) + operation.Lock() + defer operation.Unlock() + return m.stop(ctx, sessionID) +} + +func (m *Manager) stop(ctx context.Context, sessionID domain.SessionID) (Status, error) { + m.mu.Lock() + run := m.runs[sessionID] + if run == nil { + m.mu.Unlock() + return stoppedStatus(sessionID), nil + } + if run.cmd == nil { + run.status.State = StateStopped + run.status.Error = "" + status := m.statusForLocked(run) + m.mu.Unlock() + return status, nil + } + run.stopping = true + run.status.State = StateStopping + cmd := run.cmd + done := run.done + m.mu.Unlock() + + if err := terminatePreviewProcess(cmd); err != nil { + m.log.Warn("stop preview process tree", "session", sessionID, "err", err) + } + select { + case <-done: + case <-ctx.Done(): + return m.Status(sessionID), ctx.Err() + case <-time.After(5 * time.Second): + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + select { + case <-done: + case <-ctx.Done(): + return m.Status(sessionID), ctx.Err() + case <-time.After(time.Second): + message := "preview server process did not exit after it was killed" + m.mu.Lock() + if m.runs[sessionID] == run { + run.status.State = StateFailed + run.status.Error = message + } + status := m.statusForLocked(run) + m.mu.Unlock() + return status, serviceError("PREVIEW_STOP_FAILED", message) + } + } + + m.mu.Lock() + if m.runs[sessionID] == run { + run.status.State = StateStopped + run.status.Error = "" + } + status := m.statusForLocked(run) + m.mu.Unlock() + return status, nil +} + +func (m *Manager) operationLock(sessionID domain.SessionID) *sync.Mutex { + m.operationsMu.Lock() + defer m.operationsMu.Unlock() + operation := m.operations[sessionID] + if operation == nil { + operation = &sync.Mutex{} + m.operations[sessionID] = operation + } + return operation +} + +func (m *Manager) Status(sessionID domain.SessionID) Status { + m.mu.Lock() + defer m.mu.Unlock() + run := m.runs[sessionID] + if run == nil { + return stoppedStatus(sessionID) + } + return m.statusForLocked(run) +} + +// StopSession implements ports.SessionPreviewLifecycle without exposing status +// to lifecycle callers that only need best-effort teardown. +func (m *Manager) StopSession(ctx context.Context, sessionID domain.SessionID) error { + _, err := m.Stop(ctx, sessionID) + return err +} + +// Close stops every server owned by the daemon. It is safe to call repeatedly. +func (m *Manager) Close() { + m.mu.Lock() + ids := make([]domain.SessionID, 0, len(m.runs)) + for id := range m.runs { + ids = append(ids, id) + } + m.mu.Unlock() + for _, id := range ids { + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + _, _ = m.Stop(ctx, id) + cancel() + } +} + +func (m *Manager) waitForExit(sessionID domain.SessionID, run *serverRun) { + err := run.cmd.Wait() + m.mu.Lock() + if m.runs[sessionID] == run { + run.cmd = nil + if run.stopping { + if run.status.State != StateFailed { + run.status.State = StateStopped + run.status.Error = "" + } + } else { + run.status.State = StateFailed + if err != nil { + run.status.Error = fmt.Sprintf("preview server exited: %v", err) + } else { + run.status.Error = "preview server exited" + } + } + } + m.mu.Unlock() + close(run.done) +} + +func (m *Manager) failAndStop( + sessionID domain.SessionID, + run *serverRun, + code string, + message string, +) (Status, error) { + m.mu.Lock() + if m.runs[sessionID] == run { + run.status.State = StateFailed + run.status.Error = message + run.stopping = true + } + cmd := run.cmd + m.mu.Unlock() + if cmd != nil { + _ = terminatePreviewProcess(cmd) + select { + case <-run.done: + case <-time.After(3 * time.Second): + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + } + } + return m.statusFor(run), serviceError(code, message) +} + +func (m *Manager) probe(ctx context.Context, target string) error { + probeCtx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, target, nil) + if err != nil { + return err + } + resp, err := m.client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.CopyN(io.Discard, resp.Body, 1024) + if resp.StatusCode >= http.StatusInternalServerError { + return fmt.Errorf("preview returned %s", resp.Status) + } + return nil +} + +func (m *Manager) statusFor(run *serverRun) Status { + m.mu.Lock() + defer m.mu.Unlock() + return m.statusForLocked(run) +} + +func (m *Manager) statusForLocked(run *serverRun) Status { + status := run.status + status.Logs = run.logs.Last(statusLogLines) + if status.Logs == nil { + status.Logs = []string{} + } + return status +} + +func loadConfiguration(workspacePath, requestedName string) (Configuration, error) { + if strings.TrimSpace(workspacePath) == "" { + return Configuration{}, serviceError("PREVIEW_WORKSPACE_MISSING", "session workspace is unavailable") + } + configPath := filepath.Join(workspacePath, filepath.FromSlash(ConfigPath)) + data, err := os.ReadFile(configPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Configuration{}, serviceError( + "PREVIEW_CONFIG_NOT_FOUND", + fmt.Sprintf("preview configuration not found at %s", ConfigPath), + ) + } + return Configuration{}, serviceError("PREVIEW_CONFIG_INVALID", fmt.Sprintf("read %s: %v", ConfigPath, err)) + } + var file launchFile + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&file); err != nil { + return Configuration{}, serviceError("PREVIEW_CONFIG_INVALID", fmt.Sprintf("decode %s: %v", ConfigPath, err)) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("unexpected trailing JSON value") + } + return Configuration{}, serviceError("PREVIEW_CONFIG_INVALID", fmt.Sprintf("decode %s: %v", ConfigPath, err)) + } + if file.Version != 1 { + return Configuration{}, serviceError("PREVIEW_CONFIG_INVALID", "preview configuration version must be 1") + } + if len(file.Configurations) == 0 { + return Configuration{}, serviceError("PREVIEW_CONFIG_INVALID", "preview configuration has no configurations") + } + requestedName = strings.TrimSpace(requestedName) + if requestedName == "" && len(file.Configurations) > 1 { + return Configuration{}, serviceError( + "PREVIEW_CONFIGURATION_REQUIRED", + "multiple preview configurations exist; specify one by name", + ) + } + seen := make(map[string]struct{}, len(file.Configurations)) + var selected *Configuration + for i := range file.Configurations { + cfg := file.Configurations[i] + cfg.Name = strings.TrimSpace(cfg.Name) + if cfg.Name == "" { + return Configuration{}, serviceError("PREVIEW_CONFIG_INVALID", "preview configuration name is required") + } + if _, duplicate := seen[cfg.Name]; duplicate { + return Configuration{}, serviceError( + "PREVIEW_CONFIG_INVALID", + fmt.Sprintf("duplicate preview configuration name %q", cfg.Name), + ) + } + seen[cfg.Name] = struct{}{} + if err := validateConfiguration(&cfg); err != nil { + return Configuration{}, err + } + if requestedName == "" || cfg.Name == requestedName { + candidate := cfg + selected = &candidate + } + } + if selected != nil { + return *selected, nil + } + return Configuration{}, serviceError( + "PREVIEW_CONFIGURATION_NOT_FOUND", + fmt.Sprintf("preview configuration %q was not found", requestedName), + ) +} + +func validateConfiguration(cfg *Configuration) error { + cfg.RuntimeExecutable = strings.TrimSpace(cfg.RuntimeExecutable) + if cfg.RuntimeExecutable == "" { + return serviceError( + "PREVIEW_CONFIG_INVALID", + fmt.Sprintf("preview configuration %q requires runtimeExecutable", cfg.Name), + ) + } + if cfg.Port < 0 || cfg.Port > 65535 || (!cfg.AutoPort && cfg.Port == 0) { + return serviceError( + "PREVIEW_CONFIG_INVALID", + fmt.Sprintf("preview configuration %q has an invalid port", cfg.Name), + ) + } + if cfg.TargetKind == "" { + cfg.TargetKind = TargetApp + } + if cfg.TargetKind != TargetApp && cfg.TargetKind != TargetAPI { + return serviceError( + "PREVIEW_CONFIG_INVALID", + fmt.Sprintf("preview configuration %q targetKind must be app or api", cfg.Name), + ) + } + if cfg.ReadyTimeoutMillis < 0 || time.Duration(cfg.ReadyTimeoutMillis)*time.Millisecond > maxReadyTimeout { + return serviceError( + "PREVIEW_CONFIG_INVALID", + fmt.Sprintf("preview configuration %q readyTimeoutMs must be between 0 and %d", cfg.Name, maxReadyTimeout.Milliseconds()), + ) + } + for key := range cfg.Env { + if strings.TrimSpace(key) == "" || strings.ContainsAny(key, "=\x00") { + return serviceError( + "PREVIEW_CONFIG_INVALID", + fmt.Sprintf("preview configuration %q has an invalid env key %q", cfg.Name, key), + ) + } + } + return nil +} + +func resolveWorkingDirectory(workspacePath, relative string) (string, error) { + workspaceAbs, err := filepath.Abs(workspacePath) + if err != nil { + return "", serviceError("PREVIEW_CONFIG_INVALID", "resolve session workspace: "+err.Error()) + } + relative = strings.TrimSpace(relative) + if filepath.IsAbs(relative) { + return "", serviceError("PREVIEW_CONFIG_INVALID", "preview cwd must be relative to the session workspace") + } + target := filepath.Clean(filepath.Join(workspaceAbs, relative)) + rel, err := filepath.Rel(workspaceAbs, target) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", serviceError("PREVIEW_CONFIG_INVALID", "preview cwd escapes the session workspace") + } + info, err := os.Stat(target) + if err != nil || !info.IsDir() { + return "", serviceError("PREVIEW_CONFIG_INVALID", fmt.Sprintf("preview cwd %q is not a directory", relative)) + } + // Reject symlinked descendants instead of resolving them. This prevents a + // configured cwd from escaping the worker workspace while avoiding + // platform-specific canonicalization of trusted workspace ancestors. + current := workspaceAbs + if rel != "." { + for _, part := range strings.Split(rel, string(filepath.Separator)) { + current = filepath.Join(current, part) + entry, err := os.Lstat(current) + if err != nil { + return "", serviceError("PREVIEW_CONFIG_INVALID", fmt.Sprintf("inspect preview cwd %q: %v", relative, err)) + } + if entry.Mode()&os.ModeSymlink != 0 { + return "", serviceError("PREVIEW_CONFIG_INVALID", "preview cwd cannot contain symlinks") + } + } + } + return target, nil +} + +func resolveTargetURL(raw string, port int) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + raw = "http://127.0.0.1:${PORT}/" + } + parsed, err := url.Parse(interpolatePort(raw, port)) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return "", serviceError("PREVIEW_CONFIG_INVALID", "preview url must be an http(s) loopback URL") + } + if !isLoopbackHost(parsed.Hostname()) { + return "", serviceError("PREVIEW_CONFIG_INVALID", "preview url must use localhost or a loopback address") + } + return parsed.String(), nil +} + +func selectPort(preferred int, auto bool) (int, error) { + if !auto { + return preferred, nil + } + address := "127.0.0.1:0" + if preferred > 0 { + address = net.JoinHostPort("127.0.0.1", strconv.Itoa(preferred)) + } + listener, err := net.Listen("tcp", address) + if err != nil && preferred > 0 { + listener, err = net.Listen("tcp", "127.0.0.1:0") + } + if err != nil { + return 0, fmt.Errorf("select preview port: %w", err) + } + port := listener.Addr().(*net.TCPAddr).Port + if err := listener.Close(); err != nil { + return 0, fmt.Errorf("release selected preview port: %w", err) + } + return port, nil +} + +func previewEnvironment( + base []string, + configured map[string]string, + sessionID domain.SessionID, + port int, +) []string { + env := append([]string{}, base...) + for key, value := range configured { + env = append(env, key+"="+interpolatePort(value, port)) + } + env = append(env, + "PORT="+strconv.Itoa(port), + "AO_PREVIEW_PORT="+strconv.Itoa(port), + "AO_SESSION_ID="+string(sessionID), + ) + return env +} + +func interpolatePort(value string, port int) string { + return strings.ReplaceAll(value, "${PORT}", strconv.Itoa(port)) +} + +func readyTimeout(milliseconds int) time.Duration { + if milliseconds <= 0 { + return defaultReadyTimeout + } + return time.Duration(milliseconds) * time.Millisecond +} + +func isLoopbackHost(host string) bool { + if strings.EqualFold(strings.TrimSpace(host), "localhost") { + return true + } + ip := net.ParseIP(strings.Trim(host, "[]")) + return ip != nil && ip.IsLoopback() +} + +func stoppedStatus(sessionID domain.SessionID) Status { + return Status{SessionID: sessionID, State: StateStopped, Logs: []string{}} +} + +func serviceError(code, message string) Error { + return Error{Code: code, Message: message} +} + +type lineBuffer struct { + mu sync.Mutex + max int + lines []string + partial string +} + +func newLineBuffer(max int) *lineBuffer { + return &lineBuffer{max: max} +} + +func (b *lineBuffer) Write(data []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + text := b.partial + string(data) + parts := strings.Split(text, "\n") + b.partial = parts[len(parts)-1] + if len(b.partial) > maxBufferedPartialBytes { + b.partial = b.partial[len(b.partial)-maxBufferedPartialBytes:] + } + for _, line := range parts[:len(parts)-1] { + b.appendLocked(strings.TrimSuffix(line, "\r")) + } + return len(data), nil +} + +func (b *lineBuffer) Last(limit int) []string { + b.mu.Lock() + defer b.mu.Unlock() + lines := append([]string{}, b.lines...) + if b.partial != "" { + lines = append(lines, b.partial) + } + if len(lines) > limit { + lines = lines[len(lines)-limit:] + } + return lines +} + +func (b *lineBuffer) appendLocked(line string) { + b.lines = append(b.lines, line) + if len(b.lines) > b.max { + b.lines = append([]string{}, b.lines[len(b.lines)-b.max:]...) + } +} diff --git a/backend/internal/previewserver/manager_test.go b/backend/internal/previewserver/manager_test.go new file mode 100644 index 0000000000..144bdf1cf2 --- /dev/null +++ b/backend/internal/previewserver/manager_test.go @@ -0,0 +1,171 @@ +package previewserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/aoagents/agent-orchestrator/backend/internal/domain" +) + +func TestPreviewServerHelper(t *testing.T) { + if os.Getenv("AO_PREVIEW_TEST_HELPER") != "1" { + return + } + port, err := strconv.Atoi(os.Getenv("PORT")) + if err != nil || port <= 0 { + t.Fatalf("invalid helper PORT %q", os.Getenv("PORT")) + } + listener, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + t.Fatal(err) + } + fmt.Println("preview helper ready") + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = io.WriteString(w, "managed preview") + })} + if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { + t.Fatal(err) + } +} + +func TestManagerStartsIsolatedConfiguredServerAndStopsIt(t *testing.T) { + workspace := writeLaunchFile(t, []Configuration{helperConfiguration("web", TargetApp)}) + manager := New(slog.New(slog.NewTextHandler(io.Discard, nil))) + t.Cleanup(manager.Close) + + status, err := manager.Start(context.Background(), "ao-1", workspace, "") + if err != nil { + t.Fatalf("Start: %v\nstatus=%+v", err, status) + } + if status.State != StateReady || status.Configuration != "web" || status.TargetKind != TargetApp { + t.Fatalf("status = %+v, want ready web app", status) + } + if status.Port <= 0 || !strings.Contains(status.URL, strconv.Itoa(status.Port)) { + t.Fatalf("status URL/port = %q/%d", status.URL, status.Port) + } + resp, err := http.Get(status.URL) + if err != nil { + t.Fatalf("GET managed preview: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET status = %d", resp.StatusCode) + } + + stopped, err := manager.Stop(context.Background(), "ao-1") + if err != nil { + t.Fatalf("Stop: %v", err) + } + if stopped.State != StateStopped { + t.Fatalf("stopped state = %q", stopped.State) + } +} + +func TestManagerKeepsConcurrentSessionServersIsolated(t *testing.T) { + workspace := writeLaunchFile(t, []Configuration{helperConfiguration("web", TargetApp)}) + manager := New(slog.New(slog.NewTextHandler(io.Discard, nil))) + t.Cleanup(manager.Close) + + first, err := manager.Start(context.Background(), domain.SessionID("ao-1"), workspace, "") + if err != nil { + t.Fatal(err) + } + second, err := manager.Start(context.Background(), domain.SessionID("ao-2"), workspace, "") + if err != nil { + t.Fatal(err) + } + if first.Port == second.Port || first.URL == second.URL { + t.Fatalf("session previews collided: first=%+v second=%+v", first, second) + } + if manager.Status("ao-1").State != StateReady || manager.Status("ao-2").State != StateReady { + t.Fatalf("both session servers should remain ready") + } +} + +func TestManagerRequiresNameWhenConfigurationsAreAmbiguous(t *testing.T) { + workspace := writeLaunchFile(t, []Configuration{ + helperConfiguration("web", TargetApp), + helperConfiguration("api", TargetAPI), + }) + manager := New(nil) + t.Cleanup(manager.Close) + + _, err := manager.Start(context.Background(), "ao-1", workspace, "") + var serviceErr Error + if !errors.As(err, &serviceErr) || serviceErr.Code != "PREVIEW_CONFIGURATION_REQUIRED" { + t.Fatalf("error = %#v, want PREVIEW_CONFIGURATION_REQUIRED", err) + } + + status, err := manager.Start(context.Background(), "ao-1", workspace, "api") + if err != nil { + t.Fatalf("named Start: %v", err) + } + if status.TargetKind != TargetAPI { + t.Fatalf("targetKind = %q, want api", status.TargetKind) + } + _, _ = manager.Stop(context.Background(), "ao-1") +} + +func TestManagerRejectsMissingConfigAndNonLoopbackURL(t *testing.T) { + manager := New(nil) + t.Cleanup(manager.Close) + _, err := manager.Start(context.Background(), "ao-1", t.TempDir(), "") + assertPreviewErrorCode(t, err, "PREVIEW_CONFIG_NOT_FOUND") + + cfg := helperConfiguration("web", TargetApp) + cfg.URL = "https://example.com:${PORT}/" + workspace := writeLaunchFile(t, []Configuration{cfg}) + _, err = manager.Start(context.Background(), "ao-1", workspace, "") + assertPreviewErrorCode(t, err, "PREVIEW_CONFIG_INVALID") +} + +func helperConfiguration(name string, kind TargetKind) Configuration { + return Configuration{ + Name: name, + RuntimeExecutable: os.Args[0], + RuntimeArgs: []string{"-test.run=^TestPreviewServerHelper$"}, + Port: 0, + AutoPort: true, + URL: "http://127.0.0.1:${PORT}/", + TargetKind: kind, + Env: map[string]string{"AO_PREVIEW_TEST_HELPER": "1"}, + ReadyTimeoutMillis: 5000, + } +} + +func writeLaunchFile(t *testing.T, configurations []Configuration) string { + t.Helper() + workspace := t.TempDir() + dir := filepath.Join(workspace, ".ao") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + body, err := json.Marshal(launchFile{Version: 1, Configurations: configurations}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "launch.json"), body, 0o600); err != nil { + t.Fatal(err) + } + return workspace +} + +func assertPreviewErrorCode(t *testing.T, err error, code string) { + t.Helper() + var serviceErr Error + if !errors.As(err, &serviceErr) || serviceErr.Code != code { + t.Fatalf("error = %#v, want %s", err, code) + } +} diff --git a/backend/internal/previewserver/process_unix.go b/backend/internal/previewserver/process_unix.go new file mode 100644 index 0000000000..59a904f2d0 --- /dev/null +++ b/backend/internal/previewserver/process_unix.go @@ -0,0 +1,26 @@ +//go:build !windows + +package previewserver + +import ( + "errors" + "os/exec" + "syscall" +) + +func previewCommand(name string, args ...string) *exec.Cmd { + cmd := exec.Command(name, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + return cmd +} + +func terminatePreviewProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM) + if errors.Is(err, syscall.ESRCH) { + return nil + } + return err +} diff --git a/backend/internal/previewserver/process_windows.go b/backend/internal/previewserver/process_windows.go new file mode 100644 index 0000000000..e2806dfdc0 --- /dev/null +++ b/backend/internal/previewserver/process_windows.go @@ -0,0 +1,72 @@ +//go:build windows + +package previewserver + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/windows" + + aoprocess "github.com/aoagents/agent-orchestrator/backend/internal/process" +) + +func previewCommand(name string, args ...string) *exec.Cmd { + if resolved, err := exec.LookPath(name); err == nil && isWindowsBatchFile(resolved) { + shell := strings.TrimSpace(os.Getenv("COMSPEC")) + if shell == "" { + shell = "cmd.exe" + } + cmd := exec.Command(shell) + cmd.Args = nil + cmd.SysProcAttr = &syscall.SysProcAttr{ + CmdLine: `/d /s /c "` + windowsBatchCommandLine(resolved, args) + `"`, + CreationFlags: windows.CREATE_NO_WINDOW | windows.CREATE_NEW_PROCESS_GROUP, + HideWindow: true, + } + return cmd + } + cmd := exec.Command(name, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{ + CreationFlags: windows.CREATE_NO_WINDOW | windows.CREATE_NEW_PROCESS_GROUP, + HideWindow: true, + } + return cmd +} + +func isWindowsBatchFile(path string) bool { + extension := filepath.Ext(path) + return strings.EqualFold(extension, ".cmd") || strings.EqualFold(extension, ".bat") +} + +// cmd.exe uses different quoting rules from native Windows executables. The +// outer quotes are consumed by /s /c; each token remains quoted for paths and +// arguments containing spaces. Doubling embedded quotes preserves them for the +// batch shim instead of allowing them to terminate the command early. +func windowsBatchCommandLine(executable string, args []string) string { + parts := make([]string, 0, len(args)+1) + parts = append(parts, quoteWindowsBatchArg(executable)) + for _, arg := range args { + parts = append(parts, quoteWindowsBatchArg(arg)) + } + return strings.Join(parts, " ") +} + +func quoteWindowsBatchArg(value string) string { + return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` +} + +func terminatePreviewProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + err := aoprocess.Command("taskkill", "/PID", strconv.Itoa(cmd.Process.Pid), "/T", "/F").Run() + if err != nil { + return cmd.Process.Kill() + } + return nil +} diff --git a/backend/internal/previewserver/process_windows_test.go b/backend/internal/previewserver/process_windows_test.go new file mode 100644 index 0000000000..1162abd609 --- /dev/null +++ b/backend/internal/previewserver/process_windows_test.go @@ -0,0 +1,26 @@ +//go:build windows + +package previewserver + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPreviewCommandRunsWindowsBatchShim(t *testing.T) { + dir := t.TempDir() + shim := filepath.Join(dir, "preview helper.cmd") + if err := os.WriteFile(shim, []byte("@echo off\r\necho %~1\r\n"), 0o600); err != nil { + t.Fatal(err) + } + + output, err := previewCommand(shim, "argument with spaces").CombinedOutput() + if err != nil { + t.Fatalf("run batch shim: %v\n%s", err, output) + } + if got := strings.TrimSpace(string(output)); got != "argument with spaces" { + t.Fatalf("output = %q, want batch argument preserved", got) + } +} diff --git a/backend/internal/session_manager/manager.go b/backend/internal/session_manager/manager.go index f1dcc3f435..d0f5a23969 100644 --- a/backend/internal/session_manager/manager.go +++ b/backend/internal/session_manager/manager.go @@ -152,6 +152,7 @@ type Manager struct { // initial-prompt delivery, where a blocked session is impossible. messenger *sessionguard.Guard lcm lifecycleRecorder + preview ports.SessionPreviewLifecycle dataDir string clock func() time.Time // lookPath is exec.LookPath in production; tests substitute a stub so @@ -202,6 +203,7 @@ type Deps struct { Store Store Messenger ports.AgentMessenger Lifecycle lifecycleRecorder + Preview ports.SessionPreviewLifecycle // DataDir is exported to spawned agents as AO_DATA_DIR so their hook // commands can open the same store. DataDir string @@ -228,6 +230,7 @@ func New(d Deps) *Manager { workspace: d.Workspace, store: d.Store, lcm: d.Lifecycle, + preview: d.Preview, dataDir: d.DataDir, clock: d.Clock, lookPath: d.LookPath, @@ -639,6 +642,7 @@ func (m *Manager) Kill(ctx context.Context, id domain.SessionID) (bool, error) { if !ok { return false, nil // already gone: benign race } + m.stopPreviewBestEffort(ctx, id) handle := runtimeHandle(rec.Metadata) ws := workspaceInfo(rec) @@ -719,6 +723,7 @@ func (m *Manager) RetireForReplacement(ctx context.Context, id domain.SessionID) if !ok || rec.IsTerminated { return nil } + m.stopPreviewBestEffort(ctx, id) if rec.Metadata.WorkspacePath == "" || rec.Metadata.Branch == "" { if err := m.store.DeleteSessionWorktrees(ctx, rec.ID); err != nil { return fmt.Errorf("retire replacement %s: clear restore markers: %w", id, err) @@ -771,6 +776,15 @@ func (m *Manager) RetireForReplacement(ctx context.Context, id domain.SessionID) return nil } +func (m *Manager) stopPreviewBestEffort(ctx context.Context, id domain.SessionID) { + if m.preview == nil { + return + } + if err := m.preview.StopSession(ctx, id); err != nil { + m.logger.Warn("session preview cleanup failed", "sessionID", id, "error", err) + } +} + func (m *Manager) retireWorkspaceProjectForReplacement(ctx context.Context, rec domain.SessionRecord, rows []ports.WorkspaceRepoInfo) error { staleRepos := make(map[string]bool) for _, row := range rows { @@ -2009,12 +2023,14 @@ func (m *Manager) aoSkillPointer() string { skillFile := filepath.Join(dir, "SKILL.md") commandsGlob := filepath.Join(dir, "commands", "*.md") browserFile := filepath.Join(dir, "commands", "browser.md") + previewFile := filepath.Join(dir, "commands", "preview.md") return "\n\n" + "## Using the ao CLI\n\n" + - "When you need to use the `ao` CLI, read `" + skillFile + "` first (and the relevant `" + commandsGlob + "`) for the full command catalog, flags, and examples.\n\n" + + "When using `ao`, read `" + skillFile + "` and only the relevant file under `" + commandsGlob + "`; do not load unrelated command guides.\n\n" + "## AO desktop Browser panel\n\n" + - "When the user asks you to inspect, test, click, or type in the page shown in AO's desktop Browser panel, read `" + browserFile + "` and use `ao browser` from this AO session. " + + "For frontend work, read `" + previewFile + "` before previewing or starting an app: open static HTML or Markdown directly; Never create or modify `package.json` or install dependencies solely to display static files. Do not create `.ao/launch.json` unless the user asks. Automatically open the primary requested browser-displayable artifact immediately after creating or materially updating it, but do not replace an active application preview with a supporting asset. " + + "For page inspection or interaction, read `" + browserFile + "` and use `ao browser` from this AO session. Browser network capture is optional and off by default; follow that guide and never enable it for routine browser actions. " + "Do not use Codex/host in-app browser connectors, `agent.browsers.get(\"iab\")`, or a browser MCP for the AO Browser panel: those are separate browser runtimes and cannot see or control AO's session-owned page. " + - "`ao browser` deliberately operates the same live page the user sees in that panel." + "`ao browser` operates the same live page the user sees in that panel." } func (m *Manager) workspaceProjectPrompt(ctx context.Context, kind domain.SessionKind, projectID domain.ProjectID) (string, error) { diff --git a/backend/internal/session_manager/manager_test.go b/backend/internal/session_manager/manager_test.go index 6127d204b0..cc4a1f276f 100644 --- a/backend/internal/session_manager/manager_test.go +++ b/backend/internal/session_manager/manager_test.go @@ -174,6 +174,16 @@ type fakeRuntime struct { destroyedIDs []string } +type fakePreviewLifecycle struct { + stopped []domain.SessionID + err error +} + +func (f *fakePreviewLifecycle) StopSession(_ context.Context, id domain.SessionID) error { + f.stopped = append(f.stopped, id) + return f.err +} + func (r *fakeRuntime) Create(_ context.Context, cfg ports.RuntimeConfig) (ports.RuntimeHandle, error) { if r.createErr != nil { return ports.RuntimeHandle{}, r.createErr @@ -1343,6 +1353,8 @@ func TestSpawn_WorkspaceProjectRollsBackWhenWorktreeRowsFail(t *testing.T) { func TestKill_TearsDownRuntimeAndWorkspace(t *testing.T) { m, st, rt, ws := newManager() + preview := &fakePreviewLifecycle{} + m.preview = preview dataDir := t.TempDir() m.dataDir = dataDir st.sessions["mer-1"] = mkLive("mer-1") @@ -1356,6 +1368,9 @@ func TestKill_TearsDownRuntimeAndWorkspace(t *testing.T) { if rt.destroyed != 1 || ws.destroyed != 1 { t.Fatal("kill should destroy runtime and workspace") } + if !reflect.DeepEqual(preview.stopped, []domain.SessionID{"mer-1"}) { + t.Fatalf("preview stops = %v, want [mer-1]", preview.stopped) + } requireNoPromptDir(t, dataDir, "mer-1") } @@ -2140,11 +2155,16 @@ func TestSpawnOrchestrator_UsesCoordinatorPrompt(t *testing.T) { "AO desktop Browser panel", "agent.browsers.get(\"iab\")", "same live page the user sees", + "Browser network capture is optional and off by default", + "never enable it for routine browser actions", } { if !strings.Contains(systemPrompt, want) { t.Fatalf("system prompt missing %q:\n%s", want, systemPrompt) } } + if words := len(strings.Fields(m.aoSkillPointer())); words > 170 { + t.Fatalf("always-on AO skill pointer grew to %d words; keep details in routed command guides:\n%s", words, m.aoSkillPointer()) + } if strings.Contains(agent.lastLaunch.Prompt, "You are the human-facing orchestrator") { t.Fatalf("coordinator role must not be in the user prompt:\n%s", agent.lastLaunch.Prompt) } @@ -2292,6 +2312,15 @@ func TestSystemPrompt_AppendsConfidentialityGuard(t *testing.T) { if !strings.Contains(sp, "AO desktop Browser panel") || !strings.Contains(sp, "agent.browsers.get(\"iab\")") { t.Fatalf("%s: system prompt missing AO browser routing guidance:\n%s", tc.name, sp) } + if !strings.Contains(sp, "open static HTML or Markdown directly") || + !strings.Contains(sp, "Never create or modify `package.json`") || + !strings.Contains(sp, "Do not create `.ao/launch.json` unless the user asks") { + t.Fatalf("%s: system prompt missing static-first preview safeguards:\n%s", tc.name, sp) + } + if !strings.Contains(sp, "immediately after creating or materially updating it") || + !strings.Contains(sp, "do not replace an active application preview with a supporting asset") { + t.Fatalf("%s: system prompt missing automatic artifact handoff guidance:\n%s", tc.name, sp) + } }) } } diff --git a/backend/internal/skillassets/skillassets_test.go b/backend/internal/skillassets/skillassets_test.go index 211bb58dd8..852b6eaf73 100644 --- a/backend/internal/skillassets/skillassets_test.go +++ b/backend/internal/skillassets/skillassets_test.go @@ -39,6 +39,60 @@ func TestEmbeddedSkillFrontmatterIsValidYAML(t *testing.T) { } } +func TestEmbeddedPreviewGuidanceDoesNotScaffoldStaticSites(t *testing.T) { + previewBody, err := files.ReadFile("using-ao/commands/preview.md") + if err != nil { + t.Fatalf("read embedded preview guidance: %v", err) + } + previewText := string(previewBody) + normalizedPreviewText := strings.Join(strings.Fields(previewText), " ") + for _, required := range []string{ + "Static HTML and Markdown do not need a development server", + "Never create or modify `package.json`", + "Do not create `.ao/launch.json` unless the user asks", + "reuse the repository's existing dev command", + "without waiting for a separate \"open it\" request", + "Do not steal the browser from an active application", + "open the primary requested artifact", + } { + if !strings.Contains(normalizedPreviewText, required) { + t.Fatalf("preview guidance missing %q:\n%s", required, previewText) + } + } + + skillBody, err := files.ReadFile("using-ao/SKILL.md") + if err != nil { + t.Fatalf("read embedded SKILL.md: %v", err) + } + skillText := string(skillBody) + normalizedSkillText := strings.Join(strings.Fields(skillText), " ") + if !strings.Contains(normalizedSkillText, "[commands/preview.md](commands/preview.md) before acting") || + !strings.Contains(normalizedSkillText, "automatic-handoff rules are load-bearing") || + !strings.Contains(normalizedSkillText, "[commands/browser.md](commands/browser.md)") || + !strings.Contains(normalizedSkillText, "opt-in network policy") { + t.Fatalf("skill catalog is missing focused preview/browser routing:\n%s", skillText) + } +} + +func TestEmbeddedBrowserGuidanceKeepsNetworkCaptureOptional(t *testing.T) { + body, err := files.ReadFile("using-ao/commands/browser.md") + if err != nil { + t.Fatalf("read embedded browser guidance: %v", err) + } + text := strings.Join(strings.Fields(string(body)), " ") + for _, required := range []string{ + "Network capture is optional and disabled by default", + "Do not enable it for routine navigation or interaction", + "no request or response bodies, credentials, cookies, or query values", + "`network status` and `network list` never enable capture", + "The user can select or close these same tabs", + } { + if !strings.Contains(text, required) { + t.Fatalf("browser guidance missing %q:\n%s", required, body) + } + } +} + // TestInstall_WritesSkillAndIsIdempotent: Install must lay down the embedded // skill (SKILL.md plus a commands file) under /skills/using-ao, and a // second run must clobber cleanly, leaving no stale files. This is the whole diff --git a/backend/internal/skillassets/using-ao/SKILL.md b/backend/internal/skillassets/using-ao/SKILL.md index 4717f3029b..042a29955e 100644 --- a/backend/internal/skillassets/using-ao/SKILL.md +++ b/backend/internal/skillassets/using-ao/SKILL.md @@ -16,7 +16,7 @@ trigger: "Using the ao CLI in an AO workspace: spawning workers, managing sessio | `orchestrator` | List orchestrator sessions | Viewing which sessions are orchestrators | [commands/orchestrator.md](commands/orchestrator.md) | | `review` | Submit a reviewer result for a worker's PR | Completing a code review loop | [commands/review.md](commands/review.md) | | `send` | Send a message to a running agent session | Correcting or directing a live agent | [commands/send.md](commands/send.md) | -| `preview` | Open a URL in the desktop browser panel | Demoing a local server or file from inside a session | [commands/preview.md](commands/preview.md) | +| `preview` | Start a session-owned app or open an exact URL/file | Running and showing the worker's relevant app, Markdown, HTML, PDF, or image | [commands/preview.md](commands/preview.md) | | `browser` | Inspect and control the session's shared live browser | Verifying a web app through snapshots, interactions, waits, screenshots, console, and errors | [commands/browser.md](commands/browser.md) | | `start` | Fetch (if needed) and open the AO desktop app | Launching the app | [commands/start.md](commands/start.md) | | `stop` | Stop the AO daemon | Shutting down AO | [commands/stop.md](commands/stop.md) | @@ -33,6 +33,12 @@ trigger: "Using the ao CLI in an AO workspace: spawning workers, managing sessio - Session and project ids are shown by `ao session ls` and `ao project ls`. - `--agent` is an alias for `--harness` on `ao spawn`. - Every command accepts `-h / --help` for the full flag list. -- For frontend work, prefer `ao browser`: check status, open the app, snapshot, interact, wait for the update, snapshot again, and inspect `ao browser errors` before completing. +- For frontend launch, preview selection, or artifact handoff, read + [commands/preview.md](commands/preview.md) before acting. Its static-file, + project-runtime, and automatic-handoff rules are load-bearing. +- For page inspection, interaction, or request diagnosis, read + [commands/browser.md](commands/browser.md). It defines shared-tab behavior + and the opt-in network policy. -See [references.md](references.md) for natural-language-to-command mappings. +Use [references.md](references.md) only when a natural-language request does +not map clearly to a command above. diff --git a/backend/internal/skillassets/using-ao/commands/browser.md b/backend/internal/skillassets/using-ao/commands/browser.md index cf830284a2..29b0030e26 100644 --- a/backend/internal/skillassets/using-ao/commands/browser.md +++ b/backend/internal/skillassets/using-ao/commands/browser.md @@ -8,6 +8,10 @@ This is the automation interface for AO's visible desktop Browser panel. Do not ## Core workflow +If the task first requires choosing, starting, or opening a preview target, +read [preview.md](preview.md) and follow its static-file/project-runtime +decision. Once the relevant page is known: + ```bash ao browser status ao browser open http://localhost:5173 @@ -47,6 +51,11 @@ ao browser uncheck [--json] ao browser get [ref] [--json] ao browser wait (--text | --text-gone | --selector | --selector-gone | --url | --load | --dom-stable | --ms ) [--timeout ] [--json] ao browser screenshot [path] [--json] +ao browser network start [--duration ] [--json] +ao browser network status [--json] +ao browser network list [--json] +ao browser network stop [--json] +ao browser network clear [--json] ao browser console [--json] ao browser errors [--json] ``` @@ -62,12 +71,25 @@ cursor position. `press` accepts named keys and chords such as `Enter`, following browser commands, and `tab close` defaults to the active tab. Allowed page popups are captured as new AO tabs instead of opening a separate OS browser. Take a new snapshot after switching tabs because element refs are -invalidated at the tab boundary. +invalidated at the tab boundary. The user can select or close these same tabs +from the compact tab control in the Browser toolbar; the next agent command +uses whichever tab the user selected. Use `wait --load` after navigation, `--text-gone` or `--selector-gone` for transient UI, and `--dom-stable ` after HMR or a dynamic render. Conditional waits retry through brief execution-context replacement during navigation and fail with `WAIT_TIMEOUT` when `--timeout` expires. +Network capture is optional and disabled by default. Use it only when the user +explicitly asks to inspect requests, or when diagnosing loading, API, CORS, +authentication, caching, or redirect failures after snapshots, console +messages, and page errors are insufficient. Do not enable it for routine +navigation or interaction. `network start` captures only the active tab for 60 +seconds by default (maximum 300), retains at most 200 in-memory entries, and +stops automatically. It records sanitized request metadata only: no request or +response bodies, credentials, cookies, or query values. `network status` and +`network list` never enable capture. Use `network stop` as soon as the relevant +failure is reproduced, and `network clear` to discard retained entries. + Without `--json`, `screenshot` writes a PNG and refuses to overwrite an existing file. With `--json`, it returns the structured response including base64 image data. `ao preview` remains available for the passive URL/static-file workflow. Use `ao browser` when the agent needs to inspect or verify the page. diff --git a/backend/internal/skillassets/using-ao/commands/preview.md b/backend/internal/skillassets/using-ao/commands/preview.md index e2de8d5865..6f3780de9e 100644 --- a/backend/internal/skillassets/using-ao/commands/preview.md +++ b/backend/internal/skillassets/using-ao/commands/preview.md @@ -1,6 +1,42 @@ # ao preview -Open a URL in the desktop browser panel for the current session. With no argument it opens the workspace's static entry point, falling back to this session's existing preview target when no entry point exists. A local file can be opened by its absolute `file://` URL. Use `ao preview clear` to empty the panel. +Open a URL or workspace file in the desktop browser panel for the current +session, or start a deterministic session-owned dev server from +an existing `.ao/launch.json`. + +Static HTML and Markdown do not need a development server. Open them directly +with `ao preview `. Never create or modify `package.json`, +install dependencies, or introduce npm or another server solely to display +static files. + +Use a managed server only when the project genuinely has a runtime. Start an +existing `.ao/launch.json` configuration when present. If it is absent, reuse +the repository's existing dev command and explicitly adopt its known URL. +Do not create `.ao/launch.json` unless the user asks for reusable launch +configuration. + +## Automatic artifact handoff + +When a browser-displayable file is itself the artifact the user requested, +open it immediately after creating or materially updating it: + +```bash +ao preview docs/plan.md +ao preview report.html +ao preview output.pdf +ao preview diagram.svg +ao preview mockup.png +``` + +Do this without waiting for a separate "open it" request. Browser-displayable +artifacts include Markdown, HTML, PDF, SVG, and common image formats such as +PNG, JPEG, GIF, WebP, and AVIF. If the task produces several files, open the +primary requested artifact rather than cycling through every output. + +Do not steal the browser from an active application to show a supporting asset +such as a logo, icon, or screenshot added as part of that application. Verify +the application itself instead. Also honor an explicit request not to open or +preview the artifact. ## Syntax @@ -17,9 +53,55 @@ No flags beyond `-h / --help`. --- +### ao preview start + +Start a named configuration from `.ao/launch.json`, wait for its loopback URL, +and open it in this worker's Browser panel. The name is optional when exactly +one configuration exists. + +```bash +ao preview start [configuration] [--json] +ao preview status [--json] +ao preview stop [--json] +``` + +This command is for an existing, intentional project configuration. Do not +create the file as routine preview setup. Do not scan unrelated ports. +`${PORT}` is expanded in `runtimeArgs`, `url`, and `env`; AO also sets `PORT`, +`AO_PREVIEW_PORT`, and `AO_SESSION_ID`. + +```json +{ + "version": 1, + "configurations": [ + { + "name": "web", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "dev", "--", "--host", "127.0.0.1", "--port", "${PORT}"], + "cwd": ".", + "port": 5173, + "autoPort": true, + "url": "http://127.0.0.1:${PORT}/", + "targetKind": "app" + } + ] +} +``` + +Use `targetKind: "api"` for a backend that should be health-checked without +taking over the visible browser. When several configurations exist, select the +one relevant to the user's request by name. If the agent starts a server +outside this lifecycle, explicitly adopt its known URL with `ao preview `; +terminal URLs are not automatically ranked or selected. + +--- + ### ao preview (bare form) Open the workspace's static entry point, or the session's existing preview target. +This is the default for a plain static site: an `index.html` is discovered and +served through AO's isolated workspace preview without adding a project +runtime. **Examples:** @@ -35,8 +117,10 @@ ao preview http://localhost:5173 ``` ```bash -# Open a local HTML file -ao preview file://$(pwd)/index.html +# Open an exact workspace file (Markdown is rendered to HTML) +ao preview README.md +ao preview docs/guide.md +ao preview index.html ``` --- @@ -60,3 +144,6 @@ No flags beyond `-h / --help`. # Clear the preview panel ao preview clear ``` + +Stopping a managed server clears the panel only when it is still displaying +that server. A file explicitly opened afterward is preserved. diff --git a/backend/internal/skillassets/using-ao/references.md b/backend/internal/skillassets/using-ao/references.md index fe7515cd44..cac931167d 100644 --- a/backend/internal/skillassets/using-ao/references.md +++ b/backend/internal/skillassets/using-ao/references.md @@ -5,9 +5,15 @@ Natural-language-to-command mappings for common AO tasks. | You want to... | Command | |---|---| | Show me this webpage / open this page | `ao preview ""` | +| Start an existing configured dev app | `ao preview start [configuration]` | +| Check or stop the worker's managed dev app | `ao preview status` / `ao preview stop` | +| Show this Markdown or HTML file without a server | `ao preview ""` | +| Hand off a newly created browser-displayable artifact | `ao preview ""` immediately after writing the primary artifact | | Inspect and verify this webpage as the agent | `ao browser open ""`, then `ao browser snapshot` | | Click or fill a page element | `ao browser snapshot`, then `ao browser click ` or `ao browser fill ""` | | Check frontend runtime failures | `ao browser errors` and `ao browser console` | +| Diagnose a request/API/CORS/auth/redirect failure when normal page evidence is insufficient | `ao browser network start`, reproduce once, then `ao browser network stop` | +| Check network capture without enabling it | `ao browser network status` or `ao browser network list` | | Capture the page | `ao browser screenshot [path]` | | Spawn a worker on issue N | `ao spawn --project

--issue N --name "<=20 chars>" --prompt "..."` | | Message a running agent | `ao send --session --message "..."` | diff --git a/docs/STATUS.md b/docs/STATUS.md index 2b92678124..783c52560f 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -66,11 +66,16 @@ surface (`npm run sqlc`, `npm run api`). through Electron's bound debugger transport. `ao browser` supports open, compact accessibility snapshots and refs, click/fill/type, keyboard input, hover and non-mutating element highlighting, scrolling, selection and checked - state, property reads, stable logical tabs and captured popups, waits, + state, property reads, stable logical tabs and captured popups, a compact + user-facing tab selector for switching/closing tabs and popup notices, waits, including load/disappearance/DOM-stability conditions, screenshots, console - messages, and page errors while the Browser panel is hidden. Tabs within one - worker share an ephemeral Electron profile; different workers have isolated - cookies and web storage. + messages, page errors, and explicit temporary network-metadata capture while + the Browser panel is hidden. Network capture is off by default, tab-scoped, + bounded, automatically expires, and omits bodies and sensitive values. Tabs + within one worker share an ephemeral Electron profile; different workers + have isolated cookies and web storage. The toolbar activity signal is scoped + to actual agent browser commands; annotation progress is separate and its + successful-delivery confirmation clears automatically. - Real daemon wiring via the generated `openapi-fetch` typed client (`src/api/schema.ts`); mock data only in `VITE_NO_ELECTRON` web-preview mode. - Electron main handles daemon discovery, launch, and status reporting. diff --git a/docs/architecture.md b/docs/architecture.md index 0dc4f59ab8..b73f6edf5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -846,6 +846,14 @@ Electron attaches its debugger directly to the selected session's AO renderer or a different session. The loopback `/api/v1/browser` surface is blocked entirely on the opt-in LAN listener. +Request observation is an explicit, temporary browser command rather than a +standing debugger feature. Capture is off by default, bound to the active tab +that starts it, limited to 200 in-memory metadata entries, and automatically +expires within at most five minutes. AO never requests or stores request or +response bodies; it allowlists safe headers and redacts URL credentials, +fragments, and query values. Closing the tab, ending the session, or shutting +down Electron disables and discards the capture. + --- ## Load-Bearing Rules diff --git a/docs/cli/README.md b/docs/cli/README.md index d1854fcb66..70285bbc4b 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -56,6 +56,7 @@ Every product command resolves to a daemon HTTP route. Run `ao | `ao orchestrator ls` | `GET /api/v1/orchestrators` | | `ao send` | `POST /api/v1/sessions/{id}/send` | | `ao preview [url]` | `POST /api/v1/sessions/{id}/preview` | +| `ao preview start/status/stop` | `POST/GET/DELETE /api/v1/sessions/{id}/preview/server` | | `ao browser ...` | `GET /api/v1/browser/status`, `POST /api/v1/browser/commands` | | `ao hooks ` | `POST /api/v1/sessions/{id}/activity` (hidden) | @@ -81,15 +82,31 @@ spawn remains the authoritative runtime validation point. Use autodetects an `index.html` in the session workspace; with a URL argument it opens that URL verbatim (`file://`, `http`, `https`). +`ao preview start [configuration]` loads `.ao/launch.json` from the session +workspace, starts that exact command under a session-owned supervisor, selects +or records its loopback port, waits for readiness, and opens application +targets in the Browser panel. `status` reports bounded recent logs and `stop` +terminates the managed process tree. Multiple configurations must be selected +by name; AO does not assign confidence scores to arbitrary localhost servers. +This is an optional, reusable project configuration, not a prerequisite for +preview. Agents must not create it automatically. Static HTML and Markdown use +the direct file preview and must not cause package-manager scaffolding, +dependency installation, or a development server to be introduced. + +When a browser-displayable file is the requested artifact, agents should call +`ao preview ` immediately after creating or materially updating +the primary output. Markdown, HTML, PDF, SVG, and common images can be served +directly. Supporting assets must not replace an active application preview. + `ao browser` also resolves its target from `AO_SESSION_ID`, but controls the session-owned live Electron browser rather than only setting its preview URL. The target-isolated command set includes `status`, `open`, `snapshot`, `click`, `fill`, `type`, `press`, `hover`, `scroll`, `select`, `check`, `uncheck`, `get`, `highlight`, `unhighlight`, `tabs`, `tab new`, `tab select`, `tab close`, -`wait`, `screenshot`, `console`, and `errors`. Logical tab IDs remain stable -for the session, and allowed popups become AO browser tabs rather than separate -OS-browser windows. The AO desktop app must be open because Electron owns the -`WebContentsView`. +`wait`, `screenshot`, `network start/status/list/stop/clear`, `console`, and +`errors`. Logical tab IDs remain stable for the session, and allowed popups +become AO browser tabs rather than separate OS-browser windows. The AO desktop +app must be open because Electron owns the `WebContentsView`. References from a snapshot are invalidated after navigation or DOM replacement; they are also invalidated when changing tabs. Take another snapshot when a command reports `STALE_REFERENCE`. @@ -99,6 +116,12 @@ window for HMR-driven verification. Browser tabs in the same worker share a memory-only Electron profile. Different workers receive distinct partitions, so cookies, authentication, local storage, and session storage do not leak between their browser runtimes. +Network capture is disabled by default and must be started explicitly. It is +scoped to the active tab at start time, expires after 60 seconds by default +(maximum 300), retains at most 200 in-memory entries, and is cleared with the +tab/session. Captured data is metadata-only: request and response bodies are +never read, sensitive headers are omitted, and URL credentials, fragments, and +query values are redacted. `go run .` in `backend/` remains a compatibility wrapper around the daemon. diff --git a/frontend/src/api/schema.ts b/frontend/src/api/schema.ts index c30d3f46db..1deec37f87 100644 --- a/frontend/src/api/schema.ts +++ b/frontend/src/api/schema.ts @@ -590,6 +590,25 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/sessions/{sessionId}/preview/server": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get the managed preview server status for a session */ + get: operations["getSessionPreviewServer"]; + put?: never; + /** Start a session-owned server from .ao/launch.json and open its application preview */ + post: operations["startSessionPreviewServer"]; + /** Stop the managed preview server for a session */ + delete: operations["stopSessionPreviewServer"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/sessions/{sessionId}/restore": { parameters: { query?: never; @@ -1011,6 +1030,20 @@ export interface components { targetSha: string; title: string; }; + PreviewServerStatusResponse: { + configuration?: string; + error?: string; + logs: string[]; + port?: number; + sessionId: string; + /** Format: date-time */ + startedAt?: string; + /** @enum {string} */ + state: "stopped" | "starting" | "ready" | "stopping" | "failed"; + /** @enum {string} */ + targetKind?: "app" | "api"; + url?: string; + }; ProbeAgentResponse: { agent: components["schemas"]["AgentInfo"]; installed: boolean; @@ -1286,6 +1319,10 @@ export interface components { projectId: string; prompt?: string; }; + StartPreviewServerRequest: { + /** @description Named preview configuration. Optional when exactly one configuration exists. */ + configuration?: string; + }; SubmitReviewInput: { /** @description Review body recorded by AO. Required for changes_requested. */ body?: string; @@ -3415,6 +3452,205 @@ export interface operations { }; }; }; + getSessionPreviewServer: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Session identifier, e.g. project-1. */ + sessionId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreviewServerStatusResponse"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Implemented */ + 501: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + }; + }; + startSessionPreviewServer: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Session identifier, e.g. project-1. */ + sessionId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["StartPreviewServerRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreviewServerStatusResponse"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Request Timeout */ + 408: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Conflict */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Implemented */ + 501: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Gateway Timeout */ + 504: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + }; + }; + stopSessionPreviewServer: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Session identifier, e.g. project-1. */ + sessionId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreviewServerStatusResponse"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + /** @description Not Implemented */ + 501: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["APIError"]; + }; + }; + }; + }; restoreSession: { parameters: { query?: never; diff --git a/frontend/src/main/browser-view-host.test.ts b/frontend/src/main/browser-view-host.test.ts index f57c342398..60f12b631a 100644 --- a/frontend/src/main/browser-view-host.test.ts +++ b/frontend/src/main/browser-view-host.test.ts @@ -155,6 +155,8 @@ function setupHost() { function setupTabHost() { const constructorOptions: Array<{ webPreferences: { partition?: string } }> = []; + const handlers = new Map(); + const sent: Array<{ channel: string; payload: unknown }> = []; const views: Array<{ webContents: { id: number; @@ -239,10 +241,14 @@ function setupTabHost() { mainWindow: { contentView: { addChildView: () => undefined, removeChildView: () => undefined }, getContentBounds: () => ({ x: 0, y: 0, width: 800, height: 600 }), - webContents: { id: 1, focus: () => undefined, send: () => undefined }, + webContents: { + id: 1, + focus: () => undefined, + send: (channel: string, payload: unknown) => sent.push({ channel, payload }), + }, } as never, ipcMain: { - handle: () => undefined, + handle: (channel: string, fn: InvokeHandler) => handlers.set(channel, fn), on: () => undefined, removeHandler: () => undefined, off: () => undefined, @@ -255,7 +261,9 @@ function setupTabHost() { annotatePreloadPath: "/preload.js", rendererOrigin: "http://localhost:5173", }); - return { constructorOptions, host, views }; + const invoke = (channel: string, ...args: unknown[]) => + handlers.get(channel)!({ sender: { id: 1 } }, ...args) as Promise; + return { constructorOptions, host, invoke, sent, views }; } describe("new-session shortcut forwarding", () => { @@ -615,6 +623,44 @@ describe("agent browser runtime", () => { expect(errors.messages.map((entry) => entry.message)).toEqual(["boom"]); }); + it("reports agent activity only while a browser command is executing", async () => { + const { debuggerSendCommand, host, sent } = setupHost(); + let resolveSnapshot: (value: unknown) => void = () => undefined; + debuggerSendCommand.mockImplementation((method: string) => { + if (method === "Accessibility.getFullAXTree") { + return new Promise((resolve) => { + resolveSnapshot = resolve; + }); + } + return Promise.resolve({}); + }); + + const pendingSnapshot = host.execute("sess-1", "snapshot"); + await vi.waitFor(() => + expect(sent).toContainEqual({ + channel: "browser:agentActivity", + payload: { + viewId: "0:sess-1", + active: true, + action: "snapshot", + }, + }), + ); + await vi.waitFor(() => expect(debuggerSendCommand).toHaveBeenCalledWith("Accessibility.getFullAXTree")); + + resolveSnapshot({ nodes: [] }); + await pendingSnapshot; + + expect( + sent + .filter(({ channel }) => channel === "browser:agentActivity") + .map(({ payload }) => payload), + ).toEqual([ + { viewId: "0:sess-1", active: true, action: "snapshot" }, + { viewId: "0:sess-1", active: false, action: "snapshot" }, + ]); + }); + it("keeps stable logical tab IDs, separate targets, and the selected tab active", async () => { const { host, views } = setupTabHost(); await host.execute("sess-1", "open", { url: "http://localhost:3000" }); @@ -685,6 +731,189 @@ describe("agent browser runtime", () => { code: "CANNOT_CLOSE_LAST_TAB", }); }); + + it("exposes owned tab state and manual tab actions to the renderer", async () => { + const { invoke, sent, views } = setupTabHost(); + const ensured = (await invoke("browser:ensure", "sess-1")) as BrowserNavState; + + views[0].webContents.openWindow("http://localhost:3000/popup"); + await vi.waitFor(async () => { + const state = (await invoke("browser:getTabs", ensured.viewId)) as { + activeTabId: string; + tabs: Array<{ id: string }>; + }; + expect(state.tabs).toHaveLength(2); + expect(state.activeTabId).toBe("t2"); + }); + expect(sent).toContainEqual({ + channel: "browser:tabsState", + payload: expect.objectContaining({ + viewId: ensured.viewId, + change: { kind: "popup", tabId: "t2" }, + }), + }); + + const selected = (await invoke("browser:selectTab", { + viewId: ensured.viewId, + tabId: "t1", + })) as { activeTabId: string }; + expect(selected.activeTabId).toBe("t1"); + + const closed = (await invoke("browser:closeTab", { + viewId: ensured.viewId, + tabId: "t2", + })) as { tabs: Array<{ id: string }> }; + expect(closed.tabs.map((tab) => tab.id)).toEqual(["t1"]); + expect(views[1].webContents.close).toHaveBeenCalled(); + }); +}); + +describe("agent browser network capture", () => { + it("is opt-in and exposes only sanitized request metadata", async () => { + const { debuggerListeners, debuggerSendCommand, host } = setupHost(); + const emitDebuggerMessage = (method: string, params: Record) => + debuggerListeners.get("message")?.({} as never, method as never, params as never); + + emitDebuggerMessage("Network.requestWillBeSent", { + requestId: "before-start", + request: { method: "GET", url: "https://example.test/unobserved" }, + }); + expect(await host.execute("sess-1", "network-list")).toMatchObject({ + active: false, + requestCount: 0, + requests: [], + }); + + expect(await host.execute("sess-1", "network-start", { durationSeconds: 30 })).toMatchObject({ + active: true, + metadataOnly: true, + tabId: "t1", + requestCount: 0, + maxEntries: 200, + }); + expect(debuggerSendCommand).toHaveBeenCalledWith("Network.enable"); + + emitDebuggerMessage("Network.requestWillBeSent", { + requestId: "request-1", + timestamp: 12, + wallTime: 1_750_000_000, + type: "XHR", + request: { + method: "POST", + url: "https://user:password@api.example.test/items?token=secret&page=2#private", + postData: "must-not-be-stored", + headers: { + Authorization: "Bearer very-secret", + Cookie: "session=very-secret", + "Content-Type": "application/json", + Origin: "https://app.example.test", + }, + }, + }); + emitDebuggerMessage("Network.responseReceived", { + requestId: "request-1", + response: { + status: 401, + statusText: "Unauthorized", + mimeType: "application/json", + headers: { + "Set-Cookie": "session=server-secret", + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "https://app.example.test", + }, + }, + }); + emitDebuggerMessage("Network.loadingFinished", { requestId: "request-1", timestamp: 12.125 }); + + const result = (await host.execute("sess-1", "network-list")) as { + requestCount: number; + requests: Array>; + }; + expect(result.requestCount).toBe(1); + expect(result.requests[0]).toMatchObject({ + id: "n1", + method: "POST", + resourceType: "xhr", + status: 401, + durationMs: 125, + requestHeaders: { + "content-type": "application/json", + origin: "https://app.example.test", + }, + responseHeaders: { + "content-type": "application/json", + "access-control-allow-origin": "https://app.example.test", + }, + }); + expect(JSON.stringify(result)).not.toContain("very-secret"); + expect(JSON.stringify(result)).not.toContain("must-not-be-stored"); + expect(JSON.stringify(result)).not.toContain("password"); + expect(result.requests[0]?.url).toContain("token=%5Bredacted%5D"); + expect(result.requests[0]).not.toHaveProperty("protocolRequestId"); + expect(result.requests[0]).not.toHaveProperty("startedMonotonic"); + + expect(await host.execute("sess-1", "network-stop")).toMatchObject({ + active: false, + stopReason: "stopped", + requestCount: 1, + }); + expect(debuggerSendCommand).toHaveBeenCalledWith("Network.disable"); + }); + + it("retains only the newest 200 requests and validates the capture duration", async () => { + const { debuggerListeners, host } = setupHost(); + await expect(host.execute("sess-1", "network-start", { durationSeconds: 0 })).rejects.toMatchObject({ + code: "INVALID_ARGUMENT", + }); + await expect(host.execute("sess-1", "network-start", { durationSeconds: 301 })).rejects.toMatchObject({ + code: "INVALID_ARGUMENT", + }); + + await host.execute("sess-1", "network-start", { durationSeconds: 30 }); + const emitDebuggerMessage = debuggerListeners.get("message")!; + for (let index = 0; index < 205; index++) { + emitDebuggerMessage( + {} as never, + "Network.requestWillBeSent" as never, + { + requestId: `request-${index}`, + request: { + method: "GET", + url: `https://api.example.test/items?index=${index}`, + }, + } as never, + ); + } + + const result = (await host.execute("sess-1", "network-list")) as { + requestCount: number; + requests: Array<{ id: string }>; + }; + expect(result.requestCount).toBe(200); + expect(result.requests[0]?.id).toBe("n6"); + expect(result.requests.at(-1)?.id).toBe("n205"); + await host.execute("sess-1", "network-stop"); + }); + + it("expires automatically without enabling status or list checks", async () => { + vi.useFakeTimers(); + try { + const { debuggerSendCommand, host } = setupHost(); + expect(await host.execute("sess-1", "network-status")).toMatchObject({ active: false }); + expect(debuggerSendCommand).not.toHaveBeenCalledWith("Network.enable"); + + await host.execute("sess-1", "network-start", { durationSeconds: 1 }); + await vi.advanceTimersByTimeAsync(1_000); + + expect(await host.execute("sess-1", "network-status")).toMatchObject({ + active: false, + stopReason: "expired", + }); + expect(debuggerSendCommand).toHaveBeenCalledWith("Network.disable"); + } finally { + vi.useRealTimers(); + } + }); }); describe("browser:requestMirror", () => { diff --git a/frontend/src/main/browser-view-host.ts b/frontend/src/main/browser-view-host.ts index 70acbf2c54..fc97a9e269 100644 --- a/frontend/src/main/browser-view-host.ts +++ b/frontend/src/main/browser-view-host.ts @@ -30,6 +30,29 @@ export type BrowserNavState = { error?: string; }; +export type BrowserTabState = { + id: string; + url: string; + title: string; + active: boolean; +}; + +export type BrowserTabsState = { + viewId: string; + activeTabId: string; + tabs: BrowserTabState[]; + change?: { + kind: "opened" | "popup" | "selected" | "closed"; + tabId: string; + }; +}; + +export type BrowserAgentActivityState = { + viewId: string; + active: boolean; + action: string; +}; + type BrowserBoundsInput = { viewId: string; rect: BrowserRect; @@ -42,6 +65,11 @@ type BrowserNavigateInput = { url: string; }; +type BrowserTabInput = { + viewId: string; + tabId: string; +}; + type BrowserWebContents = Pick< WebContents, | "id" @@ -123,6 +151,7 @@ type BrowserEntry = { refs: Map; consoleMessages: BrowserLogEntry[]; errors: BrowserLogEntry[]; + networkCapture?: BrowserNetworkCapture; }; type BrowserSessionEntry = { @@ -135,6 +164,8 @@ type BrowserSessionEntry = { bounds: BrowserRect; visible: boolean; parked: boolean; + networkTabId?: string; + agentBrowserCommands: number; }; type BrowserLogEntry = { @@ -145,7 +176,49 @@ type BrowserLogEntry = { timestamp: string; }; +type BrowserNetworkRequest = { + id: string; + method: string; + url: string; + resourceType?: string; + startedAt: string; + status?: number; + statusText?: string; + mimeType?: string; + durationMs?: number; + failed?: boolean; + canceled?: boolean; + errorText?: string; + fromCache?: boolean; + fromServiceWorker?: boolean; + redirectedTo?: string; + requestHeaders?: Record; + responseHeaders?: Record; +}; + +type InternalBrowserNetworkRequest = BrowserNetworkRequest & { + protocolRequestId: string; + startedMonotonic?: number; +}; + +type BrowserNetworkCapture = { + active: boolean; + tabId: string; + startedAt: string; + expiresAt: string; + stoppedAt?: string; + stopReason?: string; + maxEntries: number; + nextSequence: number; + requests: InternalBrowserNetworkRequest[]; + byRequestId: Map; + timer?: ReturnType; +}; + const OFFSCREEN_BOUNDS: BrowserRect = { x: -10_000, y: -10_000, width: 0, height: 0 }; +const DEFAULT_NETWORK_CAPTURE_SECONDS = 60; +const MAX_NETWORK_CAPTURE_SECONDS = 300; +const MAX_NETWORK_REQUESTS = 200; // ponytail: file:// allowed unsanitized; preview targets are agent-trusted for now const ALLOWED_PROTOCOLS = new Set(["http:", "https:", "file:"]); @@ -215,6 +288,14 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV const forgetIfFocused = (viewId: string): void => { if (lastFocusedViewId === viewId) lastFocusedViewId = null; }; + const setAgentBrowserActivity = (session: BrowserSessionEntry, action: string, active: boolean): void => { + session.agentBrowserCommands = Math.max(0, session.agentBrowserCommands + (active ? 1 : -1)); + options.mainWindow.webContents.send("browser:agentActivity", { + viewId: session.viewId, + active: session.agentBrowserCommands > 0, + action, + } satisfies BrowserAgentActivityState); + }; let pendingMirror: { viewId: string; expires: number; frame: WebFrameMain } | null = null; const sameFrame = (a: WebFrameMain, b: WebFrameMain | null | undefined): boolean => @@ -279,7 +360,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV session.tabs.set(tabId, entry); tabsByWebContentsId.set(view.webContents.id, entry); hardenWebContents(view.webContents, options, entry, (url) => { - void openTab(session, url, true).catch((error) => { + void openTab(session, url, true, "popup").catch((error) => { pushBrowserLog(entry.errors, { level: "error", message: error instanceof Error ? error.message : "Unable to open browser popup", @@ -293,6 +374,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV entry, () => entries.get(session.viewId)?.activeTabId === entry.tabId, () => applySessionBounds(session, entry), + () => pushTabsState(options, session), ); wireAutomationEvents(view.webContents, entry); // The preview is a separate WebContentsView, so renderer-window keydown @@ -302,7 +384,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV view.webContents.on("focus", () => { lastFocusedViewId = session.viewId; }); - if (activate) activateTab(session, tabId); + if (activate) activateTab(session, tabId, false); return entry; }; @@ -324,6 +406,7 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV bounds: OFFSCREEN_BOUNDS, visible: false, parked: false, + agentBrowserCommands: 0, }; entries.set(viewId, session); viewIdsBySessionId.set(sessionId, viewId); @@ -337,7 +420,12 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV return session; }; - const openTab = async (session: BrowserSessionEntry, url: string | undefined, activate: boolean): Promise => { + const openTab = async ( + session: BrowserSessionEntry, + url: string | undefined, + activate: boolean, + reason: "opened" | "popup" = "opened", + ): Promise => { let normalizedURL: string | undefined; if (url) { const normalized = normalizeBrowserURL(url); @@ -348,13 +436,17 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV } const entry = createTab(session, activate); if (normalizedURL) { - const state = await navigateEntry(entry, normalizedURL); + const navigation = navigateEntry(entry, normalizedURL); + pushTabsState(options, session, { kind: reason, tabId: entry.tabId }); + const state = await navigation; if (state.error) throw browserError("NAVIGATION_FAILED", state.error); + } else { + pushTabsState(options, session, { kind: reason, tabId: entry.tabId }); } return entry; }; - function activateTab(session: BrowserSessionEntry, tabId: string): BrowserEntry { + function activateTab(session: BrowserSessionEntry, tabId: string, notify = true): BrowserEntry { const next = session.tabs.get(tabId); if (!next) throw browserError("TAB_NOT_FOUND", `Browser tab ${tabId} does not exist`); const previous = session.tabs.get(session.activeTabId); @@ -367,9 +459,31 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV invalidateRefs(next); applySessionBounds(session, next); pushNavState(options, next); + if (notify) pushTabsState(options, session, { kind: "selected", tabId }); return next; } + function closeTab(session: BrowserSessionEntry, tabId = session.activeTabId): BrowserTabsState { + if (session.tabs.size === 1) { + throw browserError("CANNOT_CLOSE_LAST_TAB", "The only browser tab cannot be closed"); + } + const tab = session.tabs.get(tabId); + if (!tab) throw browserError("TAB_NOT_FOUND", `Browser tab ${tabId} does not exist`); + const wasActive = tabId === session.activeTabId; + disposeNetworkCapture(tab, "tab-closed"); + if (session.networkTabId === tabId) session.networkTabId = undefined; + session.tabs.delete(tabId); + tabsByWebContentsId.delete(tab.view.webContents.id); + destroyTabView(tab); + if (wasActive) { + const nextTabId = [...session.tabs.keys()].at(-1)!; + activateTab(session, nextTabId, false); + } + const state = listTabs(session, { kind: "closed", tabId }); + options.mainWindow.webContents.send("browser:tabsState", state); + return state; + } + function applySessionBounds(session: BrowserSessionEntry, entry: BrowserEntry): void { if (!session.visible) { entry.view.setVisible?.(false); @@ -486,11 +600,15 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV // Electron has torn down contentView and the child WebContentsViews. Touching // them throws "Object has been destroyed", so just drop our reference. if (options.mainWindow.isDestroyed?.()) { - for (const entry of session.tabs.values()) tabsByWebContentsId.delete(entry.view.webContents.id); + for (const entry of session.tabs.values()) { + tabsByWebContentsId.delete(entry.view.webContents.id); + disposeNetworkCapture(entry, "session-closed"); + } return; } for (const entry of session.tabs.values()) { tabsByWebContentsId.delete(entry.view.webContents.id); + disposeNetworkCapture(entry, "session-closed"); destroyTabView(entry); } }; @@ -613,6 +731,22 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV handle("browser:stop", (event, viewId: string) => isRendererOwned(event, viewId) ? invokeNav(viewId, (contents) => contents.stop()) : emptyNavState(viewId), ); + handle("browser:getTabs", (event, viewId: string) => { + const session = entries.get(viewId); + return session && isRendererOwned(event, viewId) ? listTabs(session) : emptyTabsState(viewId); + }); + handle("browser:selectTab", (event, input: BrowserTabInput) => { + const session = entries.get(input.viewId); + if (!session || !isRendererOwned(event, input.viewId)) return emptyTabsState(input.viewId); + activateTab(session, input.tabId); + return listTabs(session); + }); + handle("browser:closeTab", (event, input: BrowserTabInput) => { + const session = entries.get(input.viewId); + return session && isRendererOwned(event, input.viewId) + ? closeTab(session, input.tabId) + : emptyTabsState(input.viewId); + }); handle("browser:annotation:setMode", (event, input: BrowserAnnotationModeInput) => setAnnotationMode(event, input)); on("browser:destroy", (event, viewId: string) => { if (isRendererOwned(event, viewId)) destroy(viewId); @@ -629,7 +763,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV if (!sessionId.trim()) throw browserError("INVALID_ARGUMENT", "sessionId is required"); const session = ensureSession(sessionId); const entry = activeEntry(session); - switch (action) { + setAgentBrowserActivity(session, action, true); + try { + switch (action) { case "open": { const url = stringArg(args, "url", "URL_REQUIRED", "url is required"); const state = await navigate({ viewId: entry.state.viewId, url }); @@ -675,22 +811,9 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV return tabResult(tab, true); } case "tab-close": { - if (session.tabs.size === 1) { - throw browserError("CANNOT_CLOSE_LAST_TAB", "The only browser tab cannot be closed"); - } const tabId = typeof args.tabId === "string" && args.tabId.trim() ? args.tabId.trim() : session.activeTabId; - const tab = session.tabs.get(tabId); - if (!tab) throw browserError("TAB_NOT_FOUND", `Browser tab ${tabId} does not exist`); - const wasActive = tabId === session.activeTabId; - session.tabs.delete(tabId); - tabsByWebContentsId.delete(tab.view.webContents.id); - destroyTabView(tab); - if (wasActive) { - const nextTabId = [...session.tabs.keys()].at(-1)!; - activateTab(session, nextTabId); - } - return { closedTabId: tabId, ...listTabs(session) }; + return { closedTabId: tabId, ...closeTab(session, tabId) }; } case "scroll": return scrollEntry( @@ -726,12 +849,29 @@ export function createBrowserViewHost(options: BrowserViewHostOptions): BrowserV return waitForEntry(entry, args); case "screenshot": return screenshotEntry(entry); + case "network-start": + return startNetworkCapture( + session, + entry, + networkDurationArg(args.durationSeconds), + ); + case "network-status": + return networkCaptureStatus(networkEntryFor(session)); + case "network-list": + return networkCaptureResult(networkEntryFor(session)); + case "network-stop": + return stopNetworkCapture(networkEntryFor(session), "stopped"); + case "network-clear": + return clearNetworkCapture(networkEntryFor(session)); case "console": return { messages: [...entry.consoleMessages] }; case "errors": return { messages: [...entry.errors] }; default: throw browserError("INVALID_ARGUMENT", `Unsupported browser action: ${action}`); + } + } finally { + setAgentBrowserActivity(session, action, false); } }, dispose: () => { @@ -804,6 +944,10 @@ function emptyNavState(viewId: string): BrowserNavState { }; } +function emptyTabsState(viewId: string): BrowserTabsState { + return { viewId, activeTabId: "", tabs: [] }; +} + function activeEntry(session: BrowserSessionEntry): BrowserEntry { const entry = session.tabs.get(session.activeTabId); if (!entry) throw browserError("BROWSER_TARGET_UNAVAILABLE", "Active browser tab is unavailable"); @@ -824,16 +968,25 @@ function tabResult(entry: BrowserEntry, active: boolean): { }; } -function listTabs(session: BrowserSessionEntry): { - activeTabId: string; - tabs: Array<{ id: string; url: string; title: string; active: boolean }>; -} { +function listTabs(session: BrowserSessionEntry, change?: BrowserTabsState["change"]): BrowserTabsState { return { + viewId: session.viewId, activeTabId: session.activeTabId, tabs: [...session.tabs.values()].map((entry) => tabResult(entry, entry.tabId === session.activeTabId)), + ...(change ? { change } : {}), }; } +function pushTabsState( + options: BrowserViewHostOptions, + session: BrowserSessionEntry, + change?: BrowserTabsState["change"], +): BrowserTabsState { + const state = listTabs(session, change); + options.mainWindow.webContents.send("browser:tabsState", state); + return state; +} + function hardenWebContents( contents: BrowserWebContents, options: BrowserViewHostOptions, @@ -863,8 +1016,10 @@ function wireNavEvents( entry: BrowserEntry, isActive: () => boolean, syncActiveBounds: () => void, + syncTabs: () => void, ): void { const update = () => { + syncTabs(); if (isActive()) pushNavState(options, entry); }; contents.on("did-navigate", () => { @@ -986,6 +1141,7 @@ function wireAutomationEvents(contents: BrowserWebContents, entry: BrowserEntry) const targetDebugger = contents.debugger; if (!targetDebugger) return; targetDebugger.on("message", (_event, method, params) => { + handleNetworkDebuggerEvent(entry, method, params as Record); if (method === "DOM.documentUpdated") { invalidateRefs(entry); return; @@ -1031,6 +1187,274 @@ async function ensureDebugger(entry: BrowserEntry): Promise { await debug.sendCommand("DOM.enable"); } +function networkEntryFor(session: BrowserSessionEntry): BrowserEntry { + if (session.networkTabId) { + const captured = session.tabs.get(session.networkTabId); + if (captured) return captured; + session.networkTabId = undefined; + } + return activeEntry(session); +} + +async function startNetworkCapture( + session: BrowserSessionEntry, + entry: BrowserEntry, + durationSeconds: number, +): Promise { + const existing = networkEntryFor(session); + if (existing.networkCapture?.active) { + return { ...networkCaptureStatus(existing), alreadyActive: true }; + } + if (existing !== entry) disposeNetworkCapture(existing, "restarted"); + disposeNetworkCapture(entry, "restarted"); + await ensureDebugger(entry); + await entry.view.webContents.debugger.sendCommand("Network.enable"); + const started = Date.now(); + const capture: BrowserNetworkCapture = { + active: true, + tabId: entry.tabId, + startedAt: new Date(started).toISOString(), + expiresAt: new Date(started + durationSeconds * 1_000).toISOString(), + maxEntries: MAX_NETWORK_REQUESTS, + nextSequence: 1, + requests: [], + byRequestId: new Map(), + }; + capture.timer = setTimeout(() => { + void stopNetworkCapture(entry, "expired"); + }, durationSeconds * 1_000); + entry.networkCapture = capture; + session.networkTabId = entry.tabId; + return networkCaptureStatus(entry); +} + +function networkCaptureStatus(entry: BrowserEntry): Record { + const capture = entry.networkCapture; + if (!capture) { + return { + active: false, + metadataOnly: true, + tabId: entry.tabId, + requestCount: 0, + maxEntries: MAX_NETWORK_REQUESTS, + }; + } + return { + active: capture.active, + metadataOnly: true, + tabId: capture.tabId, + requestCount: capture.requests.length, + maxEntries: capture.maxEntries, + startedAt: capture.startedAt, + expiresAt: capture.expiresAt, + ...(capture.stoppedAt ? { stoppedAt: capture.stoppedAt } : {}), + ...(capture.stopReason ? { stopReason: capture.stopReason } : {}), + }; +} + +function networkCaptureResult(entry: BrowserEntry): Record { + return { + ...networkCaptureStatus(entry), + requests: (entry.networkCapture?.requests ?? []).map(publicNetworkRequest), + }; +} + +async function stopNetworkCapture(entry: BrowserEntry, reason: string): Promise> { + const capture = entry.networkCapture; + if (!capture?.active) return networkCaptureResult(entry); + if (capture.timer) { + clearTimeout(capture.timer); + capture.timer = undefined; + } + capture.active = false; + capture.stoppedAt = new Date().toISOString(); + capture.stopReason = reason; + try { + await entry.view.webContents.debugger.sendCommand("Network.disable"); + } catch { + // The target may have closed while an expiry timer was firing. The in-memory + // capture is still safely stopped and can be discarded with the tab. + } + return networkCaptureResult(entry); +} + +function clearNetworkCapture(entry: BrowserEntry): Record { + const capture = entry.networkCapture; + if (capture) { + capture.requests = []; + capture.byRequestId.clear(); + } + return networkCaptureStatus(entry); +} + +function disposeNetworkCapture(entry: BrowserEntry, reason: string): void { + const capture = entry.networkCapture; + if (!capture) return; + const wasActive = capture.active; + if (capture.timer) clearTimeout(capture.timer); + capture.timer = undefined; + capture.active = false; + capture.stoppedAt = new Date().toISOString(); + capture.stopReason = reason; + try { + if (wasActive && entry.view.webContents.debugger?.isAttached()) { + void entry.view.webContents.debugger.sendCommand("Network.disable").catch(() => undefined); + } + } catch { + // Electron may already have destroyed the target during window shutdown. + } +} + +function handleNetworkDebuggerEvent(entry: BrowserEntry, method: string, params: Record): void { + const capture = entry.networkCapture; + if (!capture?.active || !method.startsWith("Network.")) return; + + const requestID = typeof params.requestId === "string" ? params.requestId : ""; + if (!requestID) return; + const timestamp = finiteNumber(params.timestamp); + + if (method === "Network.requestWillBeSent") { + const request = objectValue(params.request); + const url = typeof request.url === "string" ? request.url : ""; + const previous = capture.byRequestId.get(requestID); + const redirect = objectValue(params.redirectResponse); + if (previous && Object.keys(redirect).length > 0) { + applyNetworkResponse(previous, redirect); + finishNetworkRequest(previous, timestamp); + previous.redirectedTo = sanitizeNetworkURL(url); + } + const wallTime = finiteNumber(params.wallTime); + const item: InternalBrowserNetworkRequest = { + id: `n${capture.nextSequence++}`, + protocolRequestId: requestID, + method: typeof request.method === "string" ? request.method : "GET", + url: sanitizeNetworkURL(url), + resourceType: typeof params.type === "string" ? params.type.toLowerCase() : undefined, + startedAt: wallTime ? new Date(wallTime * 1_000).toISOString() : new Date().toISOString(), + startedMonotonic: timestamp, + requestHeaders: selectedNetworkHeaders(request.headers, "request"), + }; + appendNetworkRequest(capture, item); + capture.byRequestId.set(requestID, item); + return; + } + + const item = capture.byRequestId.get(requestID); + if (!item) return; + switch (method) { + case "Network.responseReceived": + applyNetworkResponse(item, objectValue(params.response)); + break; + case "Network.loadingFinished": + finishNetworkRequest(item, timestamp); + break; + case "Network.loadingFailed": + item.failed = true; + item.canceled = params.canceled === true; + item.errorText = typeof params.errorText === "string" ? params.errorText : "Request failed"; + finishNetworkRequest(item, timestamp); + break; + case "Network.requestServedFromCache": + item.fromCache = true; + break; + } +} + +function applyNetworkResponse(item: InternalBrowserNetworkRequest, response: Record): void { + const status = finiteNumber(response.status); + if (status !== undefined) item.status = status; + if (typeof response.statusText === "string" && response.statusText) item.statusText = response.statusText; + if (typeof response.mimeType === "string" && response.mimeType) item.mimeType = response.mimeType; + item.fromCache = + item.fromCache === true || + response.fromDiskCache === true || + response.fromPrefetchCache === true; + item.fromServiceWorker = response.fromServiceWorker === true; + item.responseHeaders = selectedNetworkHeaders(response.headers, "response"); +} + +function finishNetworkRequest(item: InternalBrowserNetworkRequest, timestamp: number | undefined): void { + if (timestamp !== undefined && item.startedMonotonic !== undefined) { + item.durationMs = Math.max(0, Math.round((timestamp - item.startedMonotonic) * 1_000)); + } +} + +function appendNetworkRequest(capture: BrowserNetworkCapture, item: InternalBrowserNetworkRequest): void { + capture.requests.push(item); + if (capture.requests.length <= capture.maxEntries) return; + const removed = capture.requests.shift(); + if (removed && capture.byRequestId.get(removed.protocolRequestId) === removed) { + capture.byRequestId.delete(removed.protocolRequestId); + } +} + +function publicNetworkRequest(item: InternalBrowserNetworkRequest): BrowserNetworkRequest { + const { protocolRequestId: _protocolRequestId, startedMonotonic: _startedMonotonic, ...result } = item; + return result; +} + +const SAFE_REQUEST_HEADERS = new Set([ + "accept", + "content-type", + "origin", + "referer", + "sec-fetch-mode", + "sec-fetch-site", +]); +const SAFE_RESPONSE_HEADERS = new Set([ + "access-control-allow-headers", + "access-control-allow-methods", + "access-control-allow-origin", + "cache-control", + "content-length", + "content-type", + "location", + "vary", +]); + +function selectedNetworkHeaders(value: unknown, kind: "request" | "response"): Record | undefined { + const headers = objectValue(value); + const allowed = kind === "request" ? SAFE_REQUEST_HEADERS : SAFE_RESPONSE_HEADERS; + const selected: Record = {}; + for (const [rawName, rawValue] of Object.entries(headers)) { + const name = rawName.toLowerCase(); + if (!allowed.has(name)) continue; + let headerValue = typeof rawValue === "string" ? rawValue : String(rawValue); + if (name === "referer" || name === "location") headerValue = sanitizeNetworkURL(headerValue); + selected[name] = headerValue.slice(0, 1_000); + } + return Object.keys(selected).length > 0 ? selected : undefined; +} + +function sanitizeNetworkURL(raw: string): string { + try { + const url = new URL(raw); + if (!["http:", "https:", "file:"].includes(url.protocol)) { + return `${url.protocol}[redacted]`; + } + url.username = ""; + url.password = ""; + url.hash = ""; + for (const name of [...url.searchParams.keys()]) { + url.searchParams.set(name, "[redacted]"); + } + return url.href; + } catch { + const withoutFragment = raw.split("#", 1)[0] ?? ""; + return (withoutFragment.split("?", 1)[0] ?? "").slice(0, 2_000); + } +} + +function objectValue(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + async function snapshotEntry(entry: BrowserEntry, interactiveOnly: boolean): Promise { await ensureDebugger(entry); await entry.view.webContents.debugger.sendCommand("Accessibility.enable"); @@ -1522,6 +1946,23 @@ function numberArg(value: unknown, min: number, max: number): number { return Math.max(min, Math.min(max, Math.round(value))); } +function networkDurationArg(value: unknown): number { + if (value === undefined) return DEFAULT_NETWORK_CAPTURE_SECONDS; + if ( + typeof value !== "number" || + !Number.isFinite(value) || + !Number.isInteger(value) || + value < 1 || + value > MAX_NETWORK_CAPTURE_SECONDS + ) { + throw browserError( + "INVALID_ARGUMENT", + `network capture duration must be an integer from 1 to ${MAX_NETWORK_CAPTURE_SECONDS} seconds`, + ); + } + return value; +} + function stringValue(value: AXValue | undefined): string { return typeof value?.value === "string" ? value.value : value?.value == null ? "" : String(value.value); } diff --git a/frontend/src/preload.ts b/frontend/src/preload.ts index 51441c4c57..108cffac49 100644 --- a/frontend/src/preload.ts +++ b/frontend/src/preload.ts @@ -1,6 +1,11 @@ import { contextBridge, ipcRenderer } from "electron"; import { KEYBOARD_SHORTCUTS_HELP_CHANNEL, NEW_SESSION_SHORTCUT_CHANNEL } from "./shared/shortcuts"; -import type { BrowserNavState, BrowserRect } from "./main/browser-view-host"; +import type { + BrowserAgentActivityState, + BrowserNavState, + BrowserRect, + BrowserTabsState, +} from "./main/browser-view-host"; import type { DaemonStatus } from "./shared/daemon-status"; import type { TelemetryBootstrap } from "./shared/telemetry"; import type { MigrationState } from "./main/app-state"; @@ -110,6 +115,11 @@ const api = { goForward: (viewId: string) => ipcRenderer.invoke("browser:goForward", viewId) as Promise, reload: (viewId: string) => ipcRenderer.invoke("browser:reload", viewId) as Promise, stop: (viewId: string) => ipcRenderer.invoke("browser:stop", viewId) as Promise, + getTabs: (viewId: string) => ipcRenderer.invoke("browser:getTabs", viewId) as Promise, + selectTab: (input: { viewId: string; tabId: string }) => + ipcRenderer.invoke("browser:selectTab", input) as Promise, + closeTab: (input: { viewId: string; tabId: string }) => + ipcRenderer.invoke("browser:closeTab", input) as Promise, destroy: (viewId: string) => ipcRenderer.send("browser:destroy", viewId), setAnnotationMode: (input: BrowserAnnotationModeInput) => ipcRenderer.invoke("browser:annotation:setMode", input) as Promise, @@ -120,6 +130,20 @@ const api = { ipcRenderer.off("browser:navState", wrapped); }; }, + onTabsState: (listener: (state: BrowserTabsState) => void) => { + const wrapped = (_event: Electron.IpcRendererEvent, state: BrowserTabsState) => listener(state); + ipcRenderer.on("browser:tabsState", wrapped); + return () => { + ipcRenderer.off("browser:tabsState", wrapped); + }; + }, + onAgentActivity: (listener: (state: BrowserAgentActivityState) => void) => { + const wrapped = (_event: Electron.IpcRendererEvent, state: BrowserAgentActivityState) => listener(state); + ipcRenderer.on("browser:agentActivity", wrapped); + return () => { + ipcRenderer.off("browser:agentActivity", wrapped); + }; + }, onAnnotationSubmit: (listener: (payload: BrowserAnnotationSubmitPayload) => void) => { const wrapped = (_event: Electron.IpcRendererEvent, payload: BrowserAnnotationSubmitPayload) => listener(payload); ipcRenderer.on("browser:annotation:submitted", wrapped); diff --git a/frontend/src/renderer/components/BrowserPanel.test.tsx b/frontend/src/renderer/components/BrowserPanel.test.tsx index 41fa0ee46c..66432c9b3a 100644 --- a/frontend/src/renderer/components/BrowserPanel.test.tsx +++ b/frontend/src/renderer/components/BrowserPanel.test.tsx @@ -1,4 +1,4 @@ -import { act, render, screen, waitFor } from "@testing-library/react"; +import { act, render, renderHook, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { BrowserPanel, BrowserPanelView, useBrowserAnnotationQueue } from "./BrowserPanel"; @@ -22,7 +22,13 @@ const hookState = vi.hoisted(() => ({ goForward: vi.fn(), reload: vi.fn(), stop: vi.fn(), + selectTab: vi.fn(), + closeTab: vi.fn(), setAnnotationMode: vi.fn(), + tabs: [{ id: "t1", url: "", title: "", active: true }], + activeTabId: "t1", + tabNotice: "", + agentBrowserActive: false, previewUrl: undefined as string | undefined, navState: { viewId: "42:sess-1", @@ -46,6 +52,12 @@ vi.mock("../hooks/useBrowserView", () => ({ goForward: hookState.goForward, reload: hookState.reload, stop: hookState.stop, + tabs: hookState.tabs, + activeTabId: hookState.activeTabId, + tabNotice: hookState.tabNotice, + agentBrowserActive: hookState.agentBrowserActive, + selectTab: hookState.selectTab, + closeTab: hookState.closeTab, annotationMode: false, setAnnotationMode: hookState.setAnnotationMode, }; @@ -122,6 +134,8 @@ describe("BrowserPanel", () => { hookState.goForward.mockReset(); hookState.reload.mockReset(); hookState.stop.mockReset(); + hookState.selectTab.mockReset(); + hookState.closeTab.mockReset(); hookState.setAnnotationMode.mockReset(); hookState.setAnnotationMode.mockResolvedValue(undefined); postMock.mockReset(); @@ -141,6 +155,10 @@ describe("BrowserPanel", () => { }; }); hookState.previewUrl = undefined; + hookState.tabs = [{ id: "t1", url: "", title: "", active: true }]; + hookState.activeTabId = "t1"; + hookState.tabNotice = ""; + hookState.agentBrowserActive = false; hookState.navState = { viewId: "42:sess-1", url: "", @@ -193,6 +211,33 @@ describe("BrowserPanel", () => { expect(hookState.stop).toHaveBeenCalled(); }); + it("shows a compact tab count and lets the user select and close tabs", async () => { + hookState.tabs = [ + { id: "t1", url: "http://localhost:3000/", title: "First app", active: false }, + { id: "t2", url: "http://localhost:4173/", title: "Second app", active: true }, + ]; + hookState.activeTabId = "t2"; + render( undefined} poppedOut={false} session={session} />); + + const tabsButton = screen.getByRole("button", { name: "Browser tabs (2)" }); + expect(tabsButton).toHaveClass("bg-accent-weak"); + await userEvent.click(tabsButton); + await userEvent.click(screen.getByText("First app")); + expect(hookState.selectTab).toHaveBeenCalledWith("t1"); + + await userEvent.click(tabsButton); + await userEvent.click(screen.getByRole("menuitem", { name: "Close tab First app" })); + expect(hookState.closeTab).toHaveBeenCalledWith("t1"); + }); + + it("surfaces a popup-created tab without adding a full tab strip", () => { + hookState.tabNotice = "Opened new tab"; + render( undefined} poppedOut={false} session={session} />); + + expect(screen.getByRole("status")).toHaveTextContent("Opened new tab"); + expect(screen.getByRole("button", { name: "Browser tabs (1)" })).toBeInTheDocument(); + }); + it("shows empty and error states", () => { hookState.navState = { ...hookState.navState, error: "Connection refused" }; render( undefined} poppedOut={false} session={session} />); @@ -219,7 +264,7 @@ describe("BrowserPanel", () => { expect(hookState.setAnnotationMode).toHaveBeenCalledWith(true); }); - it("shows the working indicator only for active agent activity", () => { + it("shows browser activity only for browser commands, not general worker activity", () => { hookState.navState = { ...hookState.navState, url: "http://localhost:5173/" }; const first = render( { ); expect(screen.getByRole("button", { name: /annotate/i })).toBeEnabled(); - expect(screen.getByText("Agent working")).toBeInTheDocument(); + expect(screen.queryByText("Agent using browser")).not.toBeInTheDocument(); first.unmount(); + hookState.agentBrowserActive = true; render( { ); expect(screen.getByRole("button", { name: /annotate/i })).toBeEnabled(); - expect(screen.queryByText("Agent working")).not.toBeInTheDocument(); + expect(screen.getByText("Agent using browser")).toBeInTheDocument(); }); it("disables annotation mode when no page is loaded", () => { @@ -499,6 +545,39 @@ describe("BrowserPanel", () => { expect(postMock).toHaveBeenCalledTimes(1); }); + it("clears the annotation delivery confirmation after two seconds", async () => { + vi.useFakeTimers(); + try { + const { result } = renderHook(() => + useBrowserAnnotationQueue({ + sessionId: "sess-1", + navUrl: "http://localhost:5173/", + }), + ); + + act(() => { + result.current.enqueue(annotationPayload("Make this button blue.")); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + expect(result.current.status).toBe("sent"); + + act(() => { + vi.advanceTimersByTime(1_999); + }); + expect(result.current.status).toBe("sent"); + + act(() => { + vi.advanceTimersByTime(1); + }); + expect(result.current.status).toBe("idle"); + } finally { + vi.useRealTimers(); + } + }); + it("shows annotation send errors", async () => { postMock.mockResolvedValue({ error: { message: "AO daemon is not ready." } }); hookState.navState = { ...hookState.navState, url: "http://localhost:5173/" }; diff --git a/frontend/src/renderer/components/BrowserPanel.tsx b/frontend/src/renderer/components/BrowserPanel.tsx index e4bac79737..1b470b86e3 100644 --- a/frontend/src/renderer/components/BrowserPanel.tsx +++ b/frontend/src/renderer/components/BrowserPanel.tsx @@ -1,12 +1,29 @@ import { useCallback, useEffect, useRef, useState, type FormEvent } from "react"; -import { ArrowLeft, ArrowRight, Globe2, Maximize2, Minimize2, MousePointer2, RefreshCw, X } from "lucide-react"; +import { + ArrowLeft, + ArrowRight, + Check, + Globe2, + Layers3, + Maximize2, + Minimize2, + MousePointer2, + RefreshCw, + X, +} from "lucide-react"; import { apiClient, apiErrorMessage } from "../lib/api-client"; import { useBrowserView, type BrowserViewModel } from "../hooks/useBrowserView"; import { formatBrowserAnnotationMessage, type BrowserAnnotationSubmitPayload } from "../../shared/browser-annotations"; import type { WorkspaceSession } from "../types/workspace"; -import { isAgentActivityWorking } from "../lib/session-presentation"; import { Button } from "./ui/button"; import { Input } from "./ui/input"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuTrigger, +} from "./ui/dropdown-menu"; import { cn } from "../lib/utils"; type BrowserPanelProps = { @@ -45,9 +62,12 @@ export function useBrowserAnnotationQueue({ const annotationSendingRef = useRef(false); const sessionIdRef = useRef(sessionId ?? ""); const generationRef = useRef(0); + const sentTimerRef = useRef(null); const resetQueue = useCallback(() => { generationRef.current += 1; + if (sentTimerRef.current !== null) window.clearTimeout(sentTimerRef.current); + sentTimerRef.current = null; annotationQueueRef.current = []; annotationSendingRef.current = false; setState({ status: "idle", error: "", queuedCount: 0 }); @@ -98,7 +118,17 @@ export function useBrowserAnnotationQueue({ const queuedCount = annotationQueueRef.current.length; setState({ status: queuedCount > 0 ? "queued" : "sent", error: "", queuedCount }); - if (queuedCount > 0) drainAnnotationQueue(); + if (queuedCount > 0) { + drainAnnotationQueue(); + } else { + if (sentTimerRef.current !== null) window.clearTimeout(sentTimerRef.current); + sentTimerRef.current = window.setTimeout(() => { + sentTimerRef.current = null; + setState((current) => + current.status === "sent" ? { status: "idle", error: "", queuedCount: 0 } : current, + ); + }, 2_000); + } } })(); }, []); @@ -113,6 +143,13 @@ export function useBrowserAnnotationQueue({ resetQueue(); }, [navUrl, resetQueue]); + useEffect( + () => () => { + if (sentTimerRef.current !== null) window.clearTimeout(sentTimerRef.current); + }, + [], + ); + const beginPicking = useCallback(() => { setState((current) => ({ ...current, status: "picking", error: "" })); }, []); @@ -181,7 +218,6 @@ export function BrowserPanel({ session, active, poppedOut, onTogglePopOut }: Bro } export function BrowserPanelView({ - session, poppedOut, onTogglePopOut, browserView, @@ -198,6 +234,12 @@ export function BrowserPanelView({ goForward, reload, stop, + tabs, + activeTabId, + tabNotice, + selectTab, + closeTab, + agentBrowserActive, annotationMode, setAnnotationMode, } = browserView; @@ -205,7 +247,6 @@ export function BrowserPanelView({ const { beginPicking, cancelPicking, enqueue, error, failPicking, queuedCount, retryQueued, status } = annotationQueue; const showStaticPreview = !window.ao?.browser && navState.url !== ""; - const sessionBusy = isAgentActivityWorking(session.activity); const canAnnotate = Boolean(window.ao?.browser && viewId && navState.url); const canRetryAnnotation = status === "error" && queuedCount > 0; @@ -340,8 +381,8 @@ export function BrowserPanelView({ > {annotationStatusLabel} - ) : sessionBusy ? ( - Agent working + ) : agentBrowserActive ? ( + Agent using browser ) : null}

+ {tabNotice ? ( + + {tabNotice} + + ) : null} + + + + + + Browser tabs + {tabs.map((tab) => { + const label = browserTabLabel(tab.title, tab.url); + return ( +
+ void selectTab(tab.id)} + textValue={`${label.title} ${label.subtitle}`} + > + + {tab.id === activeTabId ? + + {label.title} + {label.subtitle} + + + void closeTab(tab.id)} + title={tabs.length === 1 ? "The only tab cannot be closed" : `Close ${label.title}`} + > + +
+ ); + })} + +