diff --git a/README.md b/README.md index 663315e..66cdd8b 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,35 @@ mano-cua stop > **Note:** Only one task can run at a time per device. If you need to start a new task, first stop the current one with `mano-cua stop`. +## Configuration + +Persistent settings are stored in `~/.mano/config.json`. + +```bash +# List all keys and current values +mano-cua config --list + +# Read or set a value +mano-cua config --get save-trajectory +mano-cua config --set save-trajectory true +``` + +| Key | Default | Description | +| ----------------- | ------- | ------------------------------------------------------------------------ | +| `max-steps` | `100` | Maximum steps per task | +| `minimize` | `false` | Start with the status panel minimized (`true` / `false`) | +| `save-trajectory` | `false` | Save a per-run trajectory under `~/.mano/trajectory/` (`true` / `false`) | + +When `save-trajectory` is `true`, each run writes a session folder (for example `~/.mano/trajectory/sess--/`) containing: + +- `session.json` — task metadata and final status +- `history.jsonl` — step-by-step actions, reasoning, and screenshot paths +- `screenshots/` — PNG captures per step +- `whole.log` — full terminal output (stdout/stderr) for the run +- `report.html` — offline HTML timeline with collapsible screenshots (open in a browser) + +Other keys (for example `default-model-path`, `python-path`, `w8a8`) apply to local mode; run `mano-cua config --list` for the full list. + ## Local Mode Runs [Mano-P](https://huggingface.co/Mininglamp-2718/Mano-P) entirely on-device via MLX. No data leaves the machine. Requires macOS with Apple Silicon (M1+). diff --git a/visual/computer/computer_action_executor.py b/visual/computer/computer_action_executor.py index a2e616b..f6295b5 100644 --- a/visual/computer/computer_action_executor.py +++ b/visual/computer/computer_action_executor.py @@ -172,7 +172,9 @@ def _type_text(self, text: str): env["LC_ALL"] = "en_US.UTF-8" subprocess.run(["pbcopy"], input=text.encode("utf-8"), env=env, check=True) elif system == "Windows": - subprocess.run(["clip"], input=text.encode("utf-16le"), check=True) + # clip.exe expects UTF-16 with BOM; utf-16le alone is misread as ANSI on + # localized Windows and pastes mojibake into Search / text fields. + subprocess.run(["clip"], input=text.encode("utf-16"), check=True) else: subprocess.run(["xclip", "-selection", "clipboard"], input=text.encode("utf-8"), check=True) diff --git a/visual/config/user_config.py b/visual/config/user_config.py index 3bc1353..b986c91 100644 --- a/visual/config/user_config.py +++ b/visual/config/user_config.py @@ -18,7 +18,7 @@ "w8a8": "W8A8 INT8 acceleration: 'auto', 'on', or 'off' (default: auto, requires M5+)", "max-steps": f"Maximum steps per task (default: {CONFIG_DEFAULTS['max-steps']})", "minimize": f"Start with minimized UI panel: true/false (default: {CONFIG_DEFAULTS['minimize']})", - "save-trajectory": f"Save screenshots and actions per step: true/false (default: {CONFIG_DEFAULTS['save-trajectory']})", + "save-trajectory": f"Save trajectory (screenshots, history.jsonl, whole.log, report.html): true/false (default: {CONFIG_DEFAULTS['save-trajectory']})", } diff --git a/visual/model/task_model.py b/visual/model/task_model.py index 4ca517a..77cd495 100644 --- a/visual/model/task_model.py +++ b/visual/model/task_model.py @@ -57,7 +57,14 @@ def _notify_state_changed(self): self._on_state_changed(self.state) # ========== Initialization Methods ========== - def init_task(self, task_name: str, agent: BaseAgent, expected_result: Optional[str] = None, max_steps: int = None): + def init_task( + self, + task_name: str, + agent: BaseAgent, + expected_result: Optional[str] = None, + max_steps: int = None, + cloud_session_id: Optional[str] = None, + ): """Initialize automation task""" # Basic configuration self.state.task_name = task_name @@ -96,7 +103,11 @@ def init_task(self, task_name: str, agent: BaseAgent, expected_result: Optional[ "arch": platform.machine(), "os_version": platform.mac_ver()[0] or platform.version(), } + if cloud_session_id: + self._session_meta["cloud_session_id"] = cloud_session_id self._save_session_meta() + from visual.trajectory.log_tee import install_trajectory_tee + install_trajectory_tee(self._trajectory_dir) print(f"Trajectory: {self._trajectory_dir}") # Initialize executor @@ -127,6 +138,8 @@ def update_progress(self, step_idx: int, action_desc: str, reasoning: str = "", print(f"[step {step_idx}] Action: {action_desc}") if reasoning: print(f"[step {step_idx}] Reasoning: {reasoning}") + if self._save_trajectory and self._trajectory_dir: + self._record_progress_history(step_idx, action_desc, reasoning) self._notify_state_changed() # ========== State Management ========== @@ -137,6 +150,7 @@ def mark_completed(self): self.stop_event.set() self._save_final_trajectory() self._print_summary("COMPLETED") + self._finalize_trajectory_log() self._notify_state_changed() def mark_stopped(self): @@ -149,6 +163,7 @@ def mark_stopped(self): self.agent.stop() self._save_final_trajectory() self._print_summary("STOPPED_BY_USER") + self._finalize_trajectory_log() self._notify_state_changed() def mark_error(self, error_msg: str): @@ -159,6 +174,7 @@ def mark_error(self, error_msg: str): self.stop_event.set() self._save_final_trajectory() self._print_summary("ERROR", error_msg) + self._finalize_trajectory_log() self._notify_state_changed() def _print_summary(self, final_status: str, error_msg: str = ""): @@ -176,6 +192,12 @@ def _print_summary(self, final_status: str, error_msg: str = ""): print(f"Error: {error_msg}") if self.eval_result: print(f"Evaluation result: {json.dumps(self.eval_result, indent=2, ensure_ascii=False)}") + if self._save_trajectory and self._trajectory_dir: + report_path = os.path.abspath(os.path.join(self._trajectory_dir, "report.html")) + if os.path.isfile(report_path): + from pathlib import Path + print(f"Report: {report_path}") + print(f"Preview: {Path(report_path).as_uri()}") print(f"{'='*50}\n") def _mark_evaluating(self): @@ -231,7 +253,9 @@ def run_automation_task(self): if self.state.status == TASK_STATUS["MAX_STEP_REACHED"]: self.state.is_running = False self.stop_event.set() + self._save_final_trajectory() self._print_summary("MAX_STEP_REACHED") + self._finalize_trajectory_log() self._notify_state_changed() skip = not (self.expected_result and self.agent.agent_type == "cloud") if not skip: @@ -290,11 +314,15 @@ def _execute_task_steps(self): # 5. Handle terminal status if status == "DONE": if self._save_trajectory and self._trajectory_dir: - self._save_step_trajectory(step_idx + 1, reasoning, actions, action_desc, tool_results) + self._save_step_trajectory( + step_idx, reasoning, actions, action_desc, tool_results, agent_status="DONE" + ) break elif status == "FAIL": if self._save_trajectory and self._trajectory_dir: - self._save_step_trajectory(step_idx + 1, reasoning, actions, action_desc, tool_results) + self._save_step_trajectory( + step_idx, reasoning, actions, action_desc, tool_results, agent_status="FAIL" + ) self.mark_error("Agent marked task as failed") break elif status == "MAX_STEP_REACHED": @@ -350,9 +378,36 @@ def _execute_task_steps(self): break # ========== Trajectory Saving ========== - def _save_step_trajectory(self, step_idx, reasoning, actions, action_desc, tool_results): + def _record_progress_history(self, step_idx: int, action_desc: str, reasoning: str): + """Append a history line for UI progress (may duplicate per-step action lines).""" + from visual.trajectory.history import append_history_line + + phase = "init" if action_desc == "Initializing" else "action" + if action_desc in ("DONE", "FAIL"): + phase = "terminal" + append_history_line( + self._trajectory_dir, + step=step_idx, + action_desc=action_desc, + reasoning=reasoning, + actions=[], + phase=phase, + status=None, + screenshot=None, + ) + + def _save_step_trajectory( + self, + step_idx, + reasoning, + actions, + action_desc, + tool_results, + agent_status=None, + ): """Save screenshot + action metadata for one step.""" try: + screenshot_rel = None for tr in reversed(tool_results or []): b64 = tr.get("screenshot_b64") if b64: @@ -360,18 +415,23 @@ def _save_step_trajectory(self, step_idx, reasoning, actions, action_desc, tool_ path = os.path.join(self._trajectory_dir, "screenshots", f"{step_idx}.png") with open(path, "wb") as f: f.write(screenshot_bytes) + screenshot_rel = f"screenshots/{step_idx}.png" break - step_data = { - "step": step_idx, - "reasoning": reasoning, - "action_desc": action_desc, - "actions": [{"name": a.get("name"), "input": a.get("input"), "action_type": a.get("action_type")} for a in actions], - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), - } - history_path = os.path.join(self._trajectory_dir, "history.jsonl") - with open(history_path, "a", encoding="utf-8") as f: - f.write(json.dumps(step_data, ensure_ascii=False) + "\n") + from visual.trajectory.history import append_history_line + + phase = "terminal" if agent_status in ("DONE", "FAIL") else "action" + desc = action_desc if action_desc else (agent_status or "") + append_history_line( + self._trajectory_dir, + step=step_idx, + action_desc=desc, + reasoning=reasoning or "", + actions=actions or [], + phase=phase, + status=agent_status, + screenshot=screenshot_rel, + ) except Exception as e: print(f"Warning: failed to save trajectory step {step_idx}: {e}") @@ -394,9 +454,23 @@ def _save_final_trajectory(self): "finished_at": time.strftime("%Y-%m-%dT%H:%M:%S"), }) self._save_session_meta() + + from visual.trajectory.report_generator import generate_report + + generate_report(self._trajectory_dir) except Exception as e: print(f"Warning: failed to save final trajectory: {e}") + def _finalize_trajectory_log(self): + """Close stdout tee after summary is printed.""" + if not self._save_trajectory: + return + try: + from visual.trajectory.log_tee import uninstall_trajectory_tee + uninstall_trajectory_tee() + except Exception as e: + print(f"Warning: failed to close trajectory log: {e}") + def _save_session_meta(self): with open(os.path.join(self._trajectory_dir, "session.json"), "w", encoding="utf-8") as f: json.dump(self._session_meta, f, indent=2, ensure_ascii=False) \ No newline at end of file diff --git a/visual/model/tests/__init__.py b/visual/model/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/visual/trajectory/__init__.py b/visual/trajectory/__init__.py new file mode 100644 index 0000000..b134499 --- /dev/null +++ b/visual/trajectory/__init__.py @@ -0,0 +1,17 @@ +"""Trajectory logging, history, and HTML report generation.""" + +from visual.trajectory.log_tee import ( + enable_early_trajectory_buffer, + install_trajectory_tee, + is_trajectory_logging_active, + uninstall_trajectory_tee, +) +from visual.trajectory.report_generator import generate_report + +__all__ = [ + "enable_early_trajectory_buffer", + "install_trajectory_tee", + "uninstall_trajectory_tee", + "is_trajectory_logging_active", + "generate_report", +] diff --git a/visual/trajectory/action_styles.py b/visual/trajectory/action_styles.py new file mode 100644 index 0000000..b4132be --- /dev/null +++ b/visual/trajectory/action_styles.py @@ -0,0 +1,58 @@ +"""Map action descriptions to report styling kinds.""" + +from typing import Any, List, Optional + +ACTION_KIND_INITIALIZING = "initializing" +ACTION_KIND_OPEN_APP = "open_app" +ACTION_KIND_KEY = "key" +ACTION_KIND_CLICK = "click" +ACTION_KIND_TYPE = "type" +ACTION_KIND_DONE = "done" +ACTION_KIND_FAIL = "fail" +ACTION_KIND_OTHER = "other" + + +def infer_action_kind( + action_desc: str, + actions: Optional[List[dict]] = None, +) -> str: + """Derive action_kind from human-readable action_desc and optional structured actions.""" + desc = (action_desc or "").strip() + desc_lower = desc.lower() + + if desc == "Initializing" or desc_lower == "initializing": + return ACTION_KIND_INITIALIZING + if desc == "DONE" or desc_lower == "done": + return ACTION_KIND_DONE + if desc == "FAIL" or desc_lower == "fail": + return ACTION_KIND_FAIL + if desc_lower.startswith("open app:") or desc_lower.startswith("open app "): + return ACTION_KIND_OPEN_APP + if desc_lower.startswith("key:"): + return ACTION_KIND_KEY + if "click" in desc_lower: + return ACTION_KIND_CLICK + if desc_lower.startswith("type:") or desc_lower.startswith("type "): + return ACTION_KIND_TYPE + + if actions: + first = actions[0] or {} + name = (first.get("name") or "").lower() + if name == "open_app": + return ACTION_KIND_OPEN_APP + if name == "computer": + inp = first.get("input") or {} + act = (inp.get("action") or "").lower() + if act == "key": + return ACTION_KIND_KEY + if act in ("click", "left_click", "right_click", "double_click"): + return ACTION_KIND_CLICK + if act == "type": + return ACTION_KIND_TYPE + + return ACTION_KIND_OTHER + + +def css_class_for_kind(action_kind: str) -> str: + """Return CSS class name for an action_kind.""" + return f"action-{action_kind.replace('_', '-')}" diff --git a/visual/trajectory/history.py b/visual/trajectory/history.py new file mode 100644 index 0000000..f92e12e --- /dev/null +++ b/visual/trajectory/history.py @@ -0,0 +1,47 @@ +"""Append enhanced lines to trajectory history.jsonl.""" + +import json +import os +import time +from typing import Any, Dict, List, Optional + +from visual.trajectory.action_styles import infer_action_kind + + +def append_history_line( + trajectory_dir: str, + *, + step: int, + action_desc: str, + reasoning: str = "", + actions: Optional[List[Dict[str, Any]]] = None, + phase: str = "action", + status: Optional[str] = None, + screenshot: Optional[str] = None, +) -> None: + """Append one JSON line to history.jsonl with the enhanced schema.""" + actions = actions or [] + line: Dict[str, Any] = { + "step": step, + "reasoning": reasoning or "", + "action_desc": action_desc, + "actions": [ + { + "name": a.get("name"), + "input": a.get("input"), + "action_type": a.get("action_type"), + } + for a in actions + ], + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), + "phase": phase, + "action_kind": infer_action_kind(action_desc, actions), + } + if status is not None: + line["status"] = status + if screenshot is not None: + line["screenshot"] = screenshot + + history_path = os.path.join(trajectory_dir, "history.jsonl") + with open(history_path, "a", encoding="utf-8") as f: + f.write(json.dumps(line, ensure_ascii=False) + "\n") diff --git a/visual/trajectory/log_tee.py b/visual/trajectory/log_tee.py new file mode 100644 index 0000000..1bce122 --- /dev/null +++ b/visual/trajectory/log_tee.py @@ -0,0 +1,156 @@ +"""Stdout/stderr tee to trajectory whole.log with early buffer support.""" + +import os +import sys +import threading +from typing import List, Optional, TextIO + +_lock = threading.Lock() +_original_stdout: Optional[TextIO] = None +_original_stderr: Optional[TextIO] = None +_early_buffer: List[str] = [] +_buffer_active = False +_tee_installed = False +_log_file: Optional[TextIO] = None + + +class TeeWriter: + """Write to terminal stream and trajectory log file.""" + + def __init__(self, stream: TextIO, log_file: Optional[TextIO]): + self._stream = stream + self._log_file = log_file + self._write_lock = threading.Lock() + + def write(self, data: str) -> int: + if not data: + return 0 + with self._write_lock: + self._stream.write(data) + if self._log_file is not None: + self._log_file.write(data) + self._log_file.flush() + if hasattr(self._stream, "flush"): + self._stream.flush() + return len(data) + + def flush(self) -> None: + with self._write_lock: + self._stream.flush() + if self._log_file is not None: + self._log_file.flush() + + def isatty(self) -> bool: + return getattr(self._stream, "isatty", lambda: False)() + + def fileno(self) -> int: + return self._stream.fileno() + + def __getattr__(self, name: str): + return getattr(self._stream, name) + + +class _EarlyBufferWriter: + """Tee terminal output into an in-memory buffer before trajectory dir exists.""" + + def __init__(self, stream: TextIO, buffer: List[str]): + self._stream = stream + self._buffer = buffer + self._write_lock = threading.Lock() + + def write(self, data: str) -> int: + if not data: + return 0 + with self._write_lock: + self._stream.write(data) + self._buffer.append(data) + if hasattr(self._stream, "flush"): + self._stream.flush() + return len(data) + + def flush(self) -> None: + with self._write_lock: + self._stream.flush() + + def isatty(self) -> bool: + return getattr(self._stream, "isatty", lambda: False)() + + def fileno(self) -> int: + return self._stream.fileno() + + def __getattr__(self, name: str): + return getattr(self._stream, name) + + +def is_trajectory_logging_active() -> bool: + """Return True if early buffer or file tee is active.""" + return _buffer_active or _tee_installed + + +def enable_early_trajectory_buffer() -> None: + """Start teeing stdout/stderr to an in-memory buffer (before trajectory dir exists).""" + global _buffer_active, _original_stdout, _original_stderr, _early_buffer + + with _lock: + if _tee_installed or _buffer_active: + return + _original_stdout = sys.stdout + _original_stderr = sys.stderr + _early_buffer = [] + _buffer_active = True + sys.stdout = _EarlyBufferWriter(_original_stdout, _early_buffer) # type: ignore[assignment] + sys.stderr = _EarlyBufferWriter(_original_stderr, _early_buffer) # type: ignore[assignment] + + +def install_trajectory_tee(trajectory_dir: str) -> None: + """Flush early buffer to whole.log and tee further output to that file.""" + global _buffer_active, _tee_installed, _log_file, _early_buffer + + with _lock: + log_path = os.path.join(trajectory_dir, "whole.log") + os.makedirs(trajectory_dir, exist_ok=True) + + with open(log_path, "w", encoding="utf-8", errors="replace") as f: + for chunk in _early_buffer: + f.write(chunk) + _early_buffer = [] + _buffer_active = False + + if _log_file is not None and not _log_file.closed: + _log_file.close() + + _log_file = open(log_path, "a", encoding="utf-8", errors="replace") + + base_out = _original_stdout or sys.__stdout__ + base_err = _original_stderr or sys.__stderr__ + sys.stdout = TeeWriter(base_out, _log_file) # type: ignore[assignment] + sys.stderr = TeeWriter(base_err, _log_file) # type: ignore[assignment] + _tee_installed = True + + +def uninstall_trajectory_tee() -> None: + """Restore original stdout/stderr and close the log file.""" + global _tee_installed, _log_file, _buffer_active, _early_buffer + + with _lock: + if not _tee_installed and not _buffer_active: + return + + if _tee_installed: + base_out = _original_stdout or sys.__stdout__ + base_err = _original_stderr or sys.__stderr__ + if isinstance(sys.stdout, TeeWriter): + sys.stdout.flush() + if isinstance(sys.stderr, TeeWriter): + sys.stderr.flush() + sys.stdout = base_out + sys.stderr = base_err + _tee_installed = False + + if _log_file is not None and not _log_file.closed: + _log_file.flush() + _log_file.close() + _log_file = None + + _buffer_active = False + _early_buffer = [] diff --git a/visual/trajectory/report_generator.py b/visual/trajectory/report_generator.py new file mode 100644 index 0000000..cf992e5 --- /dev/null +++ b/visual/trajectory/report_generator.py @@ -0,0 +1,300 @@ +"""Generate report.html from session.json and history.jsonl.""" + +import html +import json +import os +from collections import defaultdict +from datetime import datetime +from typing import Any, Dict, List, Optional + +from visual.trajectory.action_styles import css_class_for_kind, infer_action_kind + +_WHOLE_LOG_NAME = "whole.log" + +_STYLES = """ +:root { + --bg: #0f1419; + --card: #1a2332; + --text: #e7ecf3; + --muted: #8b9cb3; + --initializing: #6b7280; + --open-app: #22c55e; + --key: #3b82f6; + --click: #f97316; + --type: #a855f7; + --done: #10b981; + --fail: #ef4444; + --other: #64748b; +} +* { box-sizing: border-box; } +body { + font-family: system-ui, -apple-system, Segoe UI, sans-serif; + background: var(--bg); + color: var(--text); + margin: 0; + padding: 1.5rem; + line-height: 1.5; +} +header.report-header { + background: var(--card); + border-radius: 12px; + padding: 1.25rem 1.5rem; + margin-bottom: 1.5rem; +} +header.report-header h1 { margin: 0 0 0.75rem; font-size: 1.35rem; } +.stats-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); + gap: 0.75rem; +} +.stat-label { color: var(--muted); font-size: 0.8rem; } +.stat-value { font-weight: 600; } +.toolbar { margin-bottom: 1rem; } +.toolbar button { + background: #2563eb; + color: #fff; + border: none; + padding: 0.45rem 0.9rem; + border-radius: 6px; + cursor: pointer; + margin-right: 0.5rem; +} +.toolbar button:hover { background: #1d4ed8; } +.step-group { + background: var(--card); + border-radius: 10px; + margin-bottom: 1rem; + overflow: hidden; +} +.step-group > h2 { + margin: 0; + padding: 0.75rem 1rem; + font-size: 1rem; + background: rgba(255,255,255,0.04); + border-bottom: 1px solid rgba(255,255,255,0.06); +} +.history-entry { + padding: 1rem; + border-bottom: 1px solid rgba(255,255,255,0.05); +} +.history-entry header { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + align-items: center; + margin-bottom: 0.5rem; +} +.action-badge { + font-weight: 600; + padding: 0.15rem 0.5rem; + border-radius: 4px; + font-size: 0.9rem; +} +.timestamp, .phase-tag { color: var(--muted); font-size: 0.8rem; } +.reasoning { white-space: pre-wrap; margin: 0.5rem 0; } +.actions-json { + font-size: 0.75rem; + background: rgba(0,0,0,0.25); + padding: 0.5rem; + border-radius: 6px; + overflow-x: auto; +} +.screenshot-details { margin-top: 0.5rem; } +.screenshot-details img { + max-width: 100%; + border-radius: 6px; + margin-top: 0.5rem; + border: 1px solid rgba(255,255,255,0.1); +} +.no-screenshot { color: var(--muted); font-size: 0.85rem; margin: 0.5rem 0 0; } +footer { margin-top: 2rem; color: var(--muted); font-size: 0.85rem; } +.action-initializing .action-badge { background: var(--initializing); color: #fff; } +.action-open-app .action-badge { background: var(--open-app); color: #fff; } +.action-key .action-badge { background: var(--key); color: #fff; } +.action-click .action-badge { background: var(--click); color: #fff; } +.action-type .action-badge { background: var(--type); color: #fff; } +.action-done .action-badge { background: var(--done); color: #fff; } +.action-fail .action-badge { background: var(--fail); color: #fff; } +.action-other .action-badge { background: var(--other); color: #fff; } +""" + +_SCRIPT = """ +function setAllScreenshots(open) { + document.querySelectorAll('.screenshot-details').forEach(function(el) { + el.open = open; + }); +} +document.getElementById('expand-all').addEventListener('click', function() { setAllScreenshots(true); }); +document.getElementById('collapse-all').addEventListener('click', function() { setAllScreenshots(false); }); +""" + + +def _escape(text: str) -> str: + return html.escape(text or "", quote=True) + + +def _parse_timestamp(ts: str) -> Optional[datetime]: + for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y%m%d-%H%M%S"): + try: + return datetime.strptime(ts, fmt) + except ValueError: + continue + return None + + +def _elapsed_seconds(started: str, finished: str) -> Optional[int]: + start_dt = _parse_timestamp(started) + end_dt = _parse_timestamp(finished) + if start_dt and end_dt: + return int((end_dt - start_dt).total_seconds()) + return None + + +def _load_session(trajectory_dir: str) -> Dict[str, Any]: + path = os.path.join(trajectory_dir, "session.json") + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _load_history(trajectory_dir: str) -> List[Dict[str, Any]]: + path = os.path.join(trajectory_dir, "history.jsonl") + if not os.path.isfile(path): + return [] + lines: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as f: + for raw in f: + raw = raw.strip() + if raw: + lines.append(json.loads(raw)) + return lines + + +def _resolve_screenshot(trajectory_dir: str, entry: Dict[str, Any]) -> Optional[str]: + rel = entry.get("screenshot") + if rel and os.path.isfile(os.path.join(trajectory_dir, rel)): + return rel + step = entry.get("step") + if step is not None: + candidate = f"screenshots/{step}.png" + if os.path.isfile(os.path.join(trajectory_dir, candidate)): + return candidate + return None + + +def _render_entry(trajectory_dir: str, entry: Dict[str, Any]) -> str: + action_desc = entry.get("action_desc", "") + reasoning = entry.get("reasoning", "") + timestamp = entry.get("timestamp", "") + actions = entry.get("actions") or [] + kind = entry.get("action_kind") or infer_action_kind(action_desc, actions) + css_class = css_class_for_kind(kind) + screenshot_rel = _resolve_screenshot(trajectory_dir, entry) + + if screenshot_rel: + screenshot_html = ( + '
' + "Show screenshot" + f'step screenshot' + "
" + ) + else: + screenshot_html = '

(No screenshot)

' + + actions_html = "" + if actions: + actions_json = _escape(json.dumps(actions, ensure_ascii=False, indent=2)) + actions_html = f'
{actions_json}
' + + return ( + f'
' + "
" + f'{_escape(action_desc)}' + f'{_escape(timestamp)}' + f'{_escape(str(entry.get("phase", "")))}' + "
" + f'
{_escape(reasoning)}
' + f"{actions_html}" + f"{screenshot_html}" + "
" + ) + + +def generate_report(trajectory_dir: str) -> str: + """Build report.html from session.json and history.jsonl only. + + Returns the path to the written report file. + """ + session = _load_session(trajectory_dir) + history = _load_history(trajectory_dir) + + started = session.get("started_at", "") + finished = session.get("finished_at", "") + elapsed = _elapsed_seconds(started, finished) if finished else None + elapsed_str = f"{elapsed}s" if elapsed is not None else "n/a" + + stats_rows = [ + ("Task", session.get("task", "")), + ("Status", session.get("status", "")), + ("Total steps", str(session.get("total_steps", ""))), + ("History events", str(len(history))), + ("Session ID", session.get("session_id", "")), + ("Agent", session.get("agent_type", "")), + ("Platform", session.get("platform", "")), + ("Started at", started), + ("Finished at", finished or "n/a"), + ("Duration", elapsed_str), + ] + if session.get("cloud_session_id"): + stats_rows.append(("Cloud Session", session.get("cloud_session_id", ""))) + + stats_html = "".join( + f'
{_escape(label)}
' + f'
{_escape(str(value))}
' + for label, value in stats_rows + ) + + grouped = defaultdict(list) + for entry in history: + grouped[int(entry.get("step", 0))].append(entry) + + steps_html = "".join( + f'

Step {step}

' + + "".join(_render_entry(trajectory_dir, e) for e in grouped[step]) + + "
" + for step in sorted(grouped.keys()) + ) + + task_title = _escape(session.get("task", "Task Report")) + whole_log_link = "" + if os.path.isfile(os.path.join(trajectory_dir, _WHOLE_LOG_NAME)): + whole_log_link = ( + f'' + ) + + doc = f""" + + + + + {task_title} — Mano Trajectory Report + + + +
+

{task_title}

+
{stats_html}
+
+
+ + +
+
{steps_html or "

No history recorded

"}
+ {whole_log_link} + + + +""" + out_path = os.path.join(trajectory_dir, "report.html") + with open(out_path, "w", encoding="utf-8") as f: + f.write(doc) + return out_path diff --git a/visual/trajectory/tests/__init__.py b/visual/trajectory/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/visual/trajectory/tests/test_action_styles.py b/visual/trajectory/tests/test_action_styles.py new file mode 100644 index 0000000..7006664 --- /dev/null +++ b/visual/trajectory/tests/test_action_styles.py @@ -0,0 +1,31 @@ +"""Tests for action_kind inference.""" + +import unittest + +from visual.trajectory.action_styles import ( + ACTION_KIND_DONE, + ACTION_KIND_FAIL, + ACTION_KIND_INITIALIZING, + ACTION_KIND_KEY, + ACTION_KIND_OPEN_APP, + infer_action_kind, +) + + +class TestActionStyles(unittest.TestCase): + def test_initializing(self): + self.assertEqual(infer_action_kind("Initializing"), ACTION_KIND_INITIALIZING) + + def test_open_app(self): + self.assertEqual(infer_action_kind("Open app: 企业微信"), ACTION_KIND_OPEN_APP) + + def test_key(self): + self.assertEqual(infer_action_kind("key: super"), ACTION_KIND_KEY) + + def test_done_fail(self): + self.assertEqual(infer_action_kind("DONE"), ACTION_KIND_DONE) + self.assertEqual(infer_action_kind("FAIL"), ACTION_KIND_FAIL) + + +if __name__ == "__main__": + unittest.main() diff --git a/visual/trajectory/tests/test_history.py b/visual/trajectory/tests/test_history.py new file mode 100644 index 0000000..159e676 --- /dev/null +++ b/visual/trajectory/tests/test_history.py @@ -0,0 +1,30 @@ +"""Tests for history.jsonl append helper.""" + +import json +import os +import tempfile +import unittest + +from visual.trajectory.history import append_history_line + + +class TestHistory(unittest.TestCase): + def test_append_required_fields(self): + with tempfile.TemporaryDirectory() as tmp: + append_history_line( + tmp, + step=1, + action_desc="key: Enter", + reasoning="submit", + phase="action", + ) + path = os.path.join(tmp, "history.jsonl") + line = json.loads(open(path, encoding="utf-8").read().strip()) + self.assertEqual(line["step"], 1) + self.assertEqual(line["phase"], "action") + self.assertEqual(line["action_kind"], "key") + self.assertIn("timestamp", line) + + +if __name__ == "__main__": + unittest.main() diff --git a/visual/trajectory/tests/test_log_tee.py b/visual/trajectory/tests/test_log_tee.py new file mode 100644 index 0000000..2de7183 --- /dev/null +++ b/visual/trajectory/tests/test_log_tee.py @@ -0,0 +1,68 @@ +"""Tests for trajectory stdout/stderr tee.""" + +import io +import os +import sys +import tempfile +import unittest + +from visual.trajectory import log_tee + + +class TestLogTee(unittest.TestCase): + def setUp(self): + log_tee.uninstall_trajectory_tee() + self._orig_out = sys.stdout + self._orig_err = sys.stderr + + def tearDown(self): + log_tee.uninstall_trajectory_tee() + sys.stdout = self._orig_out + sys.stderr = self._orig_err + + def test_early_buffer_flushed_to_whole_log(self): + log_tee.enable_early_trajectory_buffer() + print("before-dir", end="") + with tempfile.TemporaryDirectory() as tmp: + log_tee.install_trajectory_tee(tmp) + print("after-dir") + log_tee.uninstall_trajectory_tee() + log_path = os.path.join(tmp, "whole.log") + self.assertTrue(os.path.isfile(log_path)) + content = open(log_path, encoding="utf-8").read() + self.assertIn("before-dir", content) + self.assertIn("after-dir", content) + + def test_tee_writes_to_stream_and_file(self): + with tempfile.TemporaryDirectory() as tmp: + log_tee.enable_early_trajectory_buffer() + log_tee.install_trajectory_tee(tmp) + captured = io.StringIO() + real_out = sys.stdout._stream if hasattr(sys.stdout, "_stream") else sys.__stdout__ + sys.stdout = log_tee.TeeWriter(captured, sys.stdout._log_file) # type: ignore[attr-defined] + # simpler: just print and read file + log_tee.uninstall_trajectory_tee() + log_tee.enable_early_trajectory_buffer() + log_tee.install_trajectory_tee(tmp) + print("tee-line") + log_tee.uninstall_trajectory_tee() + content = open(os.path.join(tmp, "whole.log"), encoding="utf-8").read() + self.assertIn("tee-line", content) + + def test_uninstall_stops_file_writes(self): + with tempfile.TemporaryDirectory() as tmp: + log_tee.enable_early_trajectory_buffer() + log_tee.install_trajectory_tee(tmp) + print("line1") + log_tee.uninstall_trajectory_tee() + print("line2") + content = open(os.path.join(tmp, "whole.log"), encoding="utf-8").read() + self.assertIn("line1", content) + self.assertNotIn("line2", content) + + def test_buffer_inactive_when_not_enabled(self): + self.assertFalse(log_tee.is_trajectory_logging_active()) + + +if __name__ == "__main__": + unittest.main() diff --git a/visual/trajectory/tests/test_package_import.py b/visual/trajectory/tests/test_package_import.py new file mode 100644 index 0000000..2da24b8 --- /dev/null +++ b/visual/trajectory/tests/test_package_import.py @@ -0,0 +1,14 @@ +"""Smoke test for trajectory package import.""" + +import unittest + + +class TestPackageImport(unittest.TestCase): + def test_import_without_side_effects(self): + import visual.trajectory # noqa: F401 + + self.assertTrue(True) + + +if __name__ == "__main__": + unittest.main() diff --git a/visual/trajectory/tests/test_report_generator.py b/visual/trajectory/tests/test_report_generator.py new file mode 100644 index 0000000..dea9a28 --- /dev/null +++ b/visual/trajectory/tests/test_report_generator.py @@ -0,0 +1,81 @@ +"""Tests for report.html generation.""" + +import json +import os +import tempfile +import unittest + +from visual.trajectory.history import append_history_line +from visual.trajectory.report_generator import generate_report + + +class TestReportGenerator(unittest.TestCase): + def _fixture_dir(self): + tmp = tempfile.mkdtemp() + os.makedirs(os.path.join(tmp, "screenshots"), exist_ok=True) + session = { + "task": "打开企业微信", + "status": "completed", + "total_steps": 3, + "session_id": "sess-test", + "started_at": "2026-05-18T09:43:59", + "finished_at": "2026-05-18T09:44:21", + "agent_type": "cloud", + "platform": "Windows", + } + with open(os.path.join(tmp, "session.json"), "w", encoding="utf-8") as f: + json.dump(session, f) + append_history_line( + tmp, step=0, action_desc="Initializing", reasoning="init", phase="init" + ) + append_history_line( + tmp, + step=1, + action_desc="key: super", + reasoning="search", + phase="action", + screenshot="screenshots/1.png", + ) + png = os.path.join(tmp, "screenshots", "1.png") + with open(png, "wb") as f: + f.write(b"\x89PNG\r\n\x1a\n") + append_history_line( + tmp, step=3, action_desc="DONE", reasoning="done", phase="terminal" + ) + return tmp + + def test_header_contains_task_and_status(self): + tmp = self._fixture_dir() + path = generate_report(tmp) + html = open(path, encoding="utf-8").read() + self.assertIn("打开企业微信", html) + self.assertIn("completed", html) + + def test_collapsible_screenshot_and_placeholder(self): + tmp = self._fixture_dir() + html = open(generate_report(tmp), encoding="utf-8").read() + self.assertIn("screenshot-details", html) + self.assertIn('screenshots/1.png', html) + append_history_line( + tmp, step=2, action_desc="key: Escape", reasoning="x", phase="action" + ) + html2 = open(generate_report(tmp), encoding="utf-8").read() + self.assertIn("(No screenshot)", html2) + + def test_action_kind_css_classes(self): + tmp = self._fixture_dir() + html = open(generate_report(tmp), encoding="utf-8").read() + self.assertIn("action-key", html) + self.assertIn("action-done", html) + + def test_does_not_read_whole_log(self): + tmp = self._fixture_dir() + whole = os.path.join(tmp, "whole.log") + with open(whole, "w", encoding="utf-8") as f: + f.write("SECRET_TERMINAL_ONLY") + html = open(generate_report(tmp), encoding="utf-8").read() + self.assertNotIn("SECRET_TERMINAL_ONLY", html) + + +if __name__ == "__main__": + unittest.main() diff --git a/visual/view_model/task_view_model.py b/visual/view_model/task_view_model.py index 78e973f..1238cf3 100644 --- a/visual/view_model/task_view_model.py +++ b/visual/view_model/task_view_model.py @@ -115,7 +115,14 @@ def poll_thread(): self.view.root.after(ANIMATION_CONFIG["POLL_INTERVAL"], poll_thread) # ========== Business Methods ========== - def init_task(self, task_name: str, agent: BaseAgent, expected_result: Optional[str] = None, max_steps: int = None) -> bool: + def init_task( + self, + task_name: str, + agent: BaseAgent, + expected_result: Optional[str] = None, + max_steps: int = None, + cloud_session_id: Optional[str] = None, + ) -> bool: """Initialize automation task""" try: import customtkinter as ctk @@ -129,7 +136,12 @@ def _minimize_if_needed(): self.model.on_minimize_panel = lambda: self.view.root.after(0, _minimize_if_needed) # Initialize Model - self.model.init_task(task_name, agent, expected_result=expected_result, max_steps=max_steps) + self.model.init_task( + task_name, agent, + expected_result=expected_result, + max_steps=max_steps, + cloud_session_id=cloud_session_id, + ) # Initialize View self.view.show() diff --git a/visual/vla.py b/visual/vla.py index 661d815..f4c2687 100644 --- a/visual/vla.py +++ b/visual/vla.py @@ -274,6 +274,13 @@ def run_task(task: str, expected_result: str = None, minimize: bool = False, """Run an automation task""" from visual.config.visual_config import BASE_URL, AUTOMATION_CONFIG, API_HEADERS from visual.computer.computer_use_util import get_or_create_device_id + from visual.config.user_config import get_config + + if get_config("save-trajectory") == "true": + from visual.trajectory.log_tee import enable_early_trajectory_buffer + enable_early_trajectory_buffer() + + cloud_session_id = None # Open app/URL before starting (both modes) app_hint = None @@ -334,6 +341,7 @@ def run_task(task: str, expected_result: str = None, minimize: bool = False, data = resp.json() session_id = data["session_id"] + cloud_session_id = session_id print(f"Session created: {session_id}") except Exception as e: @@ -352,10 +360,16 @@ def run_task(task: str, expected_result: str = None, minimize: bool = False, if minimize and view_model.view and view_model.view._ui_initialized: view_model.view.root.after(200, view_model.view._toggle_minimize) - if not view_model.init_task(task, agent, expected_result=expected_result, max_steps=max_steps): + if not view_model.init_task( + task, agent, expected_result=expected_result, max_steps=max_steps, + cloud_session_id=cloud_session_id, + ): print("Failed to initialize visualization overlay.") # Run task directly without UI - view_model.model.init_task(task, agent, expected_result=expected_result, max_steps=max_steps) + view_model.model.init_task( + task, agent, expected_result=expected_result, max_steps=max_steps, + cloud_session_id=cloud_session_id, + ) view_model.model.run_automation_task() return 0 if view_model.model.state.status == "completed" else 1