diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7aa018a..0fb734f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,6 +54,21 @@ jobs: [registry."harbor-core.harbor.svc.cluster.local"] ca=["/etc/ssl/certs/ca-certificates.crt"] + # TEMPORARY, tied to the same fork-build stage in the Dockerfile + # (see homelab#822) -- remove once the fork's fixes land upstream + # and Dockerfile stage 1 reverts to a plain upstream FROM. + # Resolves the fork's CURRENT commit so it's passed as a build-arg + # below: Docker's cache keys on RUN command text, not on what a + # `git clone --branch main` actually fetches, so without this the + # fork-build layer can silently stay cached and stale across runs + # even after the fork gets new commits. + - name: Resolve open-terminal-app-fork HEAD sha + id: fork + run: | + sha=$(git ls-remote https://github.com/dvystrcil/open-terminal-app-fork.git refs/heads/main | cut -f1) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "Building against fork main @ $sha" + - name: Build and push uses: docker/build-push-action@v7 with: @@ -62,6 +77,8 @@ jobs: platforms: ${{ matrix.platform }} push: true provenance: false + build-args: | + FORK_SHA=${{ steps.fork.outputs.sha }} tags: ${{ env.INTERNAL_REGISTRY }}/${{ env.PROJECT }}/${{ env.IMAGE_NAME }}:${{ matrix.platform == 'linux/amd64' && 'dev-amd64' || 'dev-arm64' }} cache-from: type=registry,ref=${{ env.INTERNAL_REGISTRY }}/${{ env.PROJECT }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} cache-to: type=registry,ref=${{ env.INTERNAL_REGISTRY }}/${{ env.PROJECT }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max diff --git a/.python-version b/.python-version deleted file mode 100644 index 6324d40..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.14 diff --git a/CHANGELOG.md b/CHANGELOG.md index cf82e73..564db3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Fixed + +- 🔒🔌📄 **The `open_terminal/` fixes documented in this CHANGELOG were never actually deployed** — this repo's Dockerfile has always been a thin wrapper around a pre-built upstream image (`FROM .../ghcr-proxy/open-webui/open-terminal:latest`); it never built or installed this repo's own vendored `open_terminal/` package. Every fix this CHANGELOG has described as applied to `open_terminal/main.py` (0.20.40's connection-reset fix, 0.20.41's log-retention fix, and others discovered along the way) was real code, tested, and merged here -- but the running container has always been unmodified upstream code plus wrapper tooling, regardless. See [homelab#822](https://github.com/dvystrcil/homelab/issues/822) for the full incident. + +### Changed + +- 🍴 **Now builds from `dvystrcil/open-terminal-app-fork`, not upstream `:latest`, temporarily** — Stage 1 of the Dockerfile builds `open_terminal` from source (a real fork, `git clone` + `pip install .`, mirroring upstream's own Dockerfile) instead of pulling the pre-built upstream image. The fork carries the fixes above plus two more found in the same audit (two-tier process-result expiry; new `insert_after`/`append_to_section`/`append` file endpoints with a defensive `replace_file_content` check), submitted upstream as [open-webui/open-terminal#148](https://github.com/open-webui/open-terminal/pull/148), [#149](https://github.com/open-webui/open-terminal/pull/149), [#150](https://github.com/open-webui/open-terminal/pull/150), [#151](https://github.com/open-webui/open-terminal/pull/151). Revert to a plain `FROM .../open-terminal:latest` once all four merge and a release picks them up. +- 🗑️ **Removed the vendored `open_terminal/` package and its tests** — dead weight now that the actual fixes live in a real fork with a real upstream relationship, not a disconnected local copy that nothing built. `tests/test_actor_env.py` (tests `helpers/bible_bridge.py`, which *is* deployed) is kept; the rest tested only the vendored copy. `pyproject.toml`, `dev.sh`, and `.python-version` (all specific to developing that vendored package) removed too. +- 🔑 **Ported a homelab-specific fix to the fork** (not submitted upstream -- it's specific to our own GitHub App token-file convention): `refresh_github_token_env()` re-reads the current token from disk into the long-lived Python process's own `os.environ` before every subprocess spawn. Closes a gap `BASH_ENV`-based shell-profile sourcing doesn't cover (the plain-shell and PTY spawn paths never source `/etc/profile.d`). See [dvystrcil/homelab#701](https://github.com/dvystrcil/homelab/issues/701). + ## [0.20.41] - 2026-07-31 ### Fixed diff --git a/Dockerfile b/Dockerfile index 42ed8cf..d0f65cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,99 @@ -# Wrapper image — extends ghcr.io/open-webui/open-terminal with -# additional tools and environment configuration. -FROM harbor-core.harbor.svc.cluster.local/ghcr-proxy/open-webui/open-terminal:latest +# === STAGE 1: build open_terminal from our fork, not upstream :latest === +# +# TEMPORARY. dvystrcil/open-terminal-app-fork carries fixes not yet in +# upstream open-webui/open-terminal, submitted as: +# - open-webui/open-terminal#148 -- configurable uvicorn keep-alive +# timeout (fixes intermittent ConnectionResetError) +# - open-webui/open-terminal#149 -- process-log retention security fix +# (a log file with no in-memory record was never pruned) +# - open-webui/open-terminal#150 -- two-tier process-result expiry +# (a slow caller could lose a finished command's result forever) +# - open-webui/open-terminal#151 -- insert_after/append_to_section/ +# append endpoints + a defensive replace_file_content check +# Plus one homelab-specific commit NOT submitted upstream (GH_TOKEN +# refresh from disk before every subprocess spawn -- ties into our own +# entrypoint.sh token-rotation convention, not something upstream has +# any hook for). +# +# Once all four upstream PRs merge and a release picks them up, revert +# this stage and go back to a plain +# `FROM harbor-core.../ghcr-proxy/open-webui/open-terminal:latest` +# (see homelab#822). Mirrors upstream's own Dockerfile build steps +# exactly, substituting a git clone of our fork for `COPY . .`. +FROM python:3.12.13 AS fork-build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + coreutils findutils grep sed gawk diffutils patch \ + less file tree bc man-db \ + curl wget net-tools iputils-ping dnsutils netcat-openbsd socat telnet \ + openssh-client rsync \ + vim nano \ + git \ + build-essential cmake make \ + perl ruby-full lua5.4 \ + jq xmlstarlet sqlite3 \ + ffmpeg pandoc imagemagick texlive-latex-base \ + zip unzip tar gzip bzip2 xz-utils zstd p7zip-full \ + procps htop lsof strace sysstat \ + sudo tmux screen tini iptables ipset dnsmasq \ + ca-certificates gnupg apt-transport-https \ + libcap2-bin \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL https://get.docker.com | sh + +WORKDIR /app + +RUN pip install --no-cache-dir \ + numpy pandas scipy scikit-learn \ + matplotlib seaborn plotly \ + jupyter ipython \ + requests beautifulsoup4 lxml \ + sqlalchemy psycopg2-binary \ + pyyaml toml jsonlines \ + tqdm rich \ + openpyxl weasyprint \ + python-docx python-pptx pypdf csvkit + +# git clone stands in for upstream's `COPY . .` -- our source lives in a +# separate fork repo, not this one. FORK_SHA exists purely to bust +# Docker's build cache: `git clone --branch main` is byte-identical +# text on every build regardless of what commit main actually points +# to, so without something that changes per-build in this RUN step, +# a cached layer silently ships a stale fork clone forever. CI (see +# docker.yml) resolves the fork's current SHA via `git ls-remote` and +# passes it explicitly on every run. +ARG FORK_REF=main +ARG FORK_SHA="" +RUN echo "Building open-terminal-app-fork ref=${FORK_REF} sha=${FORK_SHA:-unpinned}" \ + && git clone --branch "${FORK_REF}" --depth 1 \ + https://github.com/dvystrcil/open-terminal-app-fork.git /build \ + && cd /build \ + && pip install --no-cache-dir . \ + && cp "$(readlink -f "$(which python3)")" /usr/local/bin/python3-ot \ + && setcap cap_setgid+ep /usr/local/bin/python3-ot \ + && sed -i "1s|.*|#!/usr/local/bin/python3-ot|" "$(which open-terminal)" \ + && rm -rf /build + +RUN useradd -m -s /bin/bash user && echo 'user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# Matches upstream's own Dockerfile tail exactly -- these are image +# metadata (ENV/WORKDIR/EXPOSE), not filesystem content, so without +# restating them here stage 2 (FROM fork-build) would silently lose +# them. The old single-stage setup got these for free by inheriting +# straight from the pre-built upstream image; building from source +# here means restating what upstream's own Dockerfile sets. +ENV SHELL=/bin/bash +ENV PATH="/home/user/.local/bin:${PATH}" +WORKDIR /home/user +EXPOSE 8000 + +# === STAGE 2: homelab wrapper -- tools + entrypoint on top of stage 1 === +FROM fork-build USER root diff --git a/README.md b/README.md index 9db29e1..384b2d0 100644 --- a/README.md +++ b/README.md @@ -102,17 +102,19 @@ Plus everything upstream exposes: `OPEN_TERMINAL_PACKAGES`, `OPEN_TERMINAL_PIP_P ## Repository layout ``` -Dockerfile # FROM ghcr.io/open-webui/open-terminal:latest + tooling +Dockerfile # builds open_terminal from dvystrcil/open-terminal-app-fork + tooling entrypoint.sh # secrets resolution, dotfile seeding, helpers, egress, bridge helpers/ bible_bridge.py # multi-project Story Bible HTTP bridge create-pr.sh # five-step PR workflow CONTAINER_TEST_PLAN.md -open_terminal/ # vendored copy of the upstream Python package (reference) -dev.sh # local dev: uv run uvicorn open_terminal.main:app --reload ``` -> The `open_terminal/` source tree is checked in for reference and local debugging via [dev.sh](dev.sh). The published image runs the upstream `open-terminal` binary from the base image, **not** this local copy — to ship code changes you would need to either pin a custom upstream version or restructure the Dockerfile to install from this tree. +> **This repo has no vendored copy of `open_terminal`'s Python source.** An earlier attempt at that (checked-in, tested, and documented in this CHANGELOG as if deployed) was never actually built into the image -- the Dockerfile just pulled a pre-built upstream image the whole time. See [homelab#822](https://github.com/dvystrcil/homelab/issues/822). +> +> The fix: this repo now genuinely builds `open_terminal` from source, via a real fork, [`dvystrcil/open-terminal-app-fork`](https://github.com/dvystrcil/open-terminal-app-fork) -- Dockerfile stage 1 does `git clone` + `pip install .` against it, mirroring upstream's own build. The fork carries a handful of fixes submitted upstream ([open-webui/open-terminal#148](https://github.com/open-webui/open-terminal/pull/148), [#149](https://github.com/open-webui/open-terminal/pull/149), [#150](https://github.com/open-webui/open-terminal/pull/150), [#151](https://github.com/open-webui/open-terminal/pull/151)) plus one homelab-specific patch (GH_TOKEN refresh, not upstream-appropriate). This is meant to be temporary: once those PRs merge into a real upstream release, switch stage 1 back to a plain `FROM .../open-terminal:latest` and the fork goes away. +> +> If you need to change `open_terminal`'s own behavior (not just this wrapper's tooling), make the change in the fork, not here. ## License diff --git a/dev.sh b/dev.sh deleted file mode 100755 index aec9132..0000000 --- a/dev.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -uv run uvicorn open_terminal.main:app --reload diff --git a/open_terminal/__init__.py b/open_terminal/__init__.py deleted file mode 100644 index 5925cdd..0000000 --- a/open_terminal/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""open-terminal — a barebones terminal interaction API.""" diff --git a/open_terminal/__main__.py b/open_terminal/__main__.py deleted file mode 100644 index 5680027..0000000 --- a/open_terminal/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from open_terminal.cli import main - -main() diff --git a/open_terminal/cli.py b/open_terminal/cli.py deleted file mode 100644 index e8be4ca..0000000 --- a/open_terminal/cli.py +++ /dev/null @@ -1,197 +0,0 @@ -import os - -import click -import uvicorn - - -@click.group() -def main(): - """open-terminal — terminal interaction API""" - pass - - -BANNER = r""" - ____ _____ _ _ - / __ \ |_ _| (_) | | - | | | |_ __ ___ _ __ | | ___ _ __ _ __ ___ _ _ __ __ _| | - | | | | '_ \ / _ | '_ \ | |/ _ | '__| '_ ` _ \| | '_ \ / _` | | - | |__| | |_) | __| | | | | | __| | | | | | | | | | | | (_| | | - \____/| .__/ \___|_| |_| \_/\___|_| |_| |_| |_|_|_| |_|\__,_|_| - | | - |_| -""" - - -@main.command() -@click.option("--host", default=None, help="Bind host (default: 0.0.0.0)") -@click.option("--port", default=None, type=int, help="Bind port (default: 8000)") -@click.option( - "--config", - "config_path", - default=None, - type=click.Path(exists=True, dir_okay=False, resolve_path=True), - help="Path to a TOML config file (overrides user-level config location).", -) -@click.option( - "--cwd", - type=click.Path(exists=True, file_okay=False, resolve_path=True, path_type=str), - default=None, - help="Working directory for the server process.", -) -@click.option( - "--api-key", - default="", - envvar="OPEN_TERMINAL_API_KEY", - help="Bearer API key (or set OPEN_TERMINAL_API_KEY env var)", -) -@click.option( - "--cors-allowed-origins", - default="*", - envvar="OPEN_TERMINAL_CORS_ALLOWED_ORIGINS", - help="Allowed CORS origins, comma-separated (default: * for all)", -) -def run( - host: str | None, - port: int | None, - config_path: str | None, - cwd: str | None, - api_key: str, - cors_allowed_origins: str, -): - """Start the sandbox API server.""" - import secrets - - from open_terminal import config - - # Load config files before resolving other settings. - cfg = config.init(config_path) - - # Resolve host/port: CLI flag > config file > built-in default - host = host or cfg.get("host", "0.0.0.0") - port = port if port is not None else cfg.get("port", 8000) - - if cwd: - os.chdir(cwd) - - # Support Docker secrets: load from _FILE variant if no key was given - if not api_key: - file_path = os.environ.get("OPEN_TERMINAL_API_KEY_FILE") - if file_path: - with open(file_path) as f: - api_key = f.read().strip() - - # Fall back to config file value - if not api_key: - api_key = cfg.get("api_key", "") - - generated = not api_key - if not api_key: - api_key = secrets.token_urlsafe(48) - - os.environ["OPEN_TERMINAL_API_KEY"] = api_key - os.environ["OPEN_TERMINAL_CORS_ALLOWED_ORIGINS"] = cors_allowed_origins - - click.echo(BANNER) - - # -- Startup info block -- - local_url = f"http://{'localhost' if host in ('0.0.0.0', '127.0.0.1') else host}:{port}" - - click.echo(f" {click.style('Local:', bold=True)} {click.style(local_url, fg='cyan')}") - if host == "0.0.0.0": - import socket - try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(("8.8.8.8", 80)) - network_ip = s.getsockname()[0] - s.close() - click.echo(f" {click.style('Network:', bold=True)} http://{network_ip}:{port}") - except Exception: - pass - click.echo() - - if generated: - click.echo(f" {click.style('API Key:', bold=True)} {api_key}") - click.echo() - - if host == "0.0.0.0": - click.echo(click.style(" Warning: Listening on all network interfaces.", fg="yellow")) - click.echo(click.style(" Use --host 127.0.0.1 to restrict to this machine.", dim=True)) - click.echo() - - if cors_allowed_origins.strip() == "*": - click.echo(click.style(" ┌─────────────────────────────────────────────────────────────┐", fg="yellow")) - click.echo(click.style(" │ ⚠ CORS is set to '*' (allow all origins) │", fg="yellow")) - click.echo(click.style(" │ │", fg="yellow")) - click.echo(click.style(" │ Any website can make requests to this server. │", fg="yellow")) - click.echo(click.style(" │ For production, restrict with: │", fg="yellow")) - click.echo(click.style(" │ --cors-allowed-origins https://your-domain.com │", fg="yellow")) - click.echo(click.style(" └─────────────────────────────────────────────────────────────┘", fg="yellow")) - click.echo() - - from open_terminal.env import UVICORN_LOOP, UVICORN_TIMEOUT_KEEP_ALIVE - uvicorn.run( - "open_terminal.main:app", - host=host, - port=port, - loop=UVICORN_LOOP, - timeout_keep_alive=UVICORN_TIMEOUT_KEEP_ALIVE, - ) - - -@main.command() -@click.option( - "--transport", - default="stdio", - type=click.Choice(["stdio", "streamable-http"]), - help="MCP transport (default: stdio)", -) -@click.option("--host", default=None, help="Bind host (streamable-http only)") -@click.option( - "--port", default=None, type=int, help="Bind port (streamable-http only)" -) -@click.option( - "--config", - "config_path", - default=None, - type=click.Path(exists=True, dir_okay=False, resolve_path=True), - help="Path to a TOML config file (overrides user-level config location).", -) -@click.option( - "--cwd", - type=click.Path(exists=True, file_okay=False, resolve_path=True, path_type=str), - default=None, - help="Working directory for the server process.", -) -def mcp( - transport: str, - host: str | None, - port: int | None, - config_path: str | None, - cwd: str | None, -): - """Start the MCP server (requires 'pip install open-terminal[mcp]').""" - from open_terminal import config - - cfg = config.init(config_path) - - host = host or cfg.get("host", "0.0.0.0") - port = port if port is not None else cfg.get("port", 8000) - - if cwd: - os.chdir(cwd) - - try: - from open_terminal.mcp_server import mcp as mcp_server - except ImportError: - click.echo( - "Missing MCP dependencies. Install with:\n" - " pip install open-terminal[mcp]", - err=True, - ) - raise SystemExit(1) - - mcp_server.run(transport=transport, host=host, port=port) - - -if __name__ == "__main__": - main() diff --git a/open_terminal/config.py b/open_terminal/config.py deleted file mode 100644 index 54e31ed..0000000 --- a/open_terminal/config.py +++ /dev/null @@ -1,91 +0,0 @@ -"""TOML configuration file support. - -Settings are resolved with this precedence (highest wins): - -1. CLI flags -2. Environment variables / Docker-secrets ``_FILE`` variants -3. User config — ``$XDG_CONFIG_HOME/open-terminal/config.toml`` - (defaults to ``~/.config/open-terminal/config.toml``) -4. System config — ``/etc/open-terminal/config.toml`` -5. Built-in defaults -""" - -import os -import sys -import tomllib -from pathlib import Path - - -def _default_user_config_path() -> Path: - """Return the XDG-compliant user config file path.""" - xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join( - os.path.expanduser("~"), ".config" - ) - return Path(xdg) / "open-terminal" / "config.toml" - - -_SYSTEM_CONFIG_PATH = Path("/etc/open-terminal/config.toml") - - -def load_config(explicit_path: str | None = None) -> dict: - """Load and merge TOML configuration files. - - Parameters - ---------- - explicit_path: - If given, this file replaces the *user-level* lookup. The - system-level config is still loaded underneath. - - Returns - ------- - dict - Merged configuration dictionary. System values are overridden - by user (or explicit) values. - """ - merged: dict = {} - - # 1. System config (lowest priority of the two files) - if _SYSTEM_CONFIG_PATH.is_file(): - try: - merged.update(tomllib.loads(_SYSTEM_CONFIG_PATH.read_text("utf-8"))) - except Exception as exc: - print( - f"Warning: failed to read {_SYSTEM_CONFIG_PATH}: {exc}", - file=sys.stderr, - ) - - # 2. User / explicit config (overrides system) - user_path = Path(explicit_path) if explicit_path else _default_user_config_path() - if user_path.is_file(): - try: - merged.update(tomllib.loads(user_path.read_text("utf-8"))) - except Exception as exc: - # If the user explicitly asked for this file, treat errors as fatal. - if explicit_path: - raise SystemExit(f"Error: failed to read {user_path}: {exc}") from exc - print( - f"Warning: failed to read {user_path}: {exc}", - file=sys.stderr, - ) - - return merged - - -# Module-level merged config, lazily populated by ``init()``. -_config: dict = {} - - -def init(explicit_path: str | None = None) -> dict: - """Load config files and cache the result module-wide. - - This should be called once, early in startup (e.g. from the CLI - entry-point), *before* ``env.py`` constants are evaluated. - """ - global _config - _config = load_config(explicit_path) - return _config - - -def get(key: str, default=None): - """Look up a value from the loaded config.""" - return _config.get(key, default) diff --git a/open_terminal/env.py b/open_terminal/env.py deleted file mode 100644 index fb8aac9..0000000 --- a/open_terminal/env.py +++ /dev/null @@ -1,200 +0,0 @@ -import os - -from open_terminal import config - - -def _resolve_file_env(var: str, default: str = "") -> str: - """Resolve an environment variable with Docker-secrets ``_FILE`` support. - - If ``_FILE`` is set, its value is treated as a path whose contents - supply the variable's value (trailing whitespace is stripped). Setting - *both* ```` and ``_FILE`` is an error. - - This follows the convention established by the official PostgreSQL Docker - image (see https://hub.docker.com/_/postgres#docker-secrets). - """ - value = os.environ.get(var) - file_path = os.environ.get(f"{var}_FILE") - - if value is not None and file_path is not None: - raise ValueError( - f"Both {var} and {var}_FILE are set, but they are mutually exclusive." - ) - - if file_path: - with open(file_path) as f: - return f.read().strip() - - return value if value is not None else default - - -API_KEY = _resolve_file_env("OPEN_TERMINAL_API_KEY", config.get("api_key", "")) -CORS_ALLOWED_ORIGINS = os.environ.get( - "OPEN_TERMINAL_CORS_ALLOWED_ORIGINS", - config.get("cors_allowed_origins", "*"), -) -LOG_DIR = os.environ.get( - "OPEN_TERMINAL_LOG_DIR", - config.get( - "log_dir", - os.path.join( - os.environ.get( - "XDG_STATE_HOME", - os.path.join(os.path.expanduser("~"), ".local", "state"), - ), - "open-terminal", - "logs", - ), - ), -) - -# Comma-separated mime type prefixes for binary files that read_file will return -# as raw binary responses (e.g. "image,audio" or "image/png,image/jpeg"). -BINARY_FILE_MIME_PREFIXES = [ - p.strip() - for p in os.environ.get( - "OPEN_TERMINAL_BINARY_MIME_PREFIXES", - config.get("binary_mime_prefixes", "image"), - ).split(",") - if p.strip() -] - -MAX_TERMINAL_SESSIONS = int( - os.environ.get( - "OPEN_TERMINAL_MAX_SESSIONS", - config.get("max_terminal_sessions", "16"), - ) -) - -ENABLE_TERMINAL = os.environ.get( - "OPEN_TERMINAL_ENABLE_TERMINAL", - str(config.get("enable_terminal", True)), -).lower() not in ("false", "0", "no") - -TERMINAL_TERM = os.environ.get( - "OPEN_TERMINAL_TERM", - config.get("term", "xterm-256color"), -) - -EXECUTE_TIMEOUT: float | None = None -_execute_timeout = os.environ.get( - "OPEN_TERMINAL_EXECUTE_TIMEOUT", - config.get("execute_timeout"), -) -if _execute_timeout is not None: - EXECUTE_TIMEOUT = float(_execute_timeout) - -EXECUTE_DESCRIPTION = os.environ.get( - "OPEN_TERMINAL_EXECUTE_DESCRIPTION", - config.get("execute_description", ""), -) - -# Maximum size (in bytes) for per-process JSONL log files. -# Once exceeded, logging stops for that process (the process keeps running). -MAX_PROCESS_LOG_SIZE = int( - os.environ.get( - "OPEN_TERMINAL_MAX_LOG_SIZE", - config.get("max_log_size", 50_000_000), # 50 MB - ) -) - -# How long (in seconds) to keep finished-process log files on disk. -# After this period, _cleanup_expired() will delete the log file. -PROCESS_LOG_RETENTION: float = float( - os.environ.get( - "OPEN_TERMINAL_LOG_RETENTION", - config.get("log_retention", 604_800), # 7 days - ) -) - -# How long (in seconds) to keep a finished process's in-memory record -# AFTER its result has been successfully delivered at least once via -# GET /execute/{id}/status. Short is fine here -- the caller already has -# what it needs. -PROCESS_EXPIRY: float = float( - os.environ.get( - "OPEN_TERMINAL_PROCESS_EXPIRY", - config.get("process_expiry", 300), # 5 minutes - ) -) - -# How long (in seconds) to keep a finished process's in-memory record if -# NOBODY has successfully polled its status yet. Deliberately much longer -# than PROCESS_EXPIRY: a caller whose own dispatch loop stalls (e.g. the -# known OWUI tool-dispatch hang, ≥300s and reportedly unbounded -- -# dvystrcil/homelab#391) still needs the result to be there when it -# eventually recovers and asks. Expiring on the same short window as a -# delivered result turns a recoverable caller-side hang into a permanent, -# silent loss (dvystrcil/open-terminal#13). -PROCESS_UNDELIVERED_EXPIRY: float = float( - os.environ.get( - "OPEN_TERMINAL_PROCESS_UNDELIVERED_EXPIRY", - config.get("process_undelivered_expiry", 1800), # 30 minutes - ) -) - -# Minimum interval (in seconds) between log flushes during command execution. -# 0 (default) = flush after every chunk (current behaviour). -# Setting this to e.g. 1.0 reduces I/O pressure on high-output commands. -LOG_FLUSH_INTERVAL: float = float( - os.environ.get( - "OPEN_TERMINAL_LOG_FLUSH_INTERVAL", - config.get("log_flush_interval", 0), - ) -) - -# Maximum unflushed buffer (in bytes) before a flush is forced. -# Only relevant when LOG_FLUSH_INTERVAL > 0. 0 = no buffer limit. -LOG_FLUSH_BUFFER: int = int( - os.environ.get( - "OPEN_TERMINAL_LOG_FLUSH_BUFFER", - config.get("log_flush_buffer", 0), - ) -) - -ENABLE_NOTEBOOKS = os.environ.get( - "OPEN_TERMINAL_ENABLE_NOTEBOOKS", - str(config.get("enable_notebooks", True)), -).lower() not in ("false", "0", "no") - -ENABLE_SYSTEM_PROMPT = os.environ.get( - "OPEN_TERMINAL_ENABLE_SYSTEM_PROMPT", - str(config.get("enable_system_prompt", True)), -).lower() not in ("false", "0", "no") - -SYSTEM_PROMPT = os.environ.get( - "OPEN_TERMINAL_SYSTEM_PROMPT", - config.get("system_prompt", ""), -) - -MULTI_USER = os.environ.get( - "OPEN_TERMINAL_MULTI_USER", - str(config.get("multi_user", False)), -).lower() not in ("false", "0", "no", "") - -USER_PREFIX = os.environ.get( - "OPEN_TERMINAL_USER_PREFIX", - config.get("user_prefix", ""), -) - -UVICORN_LOOP = os.environ.get( - "OPEN_TERMINAL_UVICORN_LOOP", - config.get("uvicorn_loop", "auto"), -) - -# uvicorn's own default keep-alive is 5s. Open WebUI's aiohttp client pools -# connections for reuse well beyond that, so on an idle gap it writes to a -# socket the server already closed, surfacing as -# "[Errno 104] Connection reset by peer" on the OWUI side even though the -# request that raced it completed successfully (homelab#709). -UVICORN_TIMEOUT_KEEP_ALIVE = int(os.environ.get( - "OPEN_TERMINAL_UVICORN_TIMEOUT_KEEP_ALIVE", - config.get("uvicorn_timeout_keep_alive", 75), -)) - -OPEN_TERMINAL_INFO = os.environ.get( - "OPEN_TERMINAL_INFO", - config.get("info", ""), -) - - diff --git a/open_terminal/main.py b/open_terminal/main.py deleted file mode 100644 index c1fdb6f..0000000 --- a/open_terminal/main.py +++ /dev/null @@ -1,2078 +0,0 @@ -import asyncio -import hmac -from importlib.metadata import version as _pkg_version -import fnmatch -import json - -import aiofiles -import aiofiles.os -import os -import platform -import re -import shutil -import signal -import socket -import sys -import time -import uuid -from dataclasses import dataclass, field -from typing import Optional - -from fastapi import Depends, FastAPI, File, HTTPException, Query, Request, UploadFile, WebSocket, WebSocketDisconnect -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse, Response -from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer -from pydantic import BaseModel, Field - -from open_terminal.env import API_KEY, BINARY_FILE_MIME_PREFIXES, CORS_ALLOWED_ORIGINS, ENABLE_NOTEBOOKS, ENABLE_SYSTEM_PROMPT, ENABLE_TERMINAL, EXECUTE_DESCRIPTION, EXECUTE_TIMEOUT, LOG_DIR, MAX_TERMINAL_SESSIONS, MULTI_USER, OPEN_TERMINAL_INFO, PROCESS_EXPIRY, PROCESS_LOG_RETENTION, PROCESS_UNDELIVERED_EXPIRY, SYSTEM_PROMPT, TERMINAL_TERM -from open_terminal.utils.runner import PipeRunner, ProcessRunner, create_runner -from open_terminal.utils.fs import UserFS -from open_terminal.utils.github_token import refresh_github_token_env - -if MULTI_USER: - from open_terminal.utils.user_isolation import check_environment, resolve_user - check_environment() - -if not API_KEY: - raise SystemExit( - "\n\033[91m" - " OPEN_TERMINAL_API_KEY is required.\n" - " Set via environment variable or --api-key flag.\n" - "\033[0m" - ) - -try: - import fcntl - import pty - import struct - import subprocess - import termios - - _PTY_AVAILABLE = True -except ImportError: - _PTY_AVAILABLE = False # Windows - - -def get_system_info() -> str: - """Gather runtime system metadata for the OpenAPI description.""" - shell = os.environ.get("SHELL", "/bin/sh") - user_part = f" as user '{os.getenv('USER', 'unknown')}'" if not MULTI_USER else "" - return ( - f"This system is running {platform.system()} {platform.release()} ({platform.machine()}) " - f"on {socket.gethostname()}{user_part} with {shell}. " - f"Python {sys.version.split()[0]} is available." - ) - - -def get_system_prompt() -> str: - """Build a default system prompt for LLM integration.""" - if SYSTEM_PROMPT: - return SYSTEM_PROMPT - - shell = os.environ.get("SHELL", "/bin/sh") - user_part = f" as user '{os.getenv('USER', 'unknown')}'" if not MULTI_USER else "" - - prompt = ( - f"You have access to a computer running {platform.system()} {platform.release()} ({platform.machine()}) " - f'on host "{socket.gethostname()}"{user_part} with {shell}. ' - f"Python {sys.version.split()[0]} is available.\n\n" - "Use your tools to directly interact with the system \u2014 run commands, read and write files, " - "and search the filesystem. " - "Prefer verifying the current state before making changes. " - "When running commands, check the output to confirm success. " - "If a command produces no output, that typically means it succeeded." - ) - - if OPEN_TERMINAL_INFO: - prompt += f"\n\n{OPEN_TERMINAL_INFO}" - - return prompt - - -_EXECUTE_DESCRIPTION = ( - "Run a shell command in the background and return a command ID.\n\n" - + get_system_info() -) -if EXECUTE_DESCRIPTION: - _EXECUTE_DESCRIPTION += "\n\n" + EXECUTE_DESCRIPTION - -bearer_scheme = HTTPBearer(auto_error=False) - - -async def verify_api_key( - credentials: Optional[HTTPAuthorizationCredentials] = Depends(bearer_scheme), -): - if not API_KEY: - return - if not credentials or not hmac.compare_digest(credentials.credentials, API_KEY): - raise HTTPException(status_code=401, detail="Invalid API key") - - -def get_filesystem(request: Request) -> UserFS: - """Build a :class:`UserFS` scoped to the requesting user. - - When multi-user mode is active and the ``X-User-Id`` header is present, - returns a ``UserFS`` that routes all I/O through ``sudo -u``. - Otherwise returns a plain ``UserFS`` using stdlib. - """ - if not MULTI_USER: - return UserFS() - user_id = request.headers.get("x-user-id") - if not user_id: - return UserFS() - username, home = resolve_user(user_id) - return UserFS(username=username, home=home) - - -app = FastAPI( - title="Open Terminal", - description="A remote terminal API.", - version=_pkg_version("open-terminal"), -) -app.add_middleware( - CORSMiddleware, - allow_origins=[o.strip() for o in CORS_ALLOWED_ORIGINS.split(",")], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - - -@app.exception_handler(PermissionError) -async def permission_error_handler(request: Request, exc: PermissionError): - return JSONResponse(status_code=403, content={"detail": str(exc)}) - - -@app.middleware("http") -async def normalize_null_query_params(request: Request, call_next): - """Strip query parameters whose value is the literal string 'null'.""" - from urllib.parse import urlencode - - raw_params = request.query_params.multi_items() - cleaned = [(k, v) for k, v in raw_params if v.lower() != "null"] - if len(cleaned) != len(raw_params): - request.scope["query_string"] = urlencode(cleaned).encode("utf-8") - return await call_next(request) - - -# --------------------------------------------------------------------------- -# Models -# --------------------------------------------------------------------------- - - -class ExecRequest(BaseModel): - command: str = Field( - ..., - description="Shell command to execute. Supports chaining (&&, ||, ;), pipes (|), and redirections.", - json_schema_extra={"examples": ["echo hello", "ls -la && whoami"]}, - ) - cwd: Optional[str] = Field( - None, - description="Working directory for the command. Defaults to the server's current directory if not set.", - ) - env: Optional[dict[str, str]] = Field( - None, - description="Extra environment variables merged into the subprocess environment.", - ) - - -class InputRequest(BaseModel): - input: str = Field( - ..., - description="Text to send to the process's stdin. Include newline characters as needed.", - ) - - -class WriteRequest(BaseModel): - path: str = Field( - ..., - description="Absolute or relative path to write to. Parent directories are created automatically.", - ) - content: str = Field( - ..., - description="Text content to write to the file.", - ) - - -class ReplacementChunk(BaseModel): - target: str = Field( - ..., - description="Exact string to find. Must match precisely, including whitespace.", - ) - replacement: str = Field( - ..., - description="Content to replace the target with.", - ) - start_line: Optional[int] = Field( - None, - description="Narrow the search to lines at or after this (1-indexed).", - ge=1, - ) - end_line: Optional[int] = Field( - None, - description="Narrow the search to lines at or before this (1-indexed).", - ge=1, - ) - allow_multiple: bool = Field( - False, - description="If true, replaces all occurrences. If false, errors when multiple matches are found.", - ) - - -class MkdirRequest(BaseModel): - path: str = Field( - ..., - description="Directory path to create. Parent directories are created automatically.", - ) - - -class MoveRequest(BaseModel): - source: str = Field( - ..., - description="Path to the file or directory to move.", - ) - destination: str = Field( - ..., - description="Destination path (new location).", - ) - - -class ReplaceRequest(BaseModel): - path: str = Field( - ..., - description="Path to the file to modify.", - ) - replacements: list[ReplacementChunk] = Field( - ..., - description="List of find-and-replace operations to apply sequentially.", - ) - - -class InsertAfterRequest(BaseModel): - path: str = Field( - ..., - description="Path to the file to modify.", - ) - anchor: str = Field( - ..., - description=( - "Substring identifying the anchor line. The first line containing this " - "string is the anchor; `content` is inserted on a new line immediately " - "after it. Use a unique-enough substring (typically a full heading " - "line) to avoid ambiguous matches." - ), - ) - content: str = Field( - ..., - description=( - "Text to insert after the anchor line. A trailing newline is added " - "automatically if not present." - ), - ) - allow_multiple: bool = Field( - False, - description=( - "If true, inserts after every matching anchor line. If false (default), " - "errors when more than one match is found." - ), - ) - - -class AppendToSectionRequest(BaseModel): - path: str = Field( - ..., - description="Path to the file to modify.", - ) - heading: str = Field( - ..., - description=( - "Substring identifying the section heading. The first matching markdown " - "heading line is the section start; `content` is appended at the end of " - "the section, immediately before the next heading at equal or shallower " - "depth (or end of file). Example: '## 4. Tactical Deployment Constraints'." - ), - ) - content: str = Field( - ..., - description=( - "Text to append at the end of the matched section. A trailing newline " - "is added automatically if not present." - ), - ) - - -class AppendRequest(BaseModel): - path: str = Field( - ..., - description="Path to the file to append to. The file must already exist.", - ) - content: str = Field( - ..., - description=( - "Text to append at the end of the file. A newline is added before the " - "appended content if the existing file doesn't already end with one." - ), - ) - - - -# --------------------------------------------------------------------------- -# Background process management -# --------------------------------------------------------------------------- - - -@dataclass -class BackgroundProcess: - id: str - command: str - runner: ProcessRunner - status: str = "running" - exit_code: Optional[int] = None - log_task: Optional[asyncio.Task] = field(default=None, repr=False) - finished_at: Optional[float] = field(default=None, repr=False) - delivered_at: Optional[float] = field(default=None, repr=False) - log_path: Optional[str] = field(default=None, repr=False) - - -_processes: dict[str, BackgroundProcess] = {} - - -from open_terminal.utils.log import log_process, read_log - - - - -def _cleanup_expired(): - """Remove finished processes that have expired. - - Two different grace periods (homelab#391 / open-terminal#13): a - process whose finished status has been successfully delivered to a - caller at least once only needs PROCESS_EXPIRY (short -- the caller - already has the result). A process nobody has successfully polled - yet gets PROCESS_UNDELIVERED_EXPIRY instead (much longer), so a - caller whose own dispatch loop stalls doesn't come back to find its - result already gone. - - Also deletes log files older than *LOG_RETENTION_SECONDS*. - """ - now = time.time() - expired = [] - for process_id, background_process in _processes.items(): - if not background_process.finished_at: - continue - if background_process.delivered_at is not None: - if now - background_process.delivered_at > PROCESS_EXPIRY: - expired.append(process_id) - elif now - background_process.finished_at > PROCESS_UNDELIVERED_EXPIRY: - expired.append(process_id) - for process_id in expired: - bp = _processes.pop(process_id) - # Delete the log file if it has exceeded the retention period. - if ( - bp.log_path - and bp.finished_at - and now - bp.finished_at > PROCESS_LOG_RETENTION - ): - try: - os.remove(bp.log_path) - except OSError: - pass - - -_log_sweep_task: "asyncio.Task | None" = None - - -def _sweep_expired_log_files(processes_dir: str, now: float | None = None) -> list[str]: - """Delete *.jsonl files in processes_dir older than PROCESS_LOG_RETENTION. - - Complements _cleanup_expired() above, which only deletes a log file - when it *also* has a matching in-memory BackgroundProcess record -- - but _processes is in-memory and doesn't survive a pod restart, while - the log files live on a persistent volume that does. Any log file - older than the last restart is therefore permanently unreachable by - that path regardless of age. Surfaced via homelab#720: a process log - from ~2 months prior was still present, containing a plaintext - GITHUB_APP_PRIVATE_KEY from a command that had echoed it, because - every restart since had reset _processes to empty. This reads mtimes - directly off disk instead, independent of any process record. - - Returns the list of paths actually deleted (for tests/observability). - """ - now = time.time() if now is None else now - deleted = [] - try: - entries = os.listdir(processes_dir) - except OSError: - return deleted - for name in entries: - if not name.endswith(".jsonl"): - continue - path = os.path.join(processes_dir, name) - try: - if now - os.path.getmtime(path) > PROCESS_LOG_RETENTION: - os.remove(path) - deleted.append(path) - except OSError: - pass - return deleted - - -async def _log_retention_sweep_loop(): - processes_dir = os.path.join(LOG_DIR, "processes") - while True: - await asyncio.sleep(3600) # hourly is plenty for a multi-day window - _sweep_expired_log_files(processes_dir) - - -@app.on_event("startup") -async def _start_log_retention_sweep(): - global _log_sweep_task - _log_sweep_task = asyncio.create_task(_log_retention_sweep_loop()) - - -def _get_process(process_id: str) -> BackgroundProcess: - _cleanup_expired() - background_process = _processes.get(process_id) - if not background_process: - raise HTTPException(status_code=404, detail="Process not found") - return background_process - - -# --------------------------------------------------------------------------- -# Health -# --------------------------------------------------------------------------- - - -@app.get( - "/health", - operation_id="health_check", - summary="Health check", - description="Returns service status. No authentication required.", -) -async def health(): - return {"status": "ok"} - - -# --------------------------------------------------------------------------- -# Config (capability discovery) -# --------------------------------------------------------------------------- - - -@app.get( - "/api/config", - include_in_schema=False, -) -async def get_config(): - """Return server feature flags for client-side discovery.""" - return { - "features": { - "terminal": ENABLE_TERMINAL, - "notebooks": ENABLE_NOTEBOOKS, - "system": ENABLE_SYSTEM_PROMPT, - }, - } - - -if ENABLE_SYSTEM_PROMPT: - - @app.get( - "/system", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], - ) - async def get_system(): - """Return a system prompt for LLM integration.""" - return {"prompt": get_system_prompt()} - - -if OPEN_TERMINAL_INFO: - - @app.get( - "/info", - operation_id="get_info", - summary="Get environment info", - description="Return operator-provided information about this environment. Use this to understand the system you are working with.", - dependencies=[Depends(verify_api_key)], - ) - async def get_info(): - return {"info": OPEN_TERMINAL_INFO} - - -# --------------------------------------------------------------------------- -# Files -# --------------------------------------------------------------------------- - - -@app.get( - "/files/cwd", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def get_cwd(fs: UserFS = Depends(get_filesystem)): - return {"cwd": fs.home} - - -@app.post( - "/files/cwd", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def set_cwd(request: MkdirRequest, fs: UserFS = Depends(get_filesystem)): - target = fs.resolve_path(request.path) - if fs.username: - # In multi-user mode, cwd is per-user; don't touch the global server cwd. - return {"cwd": target} - if not await fs.isdir(target): - raise HTTPException(status_code=404, detail="Directory not found") - try: - os.chdir(target) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - return {"cwd": target} - - -@app.get( - "/files/list", - operation_id="list_files", - summary="List directory contents", - description="Return a structured listing of files and directories at the given path.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "Directory not found."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def list_files( - directory: str = Query(".", description="Directory path to list."), - fs: UserFS = Depends(get_filesystem), -): - target = fs.resolve_path(directory) - if not await fs.isdir(target): - raise HTTPException(status_code=404, detail="Directory not found") - entries = await fs.listdir(target) - return {"dir": target, "entries": entries} - - -@app.get( - "/files/read", - operation_id="read_file", - summary="Read a file", - description="Read a file and return its contents. Supports text files and images (PNG, JPEG, WebP, etc.). For text files you can optionally request a specific line range. Images are returned as binary so you can view and analyze them directly. Use display_file to show a file to the user.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "File not found."}, - 415: {"description": "Unsupported binary file type."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def read_file( - path: str = Query(..., description="Path to the file to read."), - start_line: Optional[int] = Query( - None, description="First line to return (1-indexed, inclusive). Defaults to the beginning of the file.", ge=1 - ), - end_line: Optional[int] = Query( - None, description="Last line to return (1-indexed, inclusive). Defaults to the end of the file.", ge=1 - ), - fs: UserFS = Depends(get_filesystem), -): - - target = fs.resolve_path(path) - if not await fs.isfile(target): - raise HTTPException(status_code=404, detail="File not found") - - try: - content = await fs.read_text(target) - lines = content.splitlines(keepends=True) - except (UnicodeDecodeError, ValueError): - import mimetypes - - raw = await fs.read(target) - mime, _ = mimetypes.guess_type(target) - mime = mime or "application/octet-stream" - - # Try document text extraction (PDF, Office, OpenDocument, etc.) - from open_terminal.utils.documents import EXTRACTORS - - for ext_mime, ext_suffix, extractor in EXTRACTORS: - if (ext_mime and mime == ext_mime) or ( - ext_suffix and target.lower().endswith(ext_suffix) - ): - text = await asyncio.to_thread(extractor, target) - lines = text.splitlines(keepends=True) - start = (start_line or 1) - 1 - end = end_line or len(lines) - return { - "path": target, - "total_lines": len(lines), - "content": "".join(lines[start:end]), - } - - # Return raw binary for allowed mime type prefixes (e.g. image/*) - if any(mime.startswith(prefix) for prefix in BINARY_FILE_MIME_PREFIXES): - return Response(content=raw, media_type=mime) - - # Other binary files: reject (LLMs can't interpret raw bytes) - raise HTTPException( - status_code=415, - detail=f"Unsupported binary file type: {mime} ({len(raw)} bytes)", - ) - - start = (start_line or 1) - 1 - end = end_line or len(lines) - return { - "path": target, - "total_lines": len(lines), - "content": "".join(lines[start:end]), - } - - -@app.get( - "/files/display", - operation_id="display_file", - summary="Display a file to the user", - description="Open a file in the user's file viewer so they can see it. Use this when the user wants to view or look at a file. This does not return file content to you — use read_file if you need to read the content yourself.", - dependencies=[Depends(verify_api_key)], - responses={ - 401: {"description": "Invalid or missing API key."}, - }, -) -async def display_file( - path: str = Query(..., description="Absolute path to the file to display."), - fs: UserFS = Depends(get_filesystem), -): - """Signal that a file should be displayed to the user. - - This endpoint does not serve file content itself. It returns the resolved - path and whether the file exists. The consuming client is responsible for - intercepting this response and presenting the file in its own UI (e.g. - opening a preview pane, launching a viewer, etc.). - """ - target = fs.resolve_path(path) - exists = await fs.isfile(target) - return {"path": target, "exists": exists} - - -@app.get( - "/files/view", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def view_file( - path: str = Query(..., description="Path to the file to view."), - fs: UserFS = Depends(get_filesystem), -): - """Return raw file bytes with the appropriate Content-Type. - - Unlike read_file (which is designed for LLM consumption and restricts - binary types), this endpoint serves any file as-is for UI previewing. - """ - target = fs.resolve_path(path) - if not await fs.isfile(target): - raise HTTPException(status_code=404, detail="File not found") - - import mimetypes - - mime, _ = mimetypes.guess_type(target) - mime = mime or "application/octet-stream" - raw = await fs.read(target) - return Response(content=raw, media_type=mime) - - -@app.post( - "/files/write", - operation_id="write_file", - summary="Write a file", - description="Write text content to a file. Creates parent directories automatically. Overwrites if the file already exists.", - dependencies=[Depends(verify_api_key)], - responses={ - 401: {"description": "Invalid or missing API key."}, - }, -) -async def write_file(request: WriteRequest, fs: UserFS = Depends(get_filesystem)): - target = fs.resolve_path(request.path) - try: - await fs.write(target, request.content) - except (OSError, subprocess.CalledProcessError) as e: - raise HTTPException(status_code=400, detail=str(e)) - return {"path": target, "size": len(request.content.encode())} - - -@app.post( - "/files/mkdir", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def mkdir(request: MkdirRequest, fs: UserFS = Depends(get_filesystem)): - target = fs.resolve_path(request.path) - try: - await fs.mkdir(target) - except (OSError, subprocess.CalledProcessError) as e: - raise HTTPException(status_code=400, detail=str(e)) - return {"path": target} - - -@app.delete( - "/files/delete", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def delete_entry( - path: str = Query(..., description="Path to delete."), - fs: UserFS = Depends(get_filesystem), -): - target = fs.resolve_path(path) - if not await fs.exists(target): - raise HTTPException(status_code=404, detail="Path not found") - is_dir = await fs.isdir(target) - try: - await fs.remove(target) - except (OSError, subprocess.CalledProcessError) as e: - raise HTTPException(status_code=400, detail=str(e)) - return {"path": target, "type": "directory" if is_dir else "file"} - - -@app.post( - "/files/move", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def move_entry(request: MoveRequest, fs: UserFS = Depends(get_filesystem)): - source = fs.resolve_path(request.source) - destination = fs.resolve_path(request.destination) - - if not await fs.exists(source): - raise HTTPException(status_code=404, detail="Source path not found") - - dest_parent = os.path.dirname(destination) - if not await fs.isdir(dest_parent): - raise HTTPException(status_code=400, detail="Destination parent directory not found") - - if await fs.exists(destination): - raise HTTPException(status_code=409, detail="Destination already exists") - - try: - await fs.move(source, destination) - except (OSError, subprocess.CalledProcessError) as e: - raise HTTPException(status_code=400, detail=str(e)) - return {"source": source, "destination": destination} - - -@app.post( - "/files/replace", - operation_id="replace_file_content", - summary="Replace content in a file", - description="Find and replace exact strings in a file. Supports multiple replacements in one call with optional line range narrowing.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "File not found."}, - 400: {"description": "Target string not found or ambiguous match."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def replace_file_content(request: ReplaceRequest, fs: UserFS = Depends(get_filesystem)): - target = fs.resolve_path(request.path) - if not await fs.isfile(target): - raise HTTPException(status_code=404, detail="File not found") - - try: - content = await fs.read_text(target) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - for chunk in request.replacements: - # A target identical to the first non-empty line of the replacement is - # the smoking-gun signature of an "insert new section" being expressed - # as a find-and-replace. Refuse before the generic "Target string not - # found" 400 fires, so the caller learns the right shape instead of - # concluding the file was truncated. Tracked: dvystrcil/homelab#107. - target_stripped = chunk.target.strip() - first_replacement_line = next( - (line.strip() for line in chunk.replacement.splitlines() if line.strip()), - "", - ) - if target_stripped and target_stripped == first_replacement_line: - raise HTTPException( - status_code=400, - detail=( - "Target string is identical to the first line of the " - "replacement. This usually means you want to ADD a new " - "section but expressed it as a find-and-replace. Use " - "insert_after (insert after an existing anchor line), " - "append_to_section (extend the end of a section), or " - "append_file_content (add to end of file) instead. If " - "you're unsure where to put it, read_file first to see " - "the existing anchors." - ), - ) - - if chunk.start_line or chunk.end_line: - lines = content.splitlines(keepends=True) - start = (chunk.start_line or 1) - 1 - end = chunk.end_line or len(lines) - search_region = "".join(lines[start:end]) - else: - search_region = content - - count = search_region.count(chunk.target) - if count == 0: - raise HTTPException( - status_code=400, - detail=f"Target string not found: {chunk.target[:100]!r}", - ) - if count > 1 and not chunk.allow_multiple: - raise HTTPException( - status_code=400, - detail=f"Found {count} occurrences of target string but allow_multiple is false", - ) - - if chunk.start_line or chunk.end_line: - new_region = search_region.replace(chunk.target, chunk.replacement) - lines[start:end] = [new_region] - content = "".join(lines) - else: - content = content.replace(chunk.target, chunk.replacement) - - try: - await fs.write(target, content) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - return {"path": target, "size": len(content.encode())} - - -@app.post( - "/files/insert_after", - operation_id="insert_after_anchor", - summary="Insert content after an anchor line", - description=( - "Insert new content on a new line immediately after a line containing " - "the anchor string. Use this for ADDING content under a known heading " - "(e.g., 'insert after `## 4. Tactical Deployment Constraints`'). For " - "replacing existing content, use replace_file_content. For appending " - "at the end of a section or file, use append_to_section or " - "append_file_content." - ), - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "File not found."}, - 400: {"description": "Anchor not found or ambiguous match."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def insert_after_anchor( - request: InsertAfterRequest, fs: UserFS = Depends(get_filesystem) -): - target = fs.resolve_path(request.path) - if not await fs.isfile(target): - raise HTTPException(status_code=404, detail="File not found") - - try: - content = await fs.read_text(target) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - lines = content.splitlines(keepends=True) - matches = [i for i, line in enumerate(lines) if request.anchor in line] - if not matches: - raise HTTPException( - status_code=400, - detail=f"Anchor not found: {request.anchor[:100]!r}", - ) - if len(matches) > 1 and not request.allow_multiple: - raise HTTPException( - status_code=400, - detail=( - f"Found {len(matches)} lines matching anchor but allow_multiple " - "is false" - ), - ) - - to_insert = request.content - if not to_insert.endswith("\n"): - to_insert += "\n" - - # Iterate in reverse so earlier indices remain valid after each insert. - for idx in reversed(matches): - # If the anchor line itself doesn't end in a newline (only possible at - # EOF when the file has no trailing newline), force one so the inserted - # content lands on its own line. - if not lines[idx].endswith("\n"): - lines[idx] = lines[idx] + "\n" - lines.insert(idx + 1, to_insert) - - new_content = "".join(lines) - try: - await fs.write(target, new_content) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - return {"path": target, "size": len(new_content.encode())} - - -def _markdown_heading_depth(line: str) -> Optional[int]: - """Return the markdown heading depth (number of leading #s) or None. - - A heading is a line whose first non-whitespace token is one or more `#` - characters followed by whitespace or end-of-line. ``#hashtag`` is not a - heading; ``## My Section`` is depth 2. - """ - stripped = line.lstrip() - if not stripped.startswith("#"): - return None - depth = len(stripped) - len(stripped.lstrip("#")) - # Must be followed by whitespace (or EOL) to count as a heading, not e.g. - # `#hashtag`. - remainder = stripped[depth:] - if remainder and not remainder[0].isspace(): - return None - return depth - - -@app.post( - "/files/append_to_section", - operation_id="append_to_section", - summary="Append content at the end of a markdown section", - description=( - "Find the first markdown heading line containing the given substring, " - "then append the new content at the end of that section — immediately " - "before the next heading at equal or shallower depth, or at end of " - "file if no such heading follows. Use this for EXTENDING an existing " - "section without scanning the file for the right insertion line." - ), - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "File not found."}, - 400: {"description": "Heading not found."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def append_to_section( - request: AppendToSectionRequest, fs: UserFS = Depends(get_filesystem) -): - target = fs.resolve_path(request.path) - if not await fs.isfile(target): - raise HTTPException(status_code=404, detail="File not found") - - try: - content = await fs.read_text(target) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - lines = content.splitlines(keepends=True) - heading_idx = None - for i, line in enumerate(lines): - if request.heading in line and _markdown_heading_depth(line) is not None: - heading_idx = i - break - if heading_idx is None: - raise HTTPException( - status_code=400, - detail=f"Markdown heading not found: {request.heading[:100]!r}", - ) - - my_depth = _markdown_heading_depth(lines[heading_idx]) - end_idx = len(lines) - for j in range(heading_idx + 1, len(lines)): - d = _markdown_heading_depth(lines[j]) - if d is not None and d <= my_depth: - end_idx = j - break - - to_insert = request.content - if not to_insert.endswith("\n"): - to_insert += "\n" - # Ensure the line before the insertion ends with a newline so the appended - # block starts on its own line (most relevant when inserting at EOF on a - # file that doesn't end in a newline). - if end_idx > 0 and not lines[end_idx - 1].endswith("\n"): - lines[end_idx - 1] = lines[end_idx - 1] + "\n" - lines.insert(end_idx, to_insert) - - new_content = "".join(lines) - try: - await fs.write(target, new_content) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - return {"path": target, "size": len(new_content.encode())} - - -@app.post( - "/files/append", - operation_id="append_file_content", - summary="Append content to the end of a file", - description=( - "Append the given content at the end of an existing file. A newline " - "is added before the appended content if the existing file doesn't " - "already end with one. Use this for ADDING new top-level sections or " - "trailing content with no anchor. For inserting in the middle of a " - "file, use insert_after or append_to_section." - ), - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "File not found."}, - 400: {"description": "Filesystem error."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def append_file_content( - request: AppendRequest, fs: UserFS = Depends(get_filesystem) -): - target = fs.resolve_path(request.path) - if not await fs.isfile(target): - raise HTTPException(status_code=404, detail="File not found") - - try: - content = await fs.read_text(target) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - new_content = content - if new_content and not new_content.endswith("\n"): - new_content += "\n" - new_content += request.content - if not new_content.endswith("\n"): - new_content += "\n" - - try: - await fs.write(target, new_content) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - - return {"path": target, "size": len(new_content.encode())} - - -@app.get( - "/files/grep", - operation_id="grep_search", - summary="Search file contents", - description="Search for a text pattern across files in a directory. Returns structured matches with file paths, line numbers, and matching lines. Skips binary files.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "Search path not found."}, - 400: {"description": "Invalid regex pattern."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def grep_search( - query: str = Query(..., description="Text or regex pattern to search for."), - path: str = Query(".", description="Directory or file to search in."), - regex: bool = Query(False, description="Treat query as a regex pattern."), - case_insensitive: bool = Query( - False, description="Perform case-insensitive matching." - ), - include: Optional[list[str]] = Query( - None, - description="Glob patterns to filter files (e.g. '*.py'). Files must match at least one pattern.", - ), - match_per_line: bool = Query( - True, - description="If true, return each matching line with line numbers. If false, return only the names of matching files.", - ), - max_results: int = Query( - 50, description="Maximum number of matches to return.", ge=1, le=500 - ), - fs: UserFS = Depends(get_filesystem), -): - target = fs.resolve_path(path) - if not await aiofiles.os.path.exists(target): - raise HTTPException(status_code=404, detail="Search path not found") - - flags = re.IGNORECASE if case_insensitive else 0 - if regex: - try: - pattern = re.compile(query, flags) - except re.error as exc: - raise HTTPException(status_code=400, detail=f"Invalid regex: {exc}") - else: - pattern = re.compile(re.escape(query), flags) - - def _search_sync(): - def _matches_include(filename: str) -> bool: - if not include: - return True - return any(fnmatch.fnmatch(filename, glob) for glob in include) - - matches = [] - truncated = False - - def _search_file(file_path: str): - nonlocal truncated - if truncated: - return - try: - with open(file_path, "r", encoding="utf-8", errors="strict") as f: - for line_number, line in enumerate(f, 1): - if pattern.search(line): - if match_per_line: - matches.append( - { - "file": file_path, - "line": line_number, - "content": line.rstrip("\n\r"), - } - ) - if len(matches) >= max_results: - truncated = True - return - else: - matches.append({"file": file_path}) - if len(matches) >= max_results: - truncated = True - return # one match per file is enough - except (UnicodeDecodeError, ValueError, OSError): - pass # skip binary or unreadable files - - if os.path.isfile(target): - _search_file(target) - else: - for dirpath, dirnames, filenames in os.walk(target): - # Prune directories belonging to other users. - dirnames[:] = [ - d for d in dirnames - if fs.is_path_allowed(os.path.join(dirpath, d)) - ] - if truncated: - break - for filename in sorted(filenames): - if not _matches_include(filename): - continue - full = os.path.join(dirpath, filename) - if not fs.is_path_allowed(full): - continue - _search_file(full) - - return matches, truncated - - matches, truncated = await asyncio.to_thread(_search_sync) - return { - "query": query, - "path": target, - "matches": matches, - "truncated": truncated, - } - - -@app.get( - "/files/glob", - operation_id="glob_search", - summary="Search files by name", - description="Search for files and subdirectories by name within a specified directory using glob patterns. Results will include the relative path, type, size, and modification time.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "Search directory not found."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def glob_search( - pattern: str = Query(..., description="Glob pattern to search for (e.g. '*.py')."), - path: str = Query(".", description="Directory to search within."), - exclude: Optional[list[str]] = Query( - None, description="Glob patterns to exclude from search results." - ), - type: Optional[str] = Query( - "any", - description="Type filter: 'file', 'directory', or 'any'.", - pattern="^(file|directory|any)$", - ), - max_results: int = Query( - 50, description="Maximum number of matches to return.", ge=1, le=500 - ), - fs: UserFS = Depends(get_filesystem), -): - target = fs.resolve_path(path) - if not await aiofiles.os.path.isdir(target): - raise HTTPException(status_code=404, detail="Search directory not found") - - def _glob_sync(): - matches = [] - truncated = False - - for dirpath, dirnames, filenames in os.walk(target): - if truncated: - break - - # Prune directories belonging to other users. - dirnames[:] = [ - d for d in dirnames - if fs.is_path_allowed(os.path.join(dirpath, d)) - ] - - entries = [] - if type in ("any", "directory"): - entries.extend([(d, "directory") for d in dirnames]) - if type in ("any", "file"): - entries.extend([(f, "file") for f in filenames]) - - for name, entry_type in sorted(entries, key=lambda x: x[0]): - if truncated: - break - - full_path = os.path.join(dirpath, name) - rel_path = os.path.relpath(full_path, target) - - # Check inclusion pattern - if not fnmatch.fnmatch(name, pattern) and not fnmatch.fnmatch( - rel_path, pattern - ): - continue - - # Check exclusion patterns - if exclude and any( - fnmatch.fnmatch(name, excl) or fnmatch.fnmatch(rel_path, excl) - for excl in exclude - ): - continue - - try: - file_stat = os.stat(full_path) - matches.append( - { - "path": rel_path, - "type": entry_type, - "size": file_stat.st_size, - "modified": file_stat.st_mtime, - } - ) - - if len(matches) >= max_results: - truncated = True - break - except OSError: - pass - - return matches, truncated - - matches, truncated = await asyncio.to_thread(_glob_sync) - return { - "pattern": pattern, - "path": target, - "matches": matches, - "truncated": truncated, - } - - - - -@app.post( - "/files/upload", - include_in_schema=False, - operation_id="upload_file", - summary="Upload a file", - description="Save a file to the specified path via multipart form data.", - dependencies=[Depends(verify_api_key)], - responses={ - 401: {"description": "Invalid or missing API key."}, - }, -) -async def upload_file( - directory: str = Query(..., description="Destination directory for the file."), - file: UploadFile = File( - ..., description="The file to upload." - ), - fs: UserFS = Depends(get_filesystem), -): - content = await file.read() - filename = os.path.basename(file.filename or "upload") - - directory = fs.resolve_path(directory) - path = os.path.normpath(os.path.join(directory, filename)) - - try: - await fs.mkdir(directory) - await fs.write_bytes(path, content) - except PermissionError as e: - raise HTTPException(status_code=403, detail=str(e)) - except OSError as e: - raise HTTPException(status_code=400, detail=str(e)) - return {"path": path, "size": len(content)} - - -class ArchiveRequest(BaseModel): - paths: list[str] = Field( - ..., - description="List of file or directory paths to include in the ZIP archive.", - ) - - -@app.post( - "/files/archive", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def archive_paths( - request: ArchiveRequest, - fs: UserFS = Depends(get_filesystem), -): - """Bundle files and/or directories into a single ZIP archive.""" - import io - import zipfile - - if not request.paths: - raise HTTPException(status_code=400, detail="No paths provided") - - resolved = [] - for p in request.paths: - target = fs.resolve_path(p) - if not await fs.exists(target): - raise HTTPException(status_code=404, detail=f"Path not found: {p}") - resolved.append(target) - - # Derive a meaningful archive name from the input paths. - if len(resolved) == 1: - archive_name = os.path.basename(resolved[0].rstrip("/\\")) or "archive" - else: - archive_name = "download" - - def _build_zip() -> bytes: - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - for target in resolved: - if os.path.isfile(target): - zf.write(target, os.path.basename(target)) - elif os.path.isdir(target): - dirname = os.path.basename(target.rstrip("/\\")) or "dir" - for dirpath, dirnames, filenames in os.walk(target): - dirnames[:] = [ - d for d in dirnames - if fs.is_path_allowed(os.path.join(dirpath, d)) - ] - for fname in filenames: - full = os.path.join(dirpath, fname) - if not fs.is_path_allowed(full): - continue - arcname = os.path.join( - dirname, os.path.relpath(full, target) - ) - zf.write(full, arcname) - return buf.getvalue() - - data = await asyncio.to_thread(_build_zip) - return Response( - content=data, - media_type="application/zip", - headers={ - "Content-Disposition": f'attachment; filename="{archive_name}.zip"', - }, - ) - - -# --------------------------------------------------------------------------- -# Execute -# --------------------------------------------------------------------------- - - -@app.get( - "/execute", - operation_id="list_processes", - summary="List running commands", - description="Returns a list of all tracked background processes, including running, done, and killed.", - dependencies=[Depends(verify_api_key)], - responses={ - 401: {"description": "Invalid or missing API key."}, - }, -) -async def list_processes(): - _cleanup_expired() - return [ - { - "id": background_process.id, - "command": background_process.command, - "status": background_process.status, - "exit_code": background_process.exit_code, - "log_path": background_process.log_path, - } - for background_process in _processes.values() - ] - - -@app.post( - "/execute", - operation_id="run_command", - summary="Execute a command", - description=_EXECUTE_DESCRIPTION, - dependencies=[Depends(verify_api_key)], - responses={ - 401: {"description": "Invalid or missing API key."}, - }, -) -async def execute( - http_request: Request, - request: ExecRequest, - wait: Optional[float] = Query( - None, - description="Seconds to wait for the command to finish before returning. If the command completes in time, output is included inline. Null to return immediately.", - ge=0, - le=300, - ), - tail: Optional[int] = Query( - None, - description="Return only the last N output entries. Useful to limit response size when only recent output matters.", - ge=1, - ), -): - fs = get_filesystem(http_request) - cwd = fs.resolve_path(request.cwd) if request.cwd else (fs.home if fs.username else None) - - refresh_github_token_env() - subprocess_env = {**os.environ, **request.env} if request.env else None - runner = await create_runner( - request.command, cwd, subprocess_env, run_as_user=fs.username - ) - - process_id = time.strftime("%Y%m%d-%H%M%S-") + uuid.uuid4().hex[:6] - log_path = os.path.join(LOG_DIR, "processes", f"{process_id}.jsonl") - background_process = BackgroundProcess( - id=process_id, command=request.command, runner=runner, log_path=log_path - ) - background_process.log_task = asyncio.create_task(log_process(background_process)) - _processes[process_id] = background_process - - if wait is None and EXECUTE_TIMEOUT: - wait = EXECUTE_TIMEOUT - if wait is not None: - try: - await asyncio.wait_for( - asyncio.shield(background_process.log_task), timeout=wait - ) - except asyncio.TimeoutError: - pass - - output, next_offset, truncated = await read_log( - background_process.log_path, offset=0, tail=tail - ) - - return { - "id": process_id, - "command": request.command, - "status": background_process.status, - "exit_code": background_process.exit_code, - "output": output, - "truncated": truncated, - "next_offset": next_offset, - "log_path": background_process.log_path, - } - - -@app.get( - "/execute/{process_id}/status", - operation_id="get_process_status", - summary="Get command status and output", - description="Returns new output since the last poll, process status, and exit code. Output is drained on read to keep memory bounded.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "Process not found."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def get_status( - process_id: str, - wait: Optional[float] = Query( - None, - description="Seconds to wait for the process to finish before returning. Returns early if the process exits. Null to return immediately.", - ge=0, - le=300, - ), - offset: int = Query( - 0, - description="Number of output entries to skip. Use next_offset from the previous response to get only new output.", - ge=0, - ), - tail: Optional[int] = Query( - None, - description="Return only the last N output entries. Useful to limit response size when only recent output matters.", - ge=1, - ), -): - background_process = _get_process(process_id) - - if wait is None and EXECUTE_TIMEOUT: - wait = EXECUTE_TIMEOUT - if wait is not None and background_process.status == "running": - try: - await asyncio.wait_for( - asyncio.shield(background_process.log_task), timeout=wait - ) - except asyncio.TimeoutError: - pass - - output, next_offset, truncated = await read_log( - background_process.log_path, offset=offset, tail=tail - ) - - if background_process.status != "running" and background_process.delivered_at is None: - # First successful read of a finished process's status -- from - # here on the short PROCESS_EXPIRY grace period applies instead - # of PROCESS_UNDELIVERED_EXPIRY (open-terminal#13). - background_process.delivered_at = time.time() - - return { - "id": background_process.id, - "command": background_process.command, - "status": background_process.status, - "exit_code": background_process.exit_code, - "output": output, - "truncated": truncated, - "next_offset": next_offset, - "log_path": background_process.log_path, - } - - -@app.post( - "/execute/{process_id}/input", - operation_id="send_process_input", - summary="Send input to a running command", - description="Write text to the process's stdin. Include newline characters as needed.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "Process not found."}, - 400: {"description": "Process has already exited or stdin is closed."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def send_input(process_id: str, body: InputRequest): - background_process = _get_process(process_id) - if background_process.status != "running": - raise HTTPException(status_code=400, detail="Process has already exited") - - # Convert literal escape sequences (\n, \x03 for Ctrl-C, etc.) into real - # characters — LLMs often emit these as literal strings. - text = body.input.encode("raw_unicode_escape").decode("unicode_escape") - - try: - background_process.runner.write_input(text.encode()) - if isinstance(background_process.runner, PipeRunner): - await background_process.runner.drain_input() - except (BrokenPipeError, ConnectionResetError, OSError): - raise HTTPException(status_code=400, detail="Process stdin is closed") - - return {"status": "ok"} - - -@app.delete( - "/execute/{process_id}", - operation_id="kill_process", - summary="Kill a running command", - description="Terminate the process. Sends SIGTERM by default for graceful shutdown. Use force=true to send SIGKILL.", - dependencies=[Depends(verify_api_key)], - responses={ - 404: {"description": "Process not found."}, - 401: {"description": "Invalid or missing API key."}, - }, -) -async def kill_process( - process_id: str, - force: bool = Query(False, description="Send SIGKILL instead of SIGTERM."), -): - background_process = _get_process(process_id) - if background_process.status == "running": - background_process.runner.kill(force=force) - exit_code = await background_process.runner.wait() - background_process.runner.close() - background_process.status = "killed" - background_process.exit_code = exit_code - del _processes[process_id] - return {"status": "killed"} - - -# --------------------------------------------------------------------------- -# Port detection & proxy -# --------------------------------------------------------------------------- - -from open_terminal.utils.port import detect_listening_ports, get_descendant_pids - -@app.get( - "/ports", - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def list_ports(request: Request): - """Return TCP ports currently listening on localhost. - - In multi-user mode, only shows ports owned by the requesting user. - In single-user mode, shows ports owned by descendant processes. - """ - all_ports = await asyncio.to_thread(detect_listening_ports) - - try: - fs = get_filesystem(request) - except Exception: - # User provisioning failed (e.g. useradd rejected in restricted - # container runtimes). An unprovisioned user has no ports. - return {"ports": []} - - if fs.username: - # Filter by user UID - import pwd - try: - user_uid = pwd.getpwnam(fs.username).pw_uid - all_ports = [p for p in all_ports if p.get("uid") == user_uid] - except KeyError: - all_ports = [] - else: - own_pid = os.getpid() - descendant_pids = await asyncio.to_thread(get_descendant_pids, own_pid) - all_ports = [p for p in all_ports if p.get("pid") in descendant_pids] - - # Strip uid from response (internal detail) - for p in all_ports: - p.pop("uid", None) - - return {"ports": all_ports} - - -# -- Port proxy client (reused across requests) -- -_port_proxy_client = None - - -async def _get_port_proxy_client(): - global _port_proxy_client - if _port_proxy_client is None: - import httpx - _port_proxy_client = httpx.AsyncClient( - timeout=httpx.Timeout(300.0, connect=5.0), - follow_redirects=False, - ) - return _port_proxy_client - - -@app.api_route( - "/proxy/{port}/{path:path}", - methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], - include_in_schema=False, - dependencies=[Depends(verify_api_key)], -) -async def port_proxy(port: int, path: str, request: Request): - """Reverse-proxy a request to localhost:{port}/{path}.""" - if port < 1 or port > 65535: - raise HTTPException(status_code=422, detail="Port must be between 1 and 65535") - - target_url = f"http://localhost:{port}/{path}" - if request.query_params: - target_url += f"?{request.query_params}" - - # Forward headers, stripping hop-by-hop and host. - headers = dict(request.headers) - for h in ("host", "transfer-encoding", "connection", "authorization"): - headers.pop(h, None) - - body = await request.body() - - import httpx - - client = await _get_port_proxy_client() - try: - upstream = await client.request( - method=request.method, - url=target_url, - headers=headers, - content=body or None, - ) - except httpx.ConnectError: - raise HTTPException( - status_code=502, - detail=f"Connection refused: localhost:{port}", - ) - except httpx.TimeoutException: - raise HTTPException( - status_code=504, - detail=f"Timeout connecting to localhost:{port}", - ) - - response_headers = dict(upstream.headers) - for h in ("transfer-encoding", "connection", "content-encoding", "content-length"): - response_headers.pop(h, None) - - return Response( - content=upstream.content, - status_code=upstream.status_code, - headers=response_headers, - ) - - -# --------------------------------------------------------------------------- -# Interactive terminal sessions (resource-oriented API) -# --------------------------------------------------------------------------- - -if ENABLE_TERMINAL: - - import uuid as _uuid - from datetime import datetime as _datetime - from fastapi.responses import JSONResponse - - try: - import select as _select - except ImportError: - _select = None # Not available on all platforms in all contexts - - # Determine terminal backend: prefer Unix PTY, then pywinpty, else None - if _PTY_AVAILABLE: - _TERMINAL_BACKEND = "pty" - else: - try: - from winpty import PtyProcess as _WinPtyProcess - - _TERMINAL_BACKEND = "winpty" - except ImportError: - _TERMINAL_BACKEND = None - - # Active terminal sessions: {id: {...}} - _terminal_sessions: dict[str, dict] = {} - - - def _cleanup_session(session_id: str): - """Clean up a terminal session's resources. - - For PTY sessions the shell is spawned with ``start_new_session=True``, - giving it a dedicated process group. We signal the *entire* group so - that background jobs started inside the terminal (e.g. ``sleep 999 &``) - are also reaped, and always call ``process.wait()`` to avoid zombies. - """ - session = _terminal_sessions.pop(session_id, None) - if session is None: - return - - backend = session.get("backend") - - if backend == "pty": - try: - os.close(session["master_fd"]) - except OSError: - pass - - process = session["process"] - if process.poll() is None: - # Signal the whole process group first (graceful). - try: - os.killpg(process.pid, signal.SIGTERM) - except (ProcessLookupError, PermissionError): - pass - try: - process.wait(timeout=3) - except subprocess.TimeoutExpired: - # Forceful kill of the entire group. - try: - os.killpg(process.pid, signal.SIGKILL) - except (ProcessLookupError, PermissionError): - pass - process.wait() - - elif backend == "winpty": - pty_proc = session["pty_process"] - if pty_proc.isalive(): - pty_proc.terminate() - - - @app.post("/api/terminals", dependencies=[Depends(verify_api_key)], include_in_schema=False) - async def create_terminal(request: Request): - """Create a new terminal session and return its ID.""" - if _TERMINAL_BACKEND is None: - return JSONResponse( - {"error": "PTY not available on this platform (install pywinpty on Windows)"}, - status_code=503, - ) - - # Prune dead sessions before checking limit - if _TERMINAL_BACKEND == "pty": - dead = [sid for sid, s in _terminal_sessions.items() if s["process"].poll() is not None] - else: - dead = [sid for sid, s in _terminal_sessions.items() if not s["pty_process"].isalive()] - for sid in dead: - _cleanup_session(sid) - - if len(_terminal_sessions) >= MAX_TERMINAL_SESSIONS: - return JSONResponse( - {"error": f"Maximum number of terminal sessions ({MAX_TERMINAL_SESSIONS}) reached"}, - status_code=429, - ) - - session_id = str(_uuid.uuid4())[:8] - - if _TERMINAL_BACKEND == "pty": - try: - master_fd, slave_fd = pty.openpty() - except OSError: - return JSONResponse( - {"error": "Out of PTY devices — too many active terminals or processes"}, - status_code=503, - ) - - try: - fcntl.ioctl(slave_fd, termios.TIOCSWINSZ, struct.pack("HHHH", 24, 80, 0, 0)) - - fs = get_filesystem(request) - if fs.username: - shell_cmd = [ - "script", "-qc", - f"sudo -i -u {fs.username}", - "/dev/null", - ] - cwd = fs.home - else: - shell_cmd = [os.environ.get("SHELL", "/bin/sh")] - cwd = os.getcwd() - - refresh_github_token_env() - spawn_env = os.environ.copy() - spawn_env.setdefault("TERM", TERMINAL_TERM) - process = subprocess.Popen( - shell_cmd, - stdin=slave_fd, - stdout=slave_fd, - stderr=slave_fd, - cwd=cwd, - env=spawn_env, - start_new_session=True, - ) - except Exception: - os.close(slave_fd) - os.close(master_fd) - raise - os.close(slave_fd) - - # Set non-blocking - flags = fcntl.fcntl(master_fd, fcntl.F_GETFL) - fcntl.fcntl(master_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - - _terminal_sessions[session_id] = { - "backend": "pty", - "master_fd": master_fd, - "process": process, - "created_at": _datetime.utcnow().isoformat() + "Z", - "pid": process.pid, - } - - else: # winpty - shell = os.environ.get("COMSPEC", "cmd.exe") - spawn_env = os.environ.copy() - spawn_env.setdefault("TERM", TERMINAL_TERM) - pty_proc = _WinPtyProcess.spawn( - [shell], - cwd=os.getcwd(), - env=spawn_env, - dimensions=(24, 80), - ) - _terminal_sessions[session_id] = { - "backend": "winpty", - "pty_process": pty_proc, - "created_at": _datetime.utcnow().isoformat() + "Z", - "pid": pty_proc.pid, - } - - session = _terminal_sessions[session_id] - return { - "id": session_id, - "created_at": session["created_at"], - "pid": session["pid"], - } - - - def _session_is_alive(session: dict) -> bool: - """Check if a terminal session's process is still running.""" - if session["backend"] == "pty": - return session["process"].poll() is None - else: - return session["pty_process"].isalive() - - - @app.get("/api/terminals", dependencies=[Depends(verify_api_key)], include_in_schema=False) - async def list_terminals(request: Request): - """List active terminal sessions.""" - result = [] - to_remove = [] - for sid, session in _terminal_sessions.items(): - if not _session_is_alive(session): - to_remove.append(sid) - continue - result.append({ - "id": sid, - "created_at": session["created_at"], - "pid": session["pid"], - }) - for sid in to_remove: - _cleanup_session(sid) - return result - - - @app.get("/api/terminals/{session_id}", dependencies=[Depends(verify_api_key)], include_in_schema=False) - async def get_terminal(session_id: str, request: Request): - """Get info about a terminal session.""" - session = _terminal_sessions.get(session_id) - if session is None: - return JSONResponse({"error": "Session not found"}, status_code=404) - if not _session_is_alive(session): - _cleanup_session(session_id) - return JSONResponse({"error": "Session not found"}, status_code=404) - return { - "id": session_id, - "created_at": session["created_at"], - "pid": session["pid"], - } - - - @app.delete("/api/terminals/{session_id}", dependencies=[Depends(verify_api_key)], include_in_schema=False) - async def delete_terminal(session_id: str, request: Request): - """Kill and remove a terminal session.""" - if session_id not in _terminal_sessions: - return JSONResponse({"error": "Session not found"}, status_code=404) - _cleanup_session(session_id) - return {"status": "deleted"} - - - @app.websocket("/api/terminals/{session_id}") - async def ws_terminal(ws: WebSocket, session_id: str): - """Attach to an existing terminal session via WebSocket. - - Authentication is via **first-message auth**: after connecting, the client - must send a JSON text frame as its first message:: - - {"type": "auth", "token": ""} - - The server validates the token and closes the connection if invalid. - After authentication, the client sends keystrokes as **binary** frames - and receives PTY output as binary frames. - - To resize, send a **text** JSON frame:: - - {"type": "resize", "cols": 120, "rows": 40} - """ - session = _terminal_sessions.get(session_id) - if session is None: - await ws.close(code=4004, reason="Session not found") - return - - if not _session_is_alive(session): - _cleanup_session(session_id) - await ws.close(code=4004, reason="Session has ended") - return - - await ws.accept() - - # First-message authentication - if API_KEY: - try: - msg = await asyncio.wait_for(ws.receive_text(), timeout=10.0) - payload = json.loads(msg) - if payload.get("type") != "auth" or not hmac.compare_digest(payload.get("token", ""), API_KEY): - await ws.close(code=4001, reason="Invalid API key") - return - except (asyncio.TimeoutError, json.JSONDecodeError, Exception): - await ws.close(code=4001, reason="Auth timeout or invalid payload") - return - - backend = session["backend"] - loop = asyncio.get_event_loop() - stop_event = asyncio.Event() - - # --- Platform-specific read/write/resize helpers --- - - if backend == "pty": - master_fd = session["master_fd"] - process = session["process"] - - def _blocking_read(): - """Read from PTY using select() so we don't block forever.""" - while not stop_event.is_set(): - try: - rlist, _, _ = _select.select([master_fd], [], [], 0.1) - if rlist: - return os.read(master_fd, 4096) - except (OSError, ValueError): - return b"" - return b"" - - def _check_alive(): - return process.poll() is None - - def _write_data(data: bytes): - os.write(master_fd, data) - - def _do_resize(rows: int, cols: int): - fcntl.ioctl( - master_fd, - termios.TIOCSWINSZ, - struct.pack("HHHH", rows, cols, 0, 0), - ) - - else: # winpty - pty_proc = session["pty_process"] - - def _blocking_read(): - """Read from WinPTY process.""" - try: - data = pty_proc.read(4096) - return data.encode(errors="replace") if data else b"" - except EOFError: - return b"" - except Exception: - return b"" - - def _check_alive(): - return pty_proc.isalive() - - def _write_data(data: bytes): - pty_proc.write(data.decode(errors="replace")) - - def _do_resize(rows: int, cols: int): - pty_proc.setwinsize(rows, cols) - - # --- Reader / writer tasks --- - - async def _pty_reader(): - """Forward PTY output -> WebSocket.""" - try: - while not stop_event.is_set(): - data = await loop.run_in_executor(None, _blocking_read) - if not data: - if stop_event.is_set(): - break - if not _check_alive(): - break - continue - try: - await ws.send_bytes(data) - except Exception: - break - finally: - pass - - reader_task = asyncio.create_task(_pty_reader()) - - try: - while True: - msg = await ws.receive() - if msg["type"] == "websocket.disconnect": - break - elif "bytes" in msg and msg["bytes"]: - await loop.run_in_executor(None, _write_data, msg["bytes"]) - elif "text" in msg and msg["text"]: - try: - payload = json.loads(msg["text"]) - if payload.get("type") == "resize": - cols = payload.get("cols", 80) - rows = payload.get("rows", 24) - _do_resize(rows, cols) - except (json.JSONDecodeError, KeyError): - pass - except WebSocketDisconnect: - pass - finally: - stop_event.set() - reader_task.cancel() - try: - await reader_task - except (asyncio.CancelledError, Exception): - pass - # Clean up session on disconnect - _cleanup_session(session_id) - - -# --------------------------------------------------------------------------- -# Notebook execution (optional) -# --------------------------------------------------------------------------- - -if ENABLE_NOTEBOOKS: - from open_terminal.utils.notebooks import create_notebooks_router - - app.include_router(create_notebooks_router(verify_api_key)) - diff --git a/open_terminal/mcp_server.py b/open_terminal/mcp_server.py deleted file mode 100644 index 5a9cc6e..0000000 --- a/open_terminal/mcp_server.py +++ /dev/null @@ -1,7 +0,0 @@ -"""MCP server — exposes every FastAPI endpoint as an MCP tool.""" - -from fastmcp import FastMCP - -from open_terminal.main import app - -mcp = FastMCP.from_fastapi(app=app, name="Open Terminal") diff --git a/open_terminal/utils/__init__.py b/open_terminal/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/open_terminal/utils/documents.py b/open_terminal/utils/documents.py deleted file mode 100644 index 6629bc1..0000000 --- a/open_terminal/utils/documents.py +++ /dev/null @@ -1,240 +0,0 @@ -"""Document text extraction utilities. - -Extracts readable text from binary document formats so LLMs can -consume their content. Each ``extract_*`` function takes a file path -and returns the document's text as a plain string. - -All libraries used are permissively licensed (MIT / BSD). -""" - -import zipfile - - -def extract_pdf(file_path: str) -> str: - """Extract text from a PDF file.""" - from pypdf import PdfReader - - reader = PdfReader(file_path) - return "\n".join(page.extract_text() or "" for page in reader.pages) - - -def extract_docx(file_path: str) -> str: - """Extract text from a Word (.docx) file.""" - from docx import Document as DocxDocument - - doc = DocxDocument(file_path) - parts = [] - for para in doc.paragraphs: - parts.append(para.text) - for table in doc.tables: - for row in table.rows: - parts.append("\t".join(cell.text for cell in row.cells)) - return "\n".join(parts) - - -def extract_xlsx(file_path: str) -> str: - """Extract text from an Excel (.xlsx) file.""" - from openpyxl import load_workbook - - wb = load_workbook(file_path, read_only=True, data_only=True) - parts = [] - for sheet in wb.worksheets: - parts.append(f"--- {sheet.title} ---") - for row in sheet.iter_rows(values_only=True): - parts.append("\t".join(str(c) if c is not None else "" for c in row)) - wb.close() - return "\n".join(parts) - - -def extract_pptx(file_path: str) -> str: - """Extract text from a PowerPoint (.pptx) file.""" - from pptx import Presentation - - prs = Presentation(file_path) - parts = [] - for i, slide in enumerate(prs.slides, 1): - parts.append(f"--- Slide {i} ---") - for shape in slide.shapes: - if shape.has_text_frame: - parts.append(shape.text_frame.text) - return "\n".join(parts) - - -def extract_rtf(file_path: str) -> str: - """Extract text from a Rich Text Format (.rtf) file.""" - from striprtf.striprtf import rtf_to_text - - with open(file_path, "rb") as f: - raw = f.read() - return rtf_to_text(raw.decode("utf-8", errors="replace")) - - -def extract_xls(file_path: str) -> str: - """Extract text from a legacy Excel (.xls) file.""" - import xlrd - - wb = xlrd.open_workbook(file_path) - parts = [] - for sheet in wb.sheets(): - parts.append(f"--- {sheet.name} ---") - for row_idx in range(sheet.nrows): - parts.append("\t".join( - str(sheet.cell_value(row_idx, col_idx)) - for col_idx in range(sheet.ncols) - )) - return "\n".join(parts) - - -def extract_odt(file_path: str) -> str: - """Extract text from an OpenDocument Text (.odt) file.""" - from lxml import etree - - with zipfile.ZipFile(file_path) as zf: - with zf.open("content.xml") as f: - tree = etree.parse(f) - ns = "urn:oasis:names:tc:opendocument:xmlns:text:1.0" - return "\n".join( - "".join(p.itertext()) - for p in tree.iter(f"{{{ns}}}p") - ) - - -def extract_ods(file_path: str) -> str: - """Extract text from an OpenDocument Spreadsheet (.ods) file.""" - from lxml import etree - - with zipfile.ZipFile(file_path) as zf: - with zf.open("content.xml") as f: - tree = etree.parse(f) - ns_table = "urn:oasis:names:tc:opendocument:xmlns:table:1.0" - ns_text = "urn:oasis:names:tc:opendocument:xmlns:text:1.0" - parts = [] - for table in tree.iter(f"{{{ns_table}}}table"): - name = table.get(f"{{{ns_table}}}name", "Sheet") - parts.append(f"--- {name} ---") - for row in table.iter(f"{{{ns_table}}}table-row"): - cells = [] - for cell in row.iter(f"{{{ns_table}}}table-cell"): - cell_text = " ".join( - "".join(p.itertext()) - for p in cell.iter(f"{{{ns_text}}}p") - ) - cells.append(cell_text) - parts.append("\t".join(cells)) - return "\n".join(parts) - - -def extract_odp(file_path: str) -> str: - """Extract text from an OpenDocument Presentation (.odp) file.""" - from lxml import etree - - with zipfile.ZipFile(file_path) as zf: - with zf.open("content.xml") as f: - tree = etree.parse(f) - ns_draw = "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" - ns_text = "urn:oasis:names:tc:opendocument:xmlns:text:1.0" - parts = [] - for i, page in enumerate(tree.iter(f"{{{ns_draw}}}page"), 1): - parts.append(f"--- Slide {i} ---") - for p in page.iter(f"{{{ns_text}}}p"): - text = "".join(p.itertext()).strip() - if text: - parts.append(text) - return "\n".join(parts) - - -def extract_epub(file_path: str) -> str: - """Extract text from an EPUB e-book.""" - from lxml import etree - - parts = [] - with zipfile.ZipFile(file_path) as zf: - # Parse the container to find the root file - with zf.open("META-INF/container.xml") as cf: - container = etree.parse(cf) - ns_container = "urn:oasis:names:tc:opendocument:xmlns:container" - rootfile = container.find(f".//{{{ns_container}}}rootfile") - if rootfile is None: - rootfile = container.xpath("//*[local-name()='rootfile']") - rootfile = rootfile[0] if rootfile else None - - if rootfile is not None: - opf_path = rootfile.get("full-path", "") - opf_dir = opf_path.rsplit("/", 1)[0] + "/" if "/" in opf_path else "" - with zf.open(opf_path) as opf_file: - opf = etree.parse(opf_file) - spine_ids = [ - item.get("idref") - for item in opf.xpath("//*[local-name()='itemref']") - ] - manifest = { - item.get("id"): item.get("href") - for item in opf.xpath("//*[local-name()='item']") - } - for idref in spine_ids: - href = manifest.get(idref, "") - item_path = opf_dir + href if not href.startswith("/") else href.lstrip("/") - try: - with zf.open(item_path) as html_file: - html_tree = etree.parse(html_file, etree.HTMLParser()) - body = html_tree.find(".//body") - if body is not None: - text = "".join(body.itertext()) - parts.append(text.strip()) - except (KeyError, etree.XMLSyntaxError): - continue - else: - for name in zf.namelist(): - if name.endswith((".html", ".xhtml", ".htm")): - try: - with zf.open(name) as html_file: - html_tree = etree.parse(html_file, etree.HTMLParser()) - body = html_tree.find(".//body") - if body is not None: - text = "".join(body.itertext()) - parts.append(text.strip()) - except etree.XMLSyntaxError: - continue - return "\n\n".join(parts) - - -def extract_eml(file_path: str) -> str: - """Extract text from an email message (.eml).""" - import email - from email import policy - - with open(file_path, "rb") as f: - msg = email.message_from_binary_file(f, policy=policy.default) - parts = [] - for header in ("From", "To", "Cc", "Date", "Subject"): - val = msg.get(header) - if val: - parts.append(f"{header}: {val}") - parts.append("") # blank line after headers - body = msg.get_body(preferencelist=("plain", "html")) - if body: - content = body.get_content() - if body.get_content_type() == "text/html": - from lxml import etree - tree = etree.HTML(content) - content = "".join(tree.itertext()) if tree is not None else content - parts.append(content) - return "\n".join(parts) - - -# MIME type / extension → extractor mapping. -# Checked in order by read_file; the first match wins. -# Each entry: (mime_type_or_None, file_extension_or_None, extractor) -EXTRACTORS: list[tuple[str | None, str | None, callable]] = [ - ("application/pdf", None, extract_pdf), - ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", None, extract_docx), - ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", None, extract_xlsx), - ("application/vnd.openxmlformats-officedocument.presentationml.presentation", None, extract_pptx), - ("application/rtf", ".rtf", extract_rtf), - ("application/vnd.ms-excel", ".xls", extract_xls), - ("application/vnd.oasis.opendocument.text", ".odt", extract_odt), - ("application/vnd.oasis.opendocument.spreadsheet", ".ods", extract_ods), - ("application/vnd.oasis.opendocument.presentation", ".odp", extract_odp), - ("application/epub+zip", ".epub", extract_epub), - ("message/rfc822", ".eml", extract_eml), -] diff --git a/open_terminal/utils/fs.py b/open_terminal/utils/fs.py deleted file mode 100644 index b2483cb..0000000 --- a/open_terminal/utils/fs.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Filesystem abstraction for multi-user mode. - -Provides :class:`UserFS`, a unified interface for file operations. - -All I/O uses native Python (``aiofiles`` / ``os``). In multi-user mode -the server process is added to each provisioned user's group, and home -directories are ``chmod 2770`` (setgid + group rwx), so standard file -operations work without subprocess. - -After each write operation a ``sudo chown`` call fixes file ownership -so that files belong to the provisioned user, not the server process. -""" - -import asyncio -import os -import shutil -import subprocess - -import aiofiles -import aiofiles.os - - -class UserFS: - """Filesystem operations scoped to an optional OS user. - - *username* is used for ownership fixups after writes (``None`` = stdlib). - *home* is the user's home directory (default working directory). - - When *username* is set, path validation prevents access to other - users' home directories (``/home//…``). - """ - - def __init__(self, username: str | None = None, home: str | None = None): - self.username = username - self.home = home or os.getcwd() - - # ------------------------------------------------------------------ - # Path resolution - # ------------------------------------------------------------------ - - def resolve_path(self, path: str) -> str: - """Resolve *path* to an absolute path relative to the user's home. - - Absolute paths are normalised in place. Relative paths are joined - to ``self.home`` so that they resolve against the user's home - directory rather than the server process's ``os.getcwd()``. - - In multi-user mode, paths under ``/home/user`` (the server process's - default home) are automatically rewritten to the provisioned user's - home directory, since LLMs often hardcode that path. - """ - if os.path.isabs(path): - # Swap /home/user (and /home/usr, a common LLM hallucination) - # → user's actual home when multi-user is active - if self.username and self.home != "/home/user": - for prefix in ("/home/user", "/home/usr"): - if path == prefix: - path = self.home - break - elif path.startswith(prefix + "/"): - path = self.home + path[len(prefix):] - break - return os.path.normpath(path) - return os.path.normpath(os.path.join(self.home, path)) - - # ------------------------------------------------------------------ - # Path validation - # ------------------------------------------------------------------ - - def is_path_allowed(self, path: str) -> bool: - """Return *False* if *path* is inside another user's home directory.""" - if not self.username: - return True - resolved = os.path.abspath(path) - if not resolved.startswith("/home/"): - return True - parts = resolved.split("/") # ['', 'home', '', ...] - if len(parts) >= 3: - target_user_dir = parts[2] - own_home_name = os.path.basename(self.home) - if target_user_dir != own_home_name: - return False - return True - - def _check_path(self, path: str) -> None: - """Reject paths inside another user's home directory.""" - if not self.is_path_allowed(path): - raise PermissionError( - f"Access denied: {os.path.abspath(path)} belongs to another user" - ) - - async def _chown(self, path: str) -> None: - """Fix ownership of *path* to the provisioned user. - - Also sets group-write permission so the server process (which is - in the provisioned user's group) can overwrite the file on - subsequent writes. - """ - if self.username: - await asyncio.to_thread( - subprocess.run, - ["sudo", "chown", f"{self.username}:{self.username}", path], - check=True, capture_output=True, - ) - await asyncio.to_thread( - subprocess.run, - ["sudo", "chmod", "g+w", path], - check=True, capture_output=True, - ) - - async def _ensure_parents(self, path: str) -> None: - """Create parent directories for *path* with correct permissions. - - In multi-user mode, uses ``sudo -u`` to create directories as the - provisioned user (so creation succeeds even inside ``755`` dirs - made by ``run_command``), then sets ``2770`` on each directory in - the chain so the server process has group-write access. - - In single-user mode, falls back to plain ``makedirs``. - """ - if not self.username: - await aiofiles.os.makedirs(path, exist_ok=True) - return - # Create as the provisioned user to bypass 755 restrictions. - await asyncio.to_thread( - subprocess.run, - ["sudo", "-u", self.username, "mkdir", "-p", path], - check=True, capture_output=True, - ) - # Walk upward, setting 2770 so the server process (which is in the - # user's group) can create files inside these directories. - target = os.path.normpath(path) - home = os.path.normpath(self.home) - while target != home and target.startswith(home + "/"): - await asyncio.to_thread( - subprocess.run, - ["sudo", "chmod", "2770", target], - check=True, capture_output=True, - ) - target = os.path.dirname(target) - - # ------------------------------------------------------------------ - # Read operations - # ------------------------------------------------------------------ - - async def read(self, path: str) -> bytes: - """Read raw bytes from *path*.""" - self._check_path(path) - async with aiofiles.open(path, "rb") as f: - return await f.read() - - async def read_text(self, path: str, encoding: str = "utf-8") -> str: - """Read text from *path*.""" - self._check_path(path) - async with aiofiles.open(path, "r", encoding=encoding, errors="strict") as f: - return await f.read() - - async def exists(self, path: str) -> bool: - """Check if *path* exists.""" - self._check_path(path) - return await aiofiles.os.path.exists(path) - - async def isfile(self, path: str) -> bool: - """Check if *path* is a regular file.""" - self._check_path(path) - return await aiofiles.os.path.isfile(path) - - async def isdir(self, path: str) -> bool: - """Check if *path* is a directory.""" - self._check_path(path) - return await aiofiles.os.path.isdir(path) - - async def stat(self, path: str) -> dict: - """Return size, mtime, and type for *path*.""" - self._check_path(path) - s = await aiofiles.os.stat(path) - return { - "size": s.st_size, - "modified": s.st_mtime, - "type": "directory" if os.path.isdir(path) else "file", - } - - async def listdir(self, path: str) -> list[dict]: - """List directory contents with type, size, and mtime.""" - self._check_path(path) - def _list_sync(): - entries = [] - for name in sorted(os.listdir(path)): - full = os.path.join(path, name) - if not self.is_path_allowed(full): - continue - try: - s = os.stat(full) - entries.append({ - "name": name, - "type": "directory" if os.path.isdir(full) else "file", - "size": s.st_size, - "modified": s.st_mtime, - }) - except OSError: - continue - return entries - return await asyncio.to_thread(_list_sync) - - async def walk(self, path: str) -> list[tuple[str, list[str], list[str]]]: - """Walk directory tree. Returns list of (dirpath, dirnames, filenames). - - In multi-user mode, directories belonging to other users are pruned - so their contents are never yielded. - """ - self._check_path(path) - def _walk_filtered(): - result = [] - for dirpath, dirnames, filenames in os.walk(path): - # Prune directories belonging to other users (in-place - # modification prevents os.walk from descending into them). - dirnames[:] = [ - d for d in dirnames - if self.is_path_allowed(os.path.join(dirpath, d)) - ] - filenames = [ - f for f in filenames - if self.is_path_allowed(os.path.join(dirpath, f)) - ] - result.append((dirpath, dirnames, filenames)) - return result - return await asyncio.to_thread(_walk_filtered) - - # ------------------------------------------------------------------ - # Write operations (native Python + chown for correct ownership) - # ------------------------------------------------------------------ - - async def write(self, path: str, content: str, encoding: str = "utf-8") -> None: - """Write text *content* to *path*, creating parent dirs.""" - self._check_path(path) - parent = os.path.dirname(path) - if parent: - await self._ensure_parents(parent) - async with aiofiles.open(path, "w", encoding=encoding) as f: - await f.write(content) - await self._chown(path) - - async def write_bytes(self, path: str, data: bytes) -> None: - """Write raw *data* to *path*, creating parent dirs.""" - self._check_path(path) - parent = os.path.dirname(path) - if parent: - await self._ensure_parents(parent) - async with aiofiles.open(path, "wb") as f: - await f.write(data) - await self._chown(path) - - async def mkdir(self, path: str) -> None: - """Create directory *path* and parents.""" - self._check_path(path) - await self._ensure_parents(path) - - async def remove(self, path: str) -> None: - """Remove *path* (file or directory).""" - self._check_path(path) - if os.path.isdir(path): - await asyncio.to_thread(shutil.rmtree, path) - else: - await aiofiles.os.remove(path) - - async def move(self, source: str, destination: str) -> None: - """Move *source* to *destination*.""" - self._check_path(source) - self._check_path(destination) - await asyncio.to_thread(shutil.move, source, destination) - await self._chown(destination) diff --git a/open_terminal/utils/github_token.py b/open_terminal/utils/github_token.py deleted file mode 100644 index 6b1d9bd..0000000 --- a/open_terminal/utils/github_token.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Keep GH_TOKEN/GITHUB_TOKEN fresh in this process's own environment. - -entrypoint.sh mints a GitHub App installation token at container start and -refreshes it on disk every 50 minutes (tokens live 60 minutes), but that -refresh happens in a *separate* bash process. A long-lived Python process's -``os.environ`` is a one-time snapshot taken at interpreter start — nothing -external can mutate it, so any subprocess built from ``os.environ`` (directly, -or via ``{**os.environ, ...}``) silently inherits whatever token was valid at -boot, forever. Command execution that uses the plain-shell path (``shell=True`` -→ ``/bin/sh``, which is ``dash`` here, not bash) never sources -``/etc/profile.d`` either, so it can't self-correct that way. - -Re-reading the token file into this process's own ``os.environ`` right before -building a subprocess environment closes that gap for every execution path, -regardless of shell or ``run_as_user``. See dvystrcil/homelab#701. -""" - -import os - -TOKEN_FILE_CANDIDATES = ("/run/secrets/github_token", "/tmp/github_token") - - -def refresh_github_token_env() -> bool: - """Re-read the current GitHub App token from disk into os.environ. - - Returns True if a token file was found and applied, False if neither - candidate path exists (e.g. GH_TOKEN not in use in this deployment) — - in that case os.environ is left untouched. - """ - for path in TOKEN_FILE_CANDIDATES: - try: - with open(path, "r") as f: - token = f.read().strip() - except OSError: - continue - if not token: - continue - os.environ["GH_TOKEN"] = token - os.environ["GITHUB_TOKEN"] = token - return True - return False diff --git a/open_terminal/utils/log.py b/open_terminal/utils/log.py deleted file mode 100644 index c7e02aa..0000000 --- a/open_terminal/utils/log.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Process log management utilities. - -Handles writing, reading, and capping JSONL log files for background -processes. Extracted from ``main.py`` to keep the route module focused. -""" - -import json -import os -import time -from typing import Optional - -import aiofiles -import aiofiles.os - -from open_terminal.env import MAX_PROCESS_LOG_SIZE, LOG_FLUSH_INTERVAL, LOG_FLUSH_BUFFER - - -class BoundedLogWriter: - """Async wrapper that rotates the log file when it exceeds a size limit. - - When the total bytes written surpass *MAX_PROCESS_LOG_SIZE*, the file - is truncated to its newest half and a ``log_rotated`` marker is inserted. - Writing then continues, so the most recent output is always available. - - Flushing behaviour is controlled by *flush_interval* and *flush_buffer*: - - * ``flush_interval=0`` (default) — flush after every write (original - behaviour, safest for low-throughput commands). - * ``flush_interval>0`` — flush at most once per *flush_interval* seconds, - **or** when the unflushed buffer exceeds *flush_buffer* bytes (if set). - This dramatically reduces I/O pressure for high-output commands. - """ - - __slots__ = ( - "_file", "_log_path", "_bytes_written", "rotated", - "_flush_interval", "_flush_buffer", "_unflushed", "_last_flush", - ) - - def __init__(self, file, log_path: str, *, flush_interval: float = 0, flush_buffer: int = 0): - self._file = file - self._log_path = log_path - self._bytes_written = 0 - self.rotated = False - self._flush_interval = flush_interval - self._flush_buffer = flush_buffer - self._unflushed = 0 - self._last_flush = time.monotonic() - - async def write(self, data: str) -> None: - encoded_len = len(data.encode("utf-8", errors="replace")) - if self._bytes_written + encoded_len > MAX_PROCESS_LOG_SIZE: - await self._rotate() - await self._file.write(data) - self._bytes_written += encoded_len - self._unflushed += encoded_len - - if self._flush_interval <= 0: - # Legacy behaviour: flush on every write. - await self._file.flush() - self._unflushed = 0 - return - - now = time.monotonic() - should_flush = (now - self._last_flush) >= self._flush_interval - if not should_flush and self._flush_buffer > 0: - should_flush = self._unflushed >= self._flush_buffer - if should_flush: - await self._file.flush() - self._unflushed = 0 - self._last_flush = now - - async def flush(self) -> None: - await self._file.flush() - self._unflushed = 0 - self._last_flush = time.monotonic() - - async def _rotate(self) -> None: - """Keep the newest half of the log file and continue writing.""" - self.rotated = True - await self._file.flush() - # Close, rewrite, and reopen. - await self._file.close() - - async with aiofiles.open(self._log_path, "r", encoding="utf-8") as f: - lines = await f.readlines() - - # Keep the newest half of output lines. - keep = lines[len(lines) // 2 :] - - async with aiofiles.open(self._log_path, "w", encoding="utf-8") as f: - await f.write( - json.dumps({"type": "log_rotated", "ts": time.time()}) + "\n" - ) - for line in keep: - await f.write(line) - - # Reopen in append mode and reset byte counter. - self._file = await aiofiles.open(self._log_path, "a", encoding="utf-8") - self._bytes_written = sum(len(l.encode("utf-8", errors="replace")) for l in keep) - - -async def tail_log(log_path: str, n: int) -> list[dict]: - """Read the last *n* output entries from a JSONL log without loading the whole file. - - Uses a reverse-read strategy: read chunks from the end of the file - until enough newline-delimited records have been collected. - """ - CHUNK = 8192 - entries: list[dict] = [] - - async with aiofiles.open(log_path, "rb") as f: - await f.seek(0, 2) # seek to end - remaining = await f.tell() - buffer = b"" - - while remaining > 0 and len(entries) < n: - read_size = min(CHUNK, remaining) - remaining -= read_size - await f.seek(remaining) - chunk = await f.read(read_size) - buffer = chunk + buffer - lines = buffer.split(b"\n") - # The first element may be a partial line — keep it for next iteration. - buffer = lines[0] - for raw_line in reversed(lines[1:]): - raw_line = raw_line.strip() - if not raw_line: - continue - try: - record = json.loads(raw_line) - except (json.JSONDecodeError, UnicodeDecodeError): - continue - if record.get("type") in ("stdout", "stderr", "output"): - entries.append({"type": record["type"], "data": record["data"]}) - if len(entries) >= n: - break - - # Process any remaining buffer content. - if buffer.strip() and len(entries) < n: - try: - record = json.loads(buffer) - if record.get("type") in ("stdout", "stderr", "output"): - entries.append({"type": record["type"], "data": record["data"]}) - except (json.JSONDecodeError, UnicodeDecodeError): - pass - - entries.reverse() # restore chronological order - return entries[-n:] - - -async def log_process(background_process) -> None: - """Read process output and persist to a log file. - - When the file exceeds *MAX_PROCESS_LOG_SIZE*, the oldest half is - discarded so the most recent output is always available. - """ - log_file = None - log_rotated = False - try: - if background_process.log_path: - await aiofiles.os.makedirs( - os.path.dirname(background_process.log_path), exist_ok=True - ) - log_file = await aiofiles.open(background_process.log_path, "a", encoding="utf-8") - await log_file.write( - json.dumps( - { - "type": "start", - "command": background_process.command, - "pid": background_process.runner.pid, - "ts": time.time(), - } - ) - + "\n" - ) - await log_file.flush() - except OSError: - log_file = None - - # Wrap the log file so it rotates when the size limit is reached. - writer = ( - BoundedLogWriter( - log_file, - background_process.log_path, - flush_interval=LOG_FLUSH_INTERVAL, - flush_buffer=LOG_FLUSH_BUFFER, - ) - if log_file - else None - ) - - try: - await background_process.runner.read_output(writer) - finally: - log_rotated = writer.rotated if writer else False - exit_code = await background_process.runner.wait() - background_process.exit_code = exit_code - background_process.status = "done" - background_process.finished_at = time.time() - background_process.runner.close() - if writer: - # Flush any buffered output before writing the end marker. - await writer.flush() - if log_file: - await log_file.write( - json.dumps( - { - "type": "end", - "exit_code": background_process.exit_code, - "log_rotated": log_rotated, - "ts": time.time(), - } - ) - + "\n" - ) - await log_file.close() - - -async def read_log( - log_path: Optional[str], - offset: int = 0, - tail: Optional[int] = None, -) -> tuple[list[dict], int, bool]: - """Read output entries from a JSONL log file. - - Returns ``(entries, next_offset, truncated)``. - - When *tail* is specified and *offset* is 0, the file is read from the - end to avoid loading the entire file into memory — preventing the - memory spike that caused the OOM issue. - """ - entries: list[dict] = [] - if not log_path or not await aiofiles.os.path.isfile(log_path): - return entries, 0, False - - # --- Optimised tail-from-end path --- - if tail is not None and offset == 0: - tail_entries = await tail_log(log_path, tail) - truncated = len(tail_entries) == tail # may have been more - return tail_entries, len(tail_entries), truncated - - # --- Full scan path (bounded by offset) --- - async with aiofiles.open(log_path, encoding="utf-8") as f: - lines = await f.readlines() - - for line in lines: - line = line.strip() - if not line: - continue - try: - record = json.loads(line) - except json.JSONDecodeError: - continue - if record.get("type") in ("stdout", "stderr", "output"): - entries.append({"type": record["type"], "data": record["data"]}) - - total = len(entries) - entries = entries[offset:] - - truncated = False - if tail is not None and len(entries) > tail: - entries = entries[-tail:] - truncated = True - - return entries, total, truncated diff --git a/open_terminal/utils/notebooks.py b/open_terminal/utils/notebooks.py deleted file mode 100644 index c079e36..0000000 --- a/open_terminal/utils/notebooks.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Jupyter notebook execution endpoints. - -Provides per-cell execution with multi-session support. Each session gets its -own Jupyter kernel via nbclient. Requires the ``notebooks`` optional extra:: - - pip install open-terminal[notebooks] -""" - -import asyncio -import json -import os -import time -import uuid -from typing import Optional - -import aiofiles -import aiofiles.os -from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel, Field - -import nbformat -from nbclient import NotebookClient - - -# --------------------------------------------------------------------------- -# Session manager -# --------------------------------------------------------------------------- - -_IDLE_TIMEOUT = 30 * 60 # 30 minutes - - -class _Session: - """Wraps a NotebookClient for a specific notebook.""" - - __slots__ = ("id", "path", "nb", "client", "busy", "created_at", "last_used") - - def __init__(self, session_id: str, path: str, nb, client): - self.id = session_id - self.path = path - self.nb = nb - self.client = client - self.busy = False - self.created_at = time.time() - self.last_used = time.time() - - -_sessions: dict[str, _Session] = {} -_cleanup_task: Optional[asyncio.Task] = None - - -async def _idle_cleanup_loop(): - """Periodically remove sessions idle for more than _IDLE_TIMEOUT.""" - while True: - await asyncio.sleep(60) - now = time.time() - stale = [ - sid - for sid, s in _sessions.items() - if now - s.last_used > _IDLE_TIMEOUT and not s.busy - ] - for sid in stale: - await _destroy_session(sid) - - -async def _destroy_session(session_id: str): - session = _sessions.pop(session_id, None) - if session and session.client: - try: - await session.client._async_cleanup_kernel() - except Exception: - pass - - -def _ensure_cleanup_task(): - global _cleanup_task - if _cleanup_task is None or _cleanup_task.done(): - _cleanup_task = asyncio.create_task(_idle_cleanup_loop()) - - -# --------------------------------------------------------------------------- -# Request / response models -# --------------------------------------------------------------------------- - - -class CreateSessionRequest(BaseModel): - path: str = Field(..., description="Absolute path to the .ipynb file.") - - -class CreateSessionResponse(BaseModel): - id: str - kernel: str - status: str - - -class ExecuteCellRequest(BaseModel): - cell_index: int = Field(..., description="Zero-based cell index to execute.") - source: Optional[str] = Field( - None, description="Override cell source. If omitted, uses the source already in the notebook." - ) - - -class ExecuteCellResponse(BaseModel): - status: str - execution_count: Optional[int] = None - outputs: list = Field(default_factory=list) - - -class SessionStatusResponse(BaseModel): - id: str - path: str - kernel: str - status: str - - -# --------------------------------------------------------------------------- -# Router -# --------------------------------------------------------------------------- - - -def create_notebooks_router(verify_api_key) -> APIRouter: - """Create the notebooks router with the given auth dependency.""" - - router = APIRouter( - prefix="/notebooks", - tags=["notebooks"], - dependencies=[Depends(verify_api_key)], - ) - - - - @router.post( - "", - response_model=CreateSessionResponse, - operation_id="create_notebook_session", - summary="Create a notebook session", - description="Start a Jupyter kernel for the given notebook. Returns a session ID for subsequent execute calls.", - include_in_schema=False, - ) - async def create_session(req: CreateSessionRequest): - - _ensure_cleanup_task() - - path = os.path.abspath(req.path) - if not await aiofiles.os.path.isfile(path): - raise HTTPException(status_code=404, detail=f"Notebook not found: {path}") - - # Read and parse the notebook - async with aiofiles.open(path, encoding="utf-8") as f: - content = await f.read() - - try: - nb = nbformat.reads(content, as_version=4) - except Exception as e: - raise HTTPException(status_code=400, detail=f"Invalid notebook: {e}") - - kernel_name = nb.metadata.get("kernelspec", {}).get("name", "python3") - - # Start kernel — follow nbclient's setup_kernel pattern - client = NotebookClient(nb, kernel_name=kernel_name, timeout=120) - try: - client.create_kernel_manager() - await client.async_start_new_kernel() - await client.async_start_new_kernel_client() - except Exception as e: - raise HTTPException( - status_code=500, - detail=f"Failed to start kernel '{kernel_name}': {e}", - ) - - session_id = uuid.uuid4().hex[:12] - _sessions[session_id] = _Session(session_id, path, nb, client) - - return CreateSessionResponse( - id=session_id, kernel=kernel_name, status="ready" - ) - - @router.post( - "/{session_id}/execute", - response_model=ExecuteCellResponse, - operation_id="execute_notebook_cell", - summary="Execute a notebook cell", - description="Execute a single cell in the given session. Optionally override the cell source. " - "Updates the .ipynb file in place after execution.", - include_in_schema=False, - ) - async def execute_cell(session_id: str, req: ExecuteCellRequest): - - - session = _sessions.get(session_id) - if not session: - raise HTTPException(status_code=404, detail="Session not found") - if session.busy: - raise HTTPException(status_code=409, detail="Cell already executing") - - nb = session.nb - if req.cell_index < 0 or req.cell_index >= len(nb.cells): - raise HTTPException( - status_code=400, - detail=f"cell_index {req.cell_index} out of range (0..{len(nb.cells) - 1})", - ) - - cell = nb.cells[req.cell_index] - if req.source is not None: - cell.source = req.source - - session.busy = True - session.last_used = time.time() - - try: - await session.client.async_execute_cell(cell, req.cell_index) - except Exception as e: - session.busy = False - # Return the error as a cell output rather than HTTP error - return ExecuteCellResponse( - status="error", - outputs=[ - { - "output_type": "error", - "ename": type(e).__name__, - "evalue": str(e), - "traceback": [str(e)], - } - ], - ) - - session.busy = False - session.last_used = time.time() - - # Serialize outputs - outputs = [] - for o in cell.outputs: - od = dict(o) - if "data" in od: - od["data"] = dict(od["data"]) - outputs.append(od) - - ec = cell.get("execution_count") - - # Save notebook to disk - try: - nb_json = nbformat.writes(nb) - async with aiofiles.open(session.path, "w", encoding="utf-8") as f: - await f.write(nb_json) - except Exception: - pass # non-fatal - - return ExecuteCellResponse( - status="ok", execution_count=ec, outputs=outputs - ) - - @router.get( - "/{session_id}", - response_model=SessionStatusResponse, - operation_id="get_notebook_session", - summary="Get notebook session status", - include_in_schema=False, - ) - async def get_session(session_id: str): - session = _sessions.get(session_id) - if not session: - raise HTTPException(status_code=404, detail="Session not found") - - kernel_name = session.nb.metadata.get("kernelspec", {}).get( - "name", "python3" - ) - status = "busy" if session.busy else "ready" - return SessionStatusResponse( - id=session.id, path=session.path, kernel=kernel_name, status=status - ) - - @router.delete( - "/{session_id}", - operation_id="delete_notebook_session", - summary="Stop a notebook session", - include_in_schema=False, - ) - async def delete_session(session_id: str): - if session_id not in _sessions: - raise HTTPException(status_code=404, detail="Session not found") - await _destroy_session(session_id) - return {"status": "stopped"} - - return router diff --git a/open_terminal/utils/port.py b/open_terminal/utils/port.py deleted file mode 100644 index e6da19b..0000000 --- a/open_terminal/utils/port.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Port detection and reverse-proxy utilities.""" - -import os -import platform - - -def detect_listening_ports() -> list[dict]: - """Detect TCP ports listening on localhost. - - Strategy: - - Linux: parse /proc/net/tcp and /proc/net/tcp6 (fast, no subprocess) - - macOS / fallback: run ``lsof -iTCP -sTCP:LISTEN -nP`` - - Windows: run ``netstat -ano`` - - Returns a sorted list of dicts with keys: port, pid, process. - """ - ports: dict[int, dict] = {} # port -> {port, pid, process} - - # --- Linux: /proc/net/tcp --- - def _parse_proc_net_tcp(): - for path in ("/proc/net/tcp", "/proc/net/tcp6"): - try: - with open(path) as f: - for line in f: - parts = line.strip().split() - if len(parts) < 8 or parts[3] != "0A": # 0A = LISTEN - continue - local_addr = parts[1] - port = int(local_addr.split(":")[1], 16) - if port == 0: - continue - uid = int(parts[7]) if len(parts) > 7 else None - inode = parts[9] if len(parts) > 9 else "" - pid = _pid_from_inode(inode) if inode else None - pname = _process_name(pid) if pid else None - if port not in ports: - ports[port] = { - "port": port, - "pid": pid, - "process": pname, - "uid": uid, - } - except FileNotFoundError: - continue - - def _pid_from_inode(inode: str) -> int | None: - """Resolve a socket inode to a PID by scanning /proc/*/fd/.""" - try: - target = f"socket:[{inode}]" - for pid_dir in os.listdir("/proc"): - if not pid_dir.isdigit(): - continue - fd_dir = f"/proc/{pid_dir}/fd" - try: - for fd in os.listdir(fd_dir): - try: - link = os.readlink(f"{fd_dir}/{fd}") - if link == target: - return int(pid_dir) - except (OSError, ValueError): - continue - except PermissionError: - continue - except OSError: - pass - return None - - def _process_name(pid: int) -> str | None: - try: - with open(f"/proc/{pid}/comm") as f: - return f.read().strip() - except (FileNotFoundError, PermissionError): - return None - - # --- macOS / fallback: lsof --- - def _parse_lsof(): - import subprocess as _sp - - try: - result = _sp.run( - ["lsof", "-iTCP", "-sTCP:LISTEN", "-nP", "-F", "pcn"], - capture_output=True, - text=True, - timeout=5, - ) - except (FileNotFoundError, _sp.TimeoutExpired): - return - - current_pid = None - current_name = None - for line in result.stdout.splitlines(): - if line.startswith("p"): - current_pid = int(line[1:]) if line[1:].isdigit() else None - elif line.startswith("c"): - current_name = line[1:] - elif line.startswith("n"): - # e.g. "n*:8080" or "n127.0.0.1:3000" or "n[::1]:3000" - addr = line[1:] - colon_idx = addr.rfind(":") - if colon_idx >= 0: - port_str = addr[colon_idx + 1:] - if port_str.isdigit(): - port = int(port_str) - if port not in ports: - ports[port] = { - "port": port, - "pid": current_pid, - "process": current_name, - } - - # --- Windows: netstat --- - def _parse_netstat(): - import subprocess as _sp - - try: - result = _sp.run( - ["netstat", "-ano", "-p", "tcp"], - capture_output=True, - text=True, - timeout=5, - ) - except (FileNotFoundError, _sp.TimeoutExpired): - return - - for line in result.stdout.splitlines(): - parts = line.split() - if len(parts) >= 5 and parts[3] == "LISTENING": - local_addr = parts[1] - colon_idx = local_addr.rfind(":") - if colon_idx >= 0: - port_str = local_addr[colon_idx + 1:] - if port_str.isdigit(): - port = int(port_str) - pid = int(parts[4]) if parts[4].isdigit() else None - if port not in ports: - ports[port] = { - "port": port, - "pid": pid, - "process": None, - } - - # Choose strategy - if os.path.exists("/proc/net/tcp"): - _parse_proc_net_tcp() - elif platform.system() == "Windows": - _parse_netstat() - else: - _parse_lsof() - - return sorted(ports.values(), key=lambda p: p["port"]) - - -def get_descendant_pids(root_pid: int) -> set[int]: - """Return all PIDs that are descendants of *root_pid* (exclusive). - - Strategy: - - Linux: parse ``/proc/*/stat`` for parent PID - - macOS / fallback: run ``ps -eo pid,ppid`` - """ - children: dict[int, list[int]] = {} - - if os.path.exists("/proc"): - for entry in os.listdir("/proc"): - if not entry.isdigit(): - continue - try: - with open(f"/proc/{entry}/stat") as f: - stat = f.read().split() - ppid = int(stat[3]) - children.setdefault(ppid, []).append(int(entry)) - except (FileNotFoundError, PermissionError, IndexError, ValueError): - continue - else: - import subprocess as _sp - - try: - result = _sp.run( - ["ps", "-eo", "pid,ppid"], - capture_output=True, - text=True, - timeout=5, - ) - for line in result.stdout.strip().splitlines()[1:]: - parts = line.split() - if len(parts) >= 2: - try: - pid, ppid = int(parts[0]), int(parts[1]) - children.setdefault(ppid, []).append(pid) - except ValueError: - continue - except (FileNotFoundError, _sp.TimeoutExpired): - return set() - - descendants: set[int] = set() - queue = list(children.get(root_pid, [])) - while queue: - pid = queue.pop() - if pid not in descendants: - descendants.add(pid) - queue.extend(children.get(pid, [])) - return descendants diff --git a/open_terminal/utils/runner.py b/open_terminal/utils/runner.py deleted file mode 100644 index 2e63868..0000000 --- a/open_terminal/utils/runner.py +++ /dev/null @@ -1,296 +0,0 @@ -import asyncio -import json -import os -import shlex -import signal -import subprocess -import time -from abc import ABC, abstractmethod - -try: - import fcntl - import pty - import struct - import termios - - _PTY_AVAILABLE = True -except ImportError: - _PTY_AVAILABLE = False # Windows - -try: - from winpty import PtyProcess as WinPtyProcess - - _WINPTY_AVAILABLE = True -except ImportError: - _WINPTY_AVAILABLE = False - - -class ProcessRunner(ABC): - """Unified interface for running a subprocess via PTY or pipes.""" - - @abstractmethod - async def read_output(self, log_file) -> None: - """Read output from the process and write entries to *log_file*.""" - - @abstractmethod - def write_input(self, data: bytes) -> None: - """Send *data* to the process's stdin / PTY.""" - - @abstractmethod - def kill(self, force: bool = False) -> None: - """Terminate (SIGTERM) or kill (SIGKILL) the process.""" - - @abstractmethod - async def wait(self) -> int: - """Wait for the process to exit and return the exit code.""" - - @abstractmethod - def close(self) -> None: - """Release file descriptors and other resources.""" - - @property - @abstractmethod - def pid(self) -> int: - """PID of the child process.""" - - -class PtyRunner(ProcessRunner): - """Spawn a command under a pseudo-terminal (Unix).""" - - def __init__(self, command: str, cwd: str | None, env: dict | None, run_as_user: str | None = None): - if run_as_user: - # Build the inner command: optionally cd first, then run the command. - inner = f"cd {shlex.quote(cwd)} && {command}" if cwd else command - command = f"sudo -u {shlex.quote(run_as_user)} -- bash -c {shlex.quote(inner)}" - cwd = None # Popen runs as parent user — can't chdir into chmod 700 dirs - master_fd, slave_fd = pty.openpty() - try: - # Set a reasonable default window size (80x24). - fcntl.ioctl(slave_fd, termios.TIOCSWINSZ, struct.pack("HHHH", 24, 80, 0, 0)) - self._process = subprocess.Popen( - command, - shell=True, - stdin=slave_fd, - stdout=slave_fd, - stderr=slave_fd, - cwd=cwd, - env=env, - start_new_session=True, - ) - except Exception: - os.close(slave_fd) - os.close(master_fd) - raise - os.close(slave_fd) - self._master_fd = master_fd - - async def read_output(self, log_file) -> None: - loop = asyncio.get_event_loop() - while True: - try: - data = await loop.run_in_executor(None, os.read, self._master_fd, 4096) - if not data: - break - except OSError: - break # EIO when child exits - if log_file: - await log_file.write( - json.dumps( - { - "type": "output", - "data": data.decode(errors="replace"), - "ts": time.time(), - } - ) - + "\n" - ) - - def write_input(self, data: bytes) -> None: - os.write(self._master_fd, data) - - def _signal_group(self, sig: int) -> None: - """Send *sig* to the child's entire process group. - - Falls back to signalling just the leader if the group is already gone. - """ - try: - os.killpg(self._process.pid, sig) - except (ProcessLookupError, PermissionError): - try: - self._process.send_signal(sig) - except ProcessLookupError: - pass - - def kill(self, force: bool = False) -> None: - self._signal_group(signal.SIGKILL if force else signal.SIGTERM) - - async def wait(self) -> int: - return await asyncio.to_thread(self._process.wait) - - def close(self) -> None: - try: - os.close(self._master_fd) - except OSError: - pass - - @property - def pid(self) -> int: - return self._process.pid - - -class PipeRunner(ProcessRunner): - """Spawn a command with stdin/stdout/stderr pipes (cross-platform fallback).""" - - def __init__(self, command: str, cwd: str | None, env: dict | None): - self._process: asyncio.subprocess.Process = None # type: ignore[assignment] - self._command = command - self._cwd = cwd - self._env = env - - async def start(self) -> None: - self._process = await asyncio.create_subprocess_shell( - self._command, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - stdin=asyncio.subprocess.PIPE, - cwd=self._cwd, - env=self._env, - ) - - async def read_output(self, log_file) -> None: - async def read_stream(stream, label): - async for line in stream: - if log_file: - await log_file.write( - json.dumps( - { - "type": label, - "data": line.decode(errors="replace"), - "ts": time.time(), - } - ) - + "\n" - ) - - await asyncio.gather( - read_stream(self._process.stdout, "stdout"), - read_stream(self._process.stderr, "stderr"), - ) - - def write_input(self, data: bytes) -> None: - self._process.stdin.write(data) - - async def drain_input(self) -> None: - await self._process.stdin.drain() - - def kill(self, force: bool = False) -> None: - sig = signal.SIGKILL if force else signal.SIGTERM - try: - os.killpg(self._process.pid, sig) - except (ProcessLookupError, PermissionError, OSError): - # No dedicated process group — fall back to the child directly. - self._process.send_signal(sig) - - async def wait(self) -> int: - await self._process.wait() - return self._process.returncode - - def close(self) -> None: - pass # pipes are cleaned up automatically - - @property - def pid(self) -> int: - return self._process.pid - - -class WinPtyRunner(ProcessRunner): - """Spawn a command under a Windows pseudo-terminal (ConPTY via pywinpty).""" - - def __init__(self, command: str, cwd: str | None, env: dict | None): - spawn_env = os.environ.copy() - if env: - spawn_env.update(env) - - # Determine the executable and arguments. - # PtyProcess.spawn expects a list: [executable, *args] - shell = spawn_env.get("COMSPEC", "cmd.exe") - cmd_args = [shell, "/c", command] if command else [shell] - - self._pty = WinPtyProcess.spawn( - cmd_args, - cwd=cwd, - env=spawn_env, - dimensions=(24, 80), - ) - - async def read_output(self, log_file) -> None: - loop = asyncio.get_event_loop() - - def _read_blocking(): - try: - return self._pty.read(4096) - except EOFError: - return "" - except Exception: - return "" - - while True: - data = await loop.run_in_executor(None, _read_blocking) - if not data: - if not self._pty.isalive(): - break - await asyncio.sleep(0.05) - continue - if log_file: - await log_file.write( - json.dumps( - { - "type": "output", - "data": data, - "ts": time.time(), - } - ) - + "\n" - ) - - def write_input(self, data: bytes) -> None: - self._pty.write(data.decode(errors="replace")) - - def kill(self, force: bool = False) -> None: - if force: - self._pty.kill(signal.SIGKILL) - else: - self._pty.terminate() - - async def wait(self) -> int: - while self._pty.isalive(): - await asyncio.sleep(0.1) - return self._pty.exitstatus or 0 - - def close(self) -> None: - if self._pty.isalive(): - self._pty.terminate() - - @property - def pid(self) -> int: - return self._pty.pid - - def set_size(self, rows: int, cols: int) -> None: - """Resize the pseudo-terminal window.""" - self._pty.setwinsize(rows, cols) - - -async def create_runner( - command: str, - cwd: str | None, - env: dict | None, - run_as_user: str | None = None, -) -> ProcessRunner: - """Factory: create a PTY runner on Unix, WinPTY runner on Windows, or pipe fallback.""" - if _PTY_AVAILABLE: - return PtyRunner(command, cwd, env, run_as_user=run_as_user) - if _WINPTY_AVAILABLE: - return WinPtyRunner(command, cwd, env) - runner = PipeRunner(command, cwd, env) - await runner.start() - return runner diff --git a/open_terminal/utils/user_isolation.py b/open_terminal/utils/user_isolation.py deleted file mode 100644 index 34576a4..0000000 --- a/open_terminal/utils/user_isolation.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Per-user OS account provisioning for multi-user mode. - -When ``OPEN_TERMINAL_MULTI_USER=true``, each distinct ``X-User-Id`` is mapped -to a dedicated Linux user account. Commands and file operations then run as -that OS user via ``sudo -u``, and ``chmod 700`` on the home directory provides -kernel-enforced isolation between users. -""" - -import hashlib -import logging -import os -import platform -import pwd -import re -import shutil -import subprocess - -log = logging.getLogger(__name__) - -# In-memory cache: upstream user-id → (os_username, home_dir) -_user_cache: dict[str, tuple[str, str]] = {} - - -def _run_privileged(cmd: list[str]) -> subprocess.CompletedProcess: - """Run a command with appropriate privilege escalation. - - When the process is already running as root (UID 0), the command is - executed directly. Otherwise ``sudo`` is prepended. - """ - if os.getuid() == 0: - return subprocess.run(cmd, check=True, capture_output=True) - return subprocess.run(["sudo", *cmd], check=True, capture_output=True) - - -def check_environment() -> None: - """Validate that the host supports multi-user mode. - - Raises ``RuntimeError`` at startup when the platform is not Linux or - the required privilege escalation tools are not available. - """ - if platform.system() != "Linux": - raise RuntimeError( - "OPEN_TERMINAL_MULTI_USER requires Linux " - f"(current platform: {platform.system()})" - ) - if shutil.which("useradd") is None: - raise RuntimeError( - "OPEN_TERMINAL_MULTI_USER requires useradd to be installed" - ) - if os.getuid() != 0 and shutil.which("sudo") is None: - raise RuntimeError( - "OPEN_TERMINAL_MULTI_USER requires either running as root " - "or sudo to be installed. Use the standard image, run with " - "user: '0:0', or use Terminals for container-per-user isolation." - ) - - -def sanitize_username(user_id: str) -> str: - """Convert an arbitrary user ID into a valid Linux username. - - Uses the first 8 lowercase alphanumeric characters of the user ID, - optionally prefixed by ``OPEN_TERMINAL_USER_PREFIX``. Prepends ``u`` - only when the result starts with a digit (Linux usernames must begin - with a letter or underscore). Falls back to a short hash when the ID - contains fewer than 4 usable characters. - """ - from open_terminal.env import USER_PREFIX - - cleaned = re.sub(r"[^a-z0-9]", "", user_id.lower()) - if len(cleaned) >= 4: - name = cleaned[:8] - else: - # Fallback: hash-based name for very short / non-alphanumeric IDs - name = hashlib.sha256(user_id.encode()).hexdigest()[:8] - name = f"{USER_PREFIX}{name}" - # Linux usernames must start with a letter or underscore - if name[0].isdigit(): - name = f"u{name}" - return name - - -def ensure_os_user(username: str) -> str: - """Create the OS user if it doesn't exist (idempotent). - - Sets ``chmod 2770`` on the home directory and adds the server process - user to the new user's primary group. This allows native Python I/O - for reads while other provisioned users still get ``Permission denied``. - Returns the home directory path. - """ - try: - pw = pwd.getpwnam(username) - return pw.pw_dir - except KeyError: - pass # User doesn't exist yet — create below - - log.info("Provisioning OS user: %s", username) - _run_privileged(["useradd", "-m", "-s", "/bin/bash", username]) - home_dir = f"/home/{username}" - # Fix ownership (home dir may pre-exist from a previous run with a - # different UID assignment) and set permissions. - _run_privileged(["chown", "-R", f"{username}:{username}", home_dir]) - _run_privileged(["chmod", "2770", home_dir]) - # Add the server process user to the new user's group so Python can - # read files natively without subprocess. - server_user = os.getenv("USER", "user") - _run_privileged(["usermod", "-aG", username, server_user]) - # If the Docker socket is mounted, add the new user to its group - # so docker commands work without sudo (mirrors entrypoint.sh). - _DOCKER_SOCK = "/var/run/docker.sock" - if os.path.exists(_DOCKER_SOCK): - import grp as _grp - sock_gid = os.stat(_DOCKER_SOCK).st_gid - try: - sock_group = _grp.getgrgid(sock_gid).gr_name - _run_privileged(["usermod", "-aG", sock_group, username]) - log.info("Added %s to Docker socket group '%s'", username, sock_group) - except (KeyError, subprocess.CalledProcessError) as exc: - log.warning("Could not add %s to Docker socket group: %s", username, exc) - # Refresh the running process's supplementary group list so the new - # group takes effect immediately (normally requires re-login). - import ctypes - import ctypes.util - import grp - - pw = pwd.getpwnam(server_user) - group_ids = sorted({ - g.gr_gid for g in grp.getgrall() if server_user in g.gr_mem - } | {pw.pw_gid}) - try: - os.setgroups(group_ids) - except PermissionError: - # Container user may lack CAP_SETGID. The group will take effect - # after the next process restart; log a warning and continue. - log.warning( - "Could not refresh supplementary groups (missing CAP_SETGID). " - "Restart the server for group changes to take effect." - ) - return home_dir - - -def resolve_user(user_id: str) -> tuple[str, str]: - """Map an upstream user ID to an OS user, provisioning if needed. - - Returns ``(username, home_dir)``. Results are cached in-memory so - repeated requests for the same user skip the syscall / subprocess. - """ - cached = _user_cache.get(user_id) - if cached is not None: - return cached - - username = sanitize_username(user_id) - home_dir = ensure_os_user(username) - _user_cache[user_id] = (username, home_dir) - return username, home_dir diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 87fd475..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[project] -name = "open-terminal" -version = "0.11.38" -description = "A remote terminal API." -readme = "README.md" -authors = [ - { name = "Daniel Vystrcil", email = "dvystrcil@gmail.com" } -] -requires-python = ">=3.11" -dependencies = [ - "fastapi>=0.115.0", - "uvicorn[standard]>=0.34.0", - "click>=8.1.0", - "httpx>=0.27.0", - "python-multipart>=0.0.22", - "aiofiles>=25.1.0", - "pypdf>=5.0.0", - "python-docx>=1.0.0", - "openpyxl>=3.1.0", - "python-pptx>=1.0.0", - "striprtf>=0.0.26", - "xlrd>=2.0.0", - "nbclient>=0.10.0", - "ipykernel>=6.0.0", - "pywinpty>=2.0.0; sys_platform == 'win32'", -] - -[project.optional-dependencies] -mcp = ["fastmcp>=2.0.0"] - -[project.scripts] -open-terminal = "open_terminal.cli:main" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["open_terminal"] - -[dependency-groups] -dev = [ - "pytest>=9.0.2", -] diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 3532087..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Pytest config. Sets the env vars `open_terminal.main` requires at import time -so that test modules can `from open_terminal.main import app, ...` without the -module's startup SystemExit firing. -""" - -import os - -os.environ.setdefault("OPEN_TERMINAL_API_KEY", "test-key-not-used-by-handler") diff --git a/tests/test_github_token_refresh.py b/tests/test_github_token_refresh.py deleted file mode 100644 index 60127bc..0000000 --- a/tests/test_github_token_refresh.py +++ /dev/null @@ -1,73 +0,0 @@ -import os - -import pytest - -from open_terminal.utils.github_token import ( - TOKEN_FILE_CANDIDATES, - refresh_github_token_env, -) - - -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch): - monkeypatch.delenv("GH_TOKEN", raising=False) - monkeypatch.delenv("GITHUB_TOKEN", raising=False) - - -def test_refresh_applies_token_from_first_candidate(monkeypatch, tmp_path): - token_file = tmp_path / "github_token" - token_file.write_text("ghs_freshtoken123\n") - monkeypatch.setattr( - "open_terminal.utils.github_token.TOKEN_FILE_CANDIDATES", - (str(token_file), "/tmp/does-not-exist-github-token"), - ) - - result = refresh_github_token_env() - - assert result is True - assert os.environ["GH_TOKEN"] == "ghs_freshtoken123" - assert os.environ["GITHUB_TOKEN"] == "ghs_freshtoken123" - - -def test_refresh_falls_back_to_second_candidate(monkeypatch, tmp_path): - token_file = tmp_path / "github_token" - token_file.write_text("ghs_secondcandidate\n") - monkeypatch.setattr( - "open_terminal.utils.github_token.TOKEN_FILE_CANDIDATES", - ("/tmp/does-not-exist-github-token", str(token_file)), - ) - - result = refresh_github_token_env() - - assert result is True - assert os.environ["GH_TOKEN"] == "ghs_secondcandidate" - - -def test_refresh_overwrites_stale_env_value(monkeypatch, tmp_path): - os.environ["GH_TOKEN"] = "ghs_staletoken_from_boot" - token_file = tmp_path / "github_token" - token_file.write_text("ghs_brandnew\n") - monkeypatch.setattr( - "open_terminal.utils.github_token.TOKEN_FILE_CANDIDATES", - (str(token_file),), - ) - - refresh_github_token_env() - - assert os.environ["GH_TOKEN"] == "ghs_brandnew" - - -def test_refresh_noop_when_no_candidate_exists(monkeypatch): - monkeypatch.setattr( - "open_terminal.utils.github_token.TOKEN_FILE_CANDIDATES", - ("/tmp/does-not-exist-a", "/tmp/does-not-exist-b"), - ) - - result = refresh_github_token_env() - - assert result is False - assert "GH_TOKEN" not in os.environ - - -def test_default_candidates_are_secrets_then_tmp(): - assert TOKEN_FILE_CANDIDATES == ("/run/secrets/github_token", "/tmp/github_token") diff --git a/tests/test_insert_append.py b/tests/test_insert_append.py deleted file mode 100644 index 651aca4..0000000 --- a/tests/test_insert_append.py +++ /dev/null @@ -1,367 +0,0 @@ -""" -Tests for the new insert/append operations (homelab#108): - - POST /files/insert_after - - POST /files/append_to_section - - POST /files/append - -These are companion operations to /files/replace, designed for ADDING content -rather than replacing existing content. The model picks the verb that matches -its intent: edit-in-place → replace, add-new → insert_after/append_*. -""" - -from fastapi.testclient import TestClient - -from open_terminal.main import app, get_filesystem, verify_api_key - - -async def _noop_auth(): - return None - - -app.dependency_overrides[verify_api_key] = _noop_auth - - -class StubFS: - def __init__(self, files: dict[str, str]): - self._files = dict(files) - - def resolve_path(self, path: str) -> str: - return path - - async def isfile(self, path: str) -> bool: - return path in self._files - - async def read_text(self, path: str) -> str: - return self._files[path] - - async def write(self, path: str, content: str) -> None: - self._files[path] = content - - @property - def files(self) -> dict[str, str]: - return self._files - - -def _override_fs(files: dict[str, str]) -> StubFS: - fs = StubFS(files) - - def _provide_fs(): - return fs - - app.dependency_overrides[get_filesystem] = _provide_fs - return fs - - -def _client() -> TestClient: - return TestClient(app) - - -def teardown_function(_): - app.dependency_overrides.pop(get_filesystem, None) - - -# ============================================================================ -# /files/insert_after -# ============================================================================ - - -def test_insert_after_happy_path(): - """The canonical use case: insert a new section under an existing heading.""" - fs = _override_fs( - { - "/x.md": ( - "# Doc\n\n" - "## 1. First\n" - "body 1\n\n" - "## 2. Second\n" - "body 2\n" - ) - } - ) - r = _client().post( - "/files/insert_after", - json={ - "path": "/x.md", - "anchor": "## 2. Second", - "content": "## 3. Third\n\nbody 3\n", - }, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == ( - "# Doc\n\n" - "## 1. First\n" - "body 1\n\n" - "## 2. Second\n" - "## 3. Third\n\nbody 3\n" - "body 2\n" - ) - - -def test_insert_after_anchor_missing_returns_400(): - fs = _override_fs({"/x.md": "# Doc\n\ncontent\n"}) - r = _client().post( - "/files/insert_after", - json={"path": "/x.md", "anchor": "nonexistent heading", "content": "new\n"}, - ) - assert r.status_code == 400 - assert "Anchor not found" in r.json()["detail"] - assert fs.files["/x.md"] == "# Doc\n\ncontent\n" # unchanged - - -def test_insert_after_ambiguous_match_refused_by_default(): - fs = _override_fs({"/x.md": "## same\nfoo\n## same\nbar\n"}) - r = _client().post( - "/files/insert_after", - json={"path": "/x.md", "anchor": "## same", "content": "added\n"}, - ) - assert r.status_code == 400 - assert "allow_multiple is false" in r.json()["detail"] - assert fs.files["/x.md"] == "## same\nfoo\n## same\nbar\n" - - -def test_insert_after_ambiguous_match_allowed_with_flag(): - fs = _override_fs({"/x.md": "## same\nfoo\n## same\nbar\n"}) - r = _client().post( - "/files/insert_after", - json={ - "path": "/x.md", - "anchor": "## same", - "content": "added\n", - "allow_multiple": True, - }, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "## same\nadded\nfoo\n## same\nadded\nbar\n" - - -def test_insert_after_adds_trailing_newline_to_content(): - """Caller passes content without trailing newline; handler should add one.""" - fs = _override_fs({"/x.md": "anchor line\nafter\n"}) - r = _client().post( - "/files/insert_after", - json={"path": "/x.md", "anchor": "anchor", "content": "no-newline"}, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "anchor line\nno-newline\nafter\n" - - -def test_insert_after_eof_anchor_without_trailing_newline(): - """Edge case: anchor is the last line of a file with no trailing newline.""" - fs = _override_fs({"/x.md": "first\nlast-line-no-newline"}) - r = _client().post( - "/files/insert_after", - json={"path": "/x.md", "anchor": "last-line", "content": "appended\n"}, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "first\nlast-line-no-newline\nappended\n" - - -def test_insert_after_file_not_found(): - _override_fs({}) - r = _client().post( - "/files/insert_after", - json={"path": "/missing.md", "anchor": "x", "content": "y"}, - ) - assert r.status_code == 404 - - -# ============================================================================ -# /files/append_to_section -# ============================================================================ - - -SECTIONED_DOC = ( - "# Document\n" - "\n" - "## 1. First Section\n" - "first body\n" - "\n" - "### 1.1 Sub-section\n" - "sub body\n" - "\n" - "## 2. Second Section\n" - "second body\n" - "\n" - "## 3. Third Section\n" - "third body\n" -) - - -def test_append_to_section_inserts_before_next_same_depth_heading(): - fs = _override_fs({"/x.md": SECTIONED_DOC}) - r = _client().post( - "/files/append_to_section", - json={ - "path": "/x.md", - "heading": "## 2. Second Section", - "content": "new tail content\n", - }, - ) - assert r.status_code == 200, r.text - expected = ( - "# Document\n" - "\n" - "## 1. First Section\n" - "first body\n" - "\n" - "### 1.1 Sub-section\n" - "sub body\n" - "\n" - "## 2. Second Section\n" - "second body\n" - "\n" - "new tail content\n" - "## 3. Third Section\n" - "third body\n" - ) - assert fs.files["/x.md"] == expected - - -def test_append_to_section_skips_deeper_subheadings(): - """Adding to section 1 must skip past its h3 sub-section and stop at section 2.""" - fs = _override_fs({"/x.md": SECTIONED_DOC}) - r = _client().post( - "/files/append_to_section", - json={ - "path": "/x.md", - "heading": "## 1. First Section", - "content": "added to section 1\n", - }, - ) - assert r.status_code == 200, r.text - # The added content should land just before "## 2. Second Section", - # i.e. AFTER the sub-section body. - content = fs.files["/x.md"] - assert content.index("added to section 1") < content.index("## 2.") - assert content.index("### 1.1") < content.index("added to section 1") - - -def test_append_to_section_handles_eof(): - """Last section in file should append at EOF.""" - fs = _override_fs({"/x.md": SECTIONED_DOC}) - r = _client().post( - "/files/append_to_section", - json={ - "path": "/x.md", - "heading": "## 3. Third Section", - "content": "trailing\n", - }, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"].endswith("third body\ntrailing\n") - - -def test_append_to_section_missing_heading_returns_400(): - fs = _override_fs({"/x.md": SECTIONED_DOC}) - r = _client().post( - "/files/append_to_section", - json={ - "path": "/x.md", - "heading": "## 99. Nonexistent", - "content": "x\n", - }, - ) - assert r.status_code == 400 - assert "Markdown heading not found" in r.json()["detail"] - assert fs.files["/x.md"] == SECTIONED_DOC - - -def test_append_to_section_hashtag_is_not_a_heading(): - """`#hashtag` (no space after #) must not match as a markdown heading.""" - fs = _override_fs({"/x.md": "#hashtag\nbody\n"}) - r = _client().post( - "/files/append_to_section", - json={"path": "/x.md", "heading": "#hashtag", "content": "x\n"}, - ) - assert r.status_code == 400 # no actual heading present - - -def test_append_to_section_file_not_found(): - _override_fs({}) - r = _client().post( - "/files/append_to_section", - json={"path": "/missing.md", "heading": "## x", "content": "y"}, - ) - assert r.status_code == 404 - - -# ============================================================================ -# /files/append -# ============================================================================ - - -def test_append_basic(): - fs = _override_fs({"/x.md": "line one\n"}) - r = _client().post( - "/files/append", - json={"path": "/x.md", "content": "line two\n"}, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "line one\nline two\n" - - -def test_append_inserts_separating_newline_when_missing(): - """Existing file with no trailing newline shouldn't get content stuck on the last line.""" - fs = _override_fs({"/x.md": "line one"}) - r = _client().post( - "/files/append", - json={"path": "/x.md", "content": "line two\n"}, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "line one\nline two\n" - - -def test_append_adds_trailing_newline_to_content(): - fs = _override_fs({"/x.md": "existing\n"}) - r = _client().post( - "/files/append", - json={"path": "/x.md", "content": "no-newline"}, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "existing\nno-newline\n" - - -def test_append_empty_existing_file(): - fs = _override_fs({"/x.md": ""}) - r = _client().post( - "/files/append", - json={"path": "/x.md", "content": "first content\n"}, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "first content\n" - - -def test_append_file_not_found(): - _override_fs({}) - r = _client().post( - "/files/append", - json={"path": "/missing.md", "content": "x"}, - ) - assert r.status_code == 404 - - -# ============================================================================ -# Updated defensive check (homelab#107) should now reference the new ops -# ============================================================================ - - -def test_defensive_check_message_references_new_ops(): - """The replace defensive check's hint should now name insert_after / append_*.""" - fs = _override_fs({"/x.md": "existing\n"}) - r = _client().post( - "/files/replace", - json={ - "path": "/x.md", - "replacements": [ - {"target": "## New", "replacement": "## New\n\nbody\n"} - ], - }, - ) - assert r.status_code == 400, r.text - detail = r.json()["detail"] - # Smoking-gun pattern still triggers - assert "first line of the replacement" in detail - # New op names mentioned (closes the loop with homelab#108) - assert "insert_after" in detail - assert "append_to_section" in detail - assert "append_file_content" in detail diff --git a/tests/test_log_retention_sweep.py b/tests/test_log_retention_sweep.py deleted file mode 100644 index 62304d0..0000000 --- a/tests/test_log_retention_sweep.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -Tests for the filesystem-based process-log retention sweep (homelab#720). - -Bug: _cleanup_expired() only deletes a log file when it *also* has a -matching in-memory BackgroundProcess record. _processes is in-memory and -doesn't survive a pod restart, but the log files live on a persistent -volume that does -- so any log file older than the last restart was -permanently unreachable by that path regardless of age. Surfaced live: a -process log from ~2 months prior was still present, containing a -plaintext GITHUB_APP_PRIVATE_KEY from a command that had echoed it. - -Fix: _sweep_expired_log_files() reads mtimes directly off disk instead, -independent of any in-memory process record. -""" - -from __future__ import annotations - -import os -import time - -import open_terminal.main as main - - -def _touch(path: str, mtime: float) -> None: - with open(path, "w") as f: - f.write("{}") - os.utime(path, (mtime, mtime)) - - -def test_deletes_files_older_than_retention(tmp_path): - now = time.time() - old = tmp_path / "old.jsonl" - _touch(str(old), now - (main.PROCESS_LOG_RETENTION + 10)) - - deleted = main._sweep_expired_log_files(str(tmp_path), now=now) - - assert str(old) in deleted - assert not old.exists() - - -def test_keeps_files_within_retention(tmp_path): - now = time.time() - recent = tmp_path / "recent.jsonl" - _touch(str(recent), now - 10) - - deleted = main._sweep_expired_log_files(str(tmp_path), now=now) - - assert deleted == [] - assert recent.exists() - - -def test_survives_across_a_reset_process_registry(tmp_path): - """The exact bug: an old file with no in-memory record must still go.""" - now = time.time() - orphaned = tmp_path / "orphaned-no-in-memory-record.jsonl" - _touch(str(orphaned), now - (main.PROCESS_LOG_RETENTION + 1)) - main._processes.clear() # simulates a pod restart wiping the registry - - deleted = main._sweep_expired_log_files(str(tmp_path), now=now) - - assert str(orphaned) in deleted - assert not orphaned.exists() - - -def test_ignores_non_jsonl_files(tmp_path): - now = time.time() - other = tmp_path / "old.txt" - _touch(str(other), now - (main.PROCESS_LOG_RETENTION + 10)) - - deleted = main._sweep_expired_log_files(str(tmp_path), now=now) - - assert deleted == [] - assert other.exists() - - -def test_missing_directory_returns_empty_without_raising(tmp_path): - missing = tmp_path / "does-not-exist" - - deleted = main._sweep_expired_log_files(str(missing)) - - assert deleted == [] diff --git a/tests/test_process_expiry.py b/tests/test_process_expiry.py deleted file mode 100644 index 1f4c602..0000000 --- a/tests/test_process_expiry.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Tests for the two-tier process-expiry fix (open-terminal#13 / homelab#391). - -Bug: a finished process's in-memory record was auto-deleted 300s after -completion regardless of whether anyone had actually retrieved its -result yet. A caller whose own dispatch loop stalls past that window -(the OWUI tool-dispatch hang tracked in homelab#391 is reportedly -unbounded, not just ≥300s) would come back to "Process not found" — -permanent, silent loss of a command that actually succeeded. - -Fix: track delivered_at separately from finished_at. Undelivered -results get PROCESS_UNDELIVERED_EXPIRY (long); delivered results get -the original short PROCESS_EXPIRY, since the caller already has what -it needs. -""" - -from __future__ import annotations - -import time - -import open_terminal.main as main - - -def _fake_process(process_id: str, *, finished_at: float | None, - delivered_at: float | None = None) -> main.BackgroundProcess: - return main.BackgroundProcess( - id=process_id, - command="echo hi", - runner=None, # not touched by _cleanup_expired - status="done" if finished_at else "running", - finished_at=finished_at, - delivered_at=delivered_at, - ) - - -def setup_function(_): - main._processes.clear() - - -def test_undelivered_process_survives_past_short_expiry(): - now = time.time() - main._processes["p1"] = _fake_process( - "p1", finished_at=now - (main.PROCESS_EXPIRY + 5)) - main._cleanup_expired() - assert "p1" in main._processes - - -def test_undelivered_process_expires_after_long_window(): - now = time.time() - main._processes["p1"] = _fake_process( - "p1", finished_at=now - (main.PROCESS_UNDELIVERED_EXPIRY + 5)) - main._cleanup_expired() - assert "p1" not in main._processes - - -def test_delivered_process_expires_on_short_window_not_long_one(): - now = time.time() - main._processes["p1"] = _fake_process( - "p1", - finished_at=now - (main.PROCESS_UNDELIVERED_EXPIRY - 5), - delivered_at=now - (main.PROCESS_EXPIRY + 5), - ) - main._cleanup_expired() - assert "p1" not in main._processes - - -def test_delivered_process_survives_within_short_window(): - now = time.time() - main._processes["p1"] = _fake_process( - "p1", finished_at=now - 10, delivered_at=now - 10) - main._cleanup_expired() - assert "p1" in main._processes - - -def test_running_process_never_expires(): - main._processes["p1"] = _fake_process("p1", finished_at=None) - main._cleanup_expired() - assert "p1" in main._processes diff --git a/tests/test_replace_defensive.py b/tests/test_replace_defensive.py deleted file mode 100644 index 0574bcc..0000000 --- a/tests/test_replace_defensive.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -Regression tests for the /files/replace defensive check (homelab#107). - -The check refuses a call where `target` equals the first non-empty line of -`replacement` — the signature of "model wants to insert a new section but -expressed it as a find-and-replace at an anchor it just invented." Without -this check, the call would get a generic 400 "Target string not found" and -the model typically follows up with "the file appears to have been truncated" -and goes into an unproductive re-read loop. -""" - -from fastapi.testclient import TestClient - -from open_terminal.main import app, get_filesystem, verify_api_key - - -# Stub out auth for all tests so we exercise the handler logic, not the -# auth path. The auth contract is tested elsewhere (or could be added in -# a separate file); these tests are about the /files/replace defensive check. -async def _noop_auth(): - return None - - -app.dependency_overrides[verify_api_key] = _noop_auth - - -class StubFS: - """In-memory UserFS stand-in for the handler's three filesystem touchpoints.""" - - def __init__(self, files: dict[str, str]): - self._files = dict(files) - - def resolve_path(self, path: str) -> str: - return path - - async def isfile(self, path: str) -> bool: - return path in self._files - - async def read_text(self, path: str) -> str: - return self._files[path] - - async def write(self, path: str, content: str) -> None: - self._files[path] = content - - @property - def files(self) -> dict[str, str]: - return self._files - - -def _override_fs(files: dict[str, str]) -> StubFS: - fs = StubFS(files) - - def _provide_fs(): - return fs - - app.dependency_overrides[get_filesystem] = _provide_fs - return fs - - -def _client() -> TestClient: - return TestClient(app) - - -def teardown_function(_): - # Drop per-test fs override but keep the module-level auth no-op so - # subsequent tests don't re-trip auth. - app.dependency_overrides.pop(get_filesystem, None) - - -# --- The defensive check itself ------------------------------------------------- - - -def test_target_equals_first_line_of_replacement_is_refused(): - """Smoking-gun pattern: target == replacement[0]. Refuse with hint, file untouched.""" - fs = _override_fs({"/x.md": "# Existing\n\nbody\n"}) - r = _client().post( - "/files/replace", - json={ - "path": "/x.md", - "replacements": [ - { - "target": "### New Section", - "replacement": "### New Section\n\nnew body\n", - } - ], - }, - ) - - assert r.status_code == 400, r.text - detail = r.json()["detail"] - assert "first line of the replacement" in detail - # Error hint should still mention read_file (for anchor discovery). - # The mention of the new insert/append ops is covered specifically by - # test_insert_append.py::test_defensive_check_message_references_new_ops. - assert "read_file" in detail - # File must be unchanged. - assert fs.files["/x.md"] == "# Existing\n\nbody\n" - - -def test_leading_blank_lines_in_replacement_dont_bypass_the_check(): - """First *non-empty* line is what counts, not the literal first line.""" - fs = _override_fs({"/x.md": "# Existing\n"}) - r = _client().post( - "/files/replace", - json={ - "path": "/x.md", - "replacements": [ - { - "target": "## My Section", - "replacement": "\n\n## My Section\n\nbody\n", - } - ], - }, - ) - assert r.status_code == 400, r.text - assert "first line of the replacement" in r.json()["detail"] - assert fs.files["/x.md"] == "# Existing\n" - - -def test_whitespace_only_target_does_not_trip_the_check(): - """Empty/whitespace target shouldn't trigger the defensive branch — let the - existing 'Target string not found' path handle it normally.""" - _override_fs({"/x.md": "hello world\n"}) - r = _client().post( - "/files/replace", - json={ - "path": "/x.md", - "replacements": [{"target": " ", "replacement": " replacement\n"}], - }, - ) - # Whatever the existing handler returns is fine; only assert we didn't - # short-circuit with the defensive check's message. - assert "first line of the replacement" not in r.text - - -# --- Regressions: legitimate replaces must still work -------------------------- - - -def test_target_in_middle_of_replacement_still_works(): - """The AC's explicit regression case. Anchor appears IN the replacement - (e.g., wrap an existing string in surrounding context) — must succeed.""" - fs = _override_fs({"/x.md": "before\nFOO\nafter\n"}) - r = _client().post( - "/files/replace", - json={ - "path": "/x.md", - "replacements": [ - { - "target": "FOO", - "replacement": "header\nFOO\ntrailing", - } - ], - }, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "before\nheader\nFOO\ntrailing\nafter\n" - - -def test_simple_replace_unaffected(): - """Baseline: normal find-and-replace where target != replacement[0] works.""" - fs = _override_fs({"/x.md": "alpha BETA gamma\n"}) - r = _client().post( - "/files/replace", - json={ - "path": "/x.md", - "replacements": [{"target": "BETA", "replacement": "DELTA"}], - }, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "alpha DELTA gamma\n" - - -def test_multiline_replacement_with_different_first_line_works(): - """Replace a heading with a multi-line block whose first line is different.""" - fs = _override_fs({"/x.md": "## Old Heading\n\nold body\n"}) - r = _client().post( - "/files/replace", - json={ - "path": "/x.md", - "replacements": [ - { - "target": "## Old Heading\n\nold body\n", - "replacement": "## Renamed Heading\n\nrewritten body\n", - } - ], - }, - ) - assert r.status_code == 200, r.text - assert fs.files["/x.md"] == "## Renamed Heading\n\nrewritten body\n"