Feature: Virtual Desktop & noVNC Integration for GUI-based "Computer Use" capabilities.

Dockerfile

@@ -29,13 +29,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates gnupg apt-transport-https \
     # Capabilities (needed for setcap on Python binary)
     libcap2-bin \
+    # Virtual desktop ("Computer Use")
+    xvfb x11vnc novnc openbox xdotool scrot xauth \
+    xterm x11-xserver-utils \
+    fonts-liberation fonts-noto-color-emoji \
+    dmz-cursor-theme \
     && rm -rf /var/lib/apt/lists/*
 
 # Node.js (LTS)
 RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/*
 
+# Chromium (for headful browser automation via the virtual desktop)
+RUN apt-get update && apt-get install -y --no-install-recommends chromium \
+    && rm -rf /var/lib/apt/lists/*
+
 # Docker CLI + Compose + Buildx (mount socket at runtime for access)
 RUN curl -fsSL https://get.docker.com | sh
 
@@ -68,12 +77,22 @@ RUN pip install --no-cache-dir . \
 
 RUN useradd -m -s /bin/bash user && echo 'user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
+# Chromium needs a writable /dev/shm for shared memory.  When running
+# without --no-sandbox (the default inside Docker) we need at least 64 MB.
+# Some container runtimes mount /dev/shm as 64 MB which is too small for
+# Chromium; the kernel will silently OOM the renderer.  We work around
+# this by creating a small tmpfs in the user's home directory.
+RUN echo "kernel.shmmax = 268435456" >> /etc/sysctl.conf || true
+
+RUN printf '#!/bin/sh\nexport CHROMIUM_FLAGS="$CHROMIUM_FLAGS --no-sandbox --disable-gpu --disable-software-rasterizer"\n' \
+    > /etc/chromium.d/00-container
+
 USER user
 ENV SHELL=/bin/bash
 ENV PATH="/home/user/.local/bin:${PATH}"
 WORKDIR /home/user
 
-EXPOSE 8000
+EXPOSE 8000 6080
 
 COPY entrypoint.sh /app/entrypoint.sh
 

entrypoint.sh

@@ -81,6 +81,32 @@ if [ -n "${OPEN_TERMINAL_NPM_PACKAGES:-}" ]; then
     fi
 fi
 
+# -----------------------------------------------------------------------
+# Virtual Desktop ("Computer Use")
+#
+# When OPEN_TERMINAL_ENABLE_DESKTOP is true, start Xvfb, x11vnc, and
+# noVNC so that the agent has a virtual display for GUI interaction.
+# The Python app will manage the actual lifecycle (start/stop) via the
+# DesktopManager, but we pre-seed DISPLAY and clean up stale lock files
+# so the first API call starts faster.
+# -----------------------------------------------------------------------
+if [ "${OPEN_TERMINAL_ENABLE_DESKTOP:-false}" = "true" ]; then
+    DISPLAY="${OPEN_TERMINAL_DESKTOP_DISPLAY:-:0}"
+    SCREEN="${OPEN_TERMINAL_DESKTOP_SCREEN_SIZE:-1280x720x24}"
+    VNC_PORT="${OPEN_TERMINAL_DESKTOP_VNC_PORT:-5900}"
+    NOVNC_PORT="${OPEN_TERMINAL_DESKTOP_NOVNC_PORT:-6080}"
+
+    # Clean up stale lock/pid files from previous runs
+    DISPLAY_NUM="${DISPLAY#:}"
+    DISPLAY_NUM="${DISPLAY_NUM%%.*}"
+    rm -f "/tmp/.X${DISPLAY_NUM}-lock" "/tmp/.X11-unix/X${DISPLAY_NUM}" 2>/dev/null || true
+
+    export DISPLAY
+    echo "Virtual desktop configured: display=${DISPLAY} screen=${SCREEN}"
+    echo "  VNC port: ${VNC_PORT}  |  noVNC port: ${NOVNC_PORT}"
+    echo "  Access noVNC at: http://localhost:${NOVNC_PORT}/vnc.html"
+fi
+
 # -----------------------------------------------------------------------
 # Network egress filtering via DNS whitelist + iptables + capability drop
 #

open_terminal/env.py

@@ -170,4 +170,37 @@ def _resolve_file_env(var: str, default: str = "") -> str:
     )
 )
 
+# ---------------------------------------------------------------------------
+# Virtual Desktop ("Computer Use")
+# ---------------------------------------------------------------------------
+
+ENABLE_DESKTOP = os.environ.get(
+    "OPEN_TERMINAL_ENABLE_DESKTOP",
+    str(config.get("enable_desktop", False)),
+).lower() not in ("false", "0", "no", "")
+
+DESKTOP_DISPLAY = os.environ.get(
+    "OPEN_TERMINAL_DESKTOP_DISPLAY",
+    config.get("desktop_display", ":0"),
+)
+
+DESKTOP_SCREEN_SIZE = os.environ.get(
+    "OPEN_TERMINAL_DESKTOP_SCREEN_SIZE",
+    config.get("desktop_screen_size", "1280x720x24"),
+)
+
+DESKTOP_VNC_PORT = int(
+    os.environ.get(
+        "OPEN_TERMINAL_DESKTOP_VNC_PORT",
+        config.get("desktop_vnc_port", "5900"),
+    )
+)
+
+DESKTOP_NOVNC_PORT = int(
+    os.environ.get(
+        "OPEN_TERMINAL_DESKTOP_NOVNC_PORT",
+        config.get("desktop_novnc_port", "6080"),
+    )
+)
+