diff --git a/packages/coding-agent/.changes/eng-5343-helper-uv-verification.md b/packages/coding-agent/.changes/eng-5343-helper-uv-verification.md new file mode 100644 index 0000000000..d1d6ee3735 --- /dev/null +++ b/packages/coding-agent/.changes/eng-5343-helper-uv-verification.md @@ -0,0 +1 @@ +- Changed helper provisioning to download pinned fd, ripgrep, and uv releases, verify their SHA-256 and archive contents before installing into `~/.prime/agent/bin`, and clean up partial downloads; uv is no longer installed by piping a remote script into `sh`. diff --git a/packages/coding-agent/docs/rlm-runtime.md b/packages/coding-agent/docs/rlm-runtime.md index 63625ccbc9..285fa72280 100644 --- a/packages/coding-agent/docs/rlm-runtime.md +++ b/packages/coding-agent/docs/rlm-runtime.md @@ -81,6 +81,8 @@ The kernel is created lazily on first Python REPL use. Python resolution is: The managed environment includes Python 3.11, `prime-agent-runtime`, `dill`, and the default Python packages. A bootstrap marker detects stale environments. +When no `uv` is found on `PATH`, in `~/.prime/agent/bin`, or in `~/.local/bin`, the bootstrap offers to install one. Set `PRIME_AGENT_INSTALL_UV=1` to accept without a prompt (the installer does this) or `PRIME_AGENT_INSTALL_UV=0` to refuse. The install downloads the pinned `uv` release archive from GitHub, verifies it against the SHA-256 bundled in `src/utils/helper-tool-releases.ts`, checks that the extracted binary runs, and only then places it in `~/.prime/agent/bin/uv`; no remote script is executed. The optional `fd` and `rg` search helpers are provisioned into the same directory the same way. Pinned versions are bumped with `scripts/pin-helper-tools.ts`. + Startup spawns `python -m rlm.repl` and exchanges newline-delimited JSON over stdio: the runtime announces itself with a single `ready` event, then requests and events flow one JSON object per line (see `prime-agent-runtime/src/rlm/repl.md`). The manager owns the child process and a bounded stderr tail. Shutdown sends a `shutdown` request, waits for the process to exit, and terminates it as a fallback. Persistent sessions may snapshot the kernel namespace into their session artifact directory for revival. diff --git a/packages/coding-agent/scripts/pin-helper-tools.ts b/packages/coding-agent/scripts/pin-helper-tools.ts new file mode 100644 index 0000000000..c2bf8e7345 --- /dev/null +++ b/packages/coding-agent/scripts/pin-helper-tools.ts @@ -0,0 +1,59 @@ +// Recompute the pinned SHA-256 table for a helper tool release. +// +// Usage (from packages/coding-agent): npx tsx scripts/pin-helper-tools.ts +// +// Downloads every supported asset of that release from GitHub, prints its SHA-256, and +// cross-checks the upstream `.sha256` file when the project publishes one. Paste +// the printed table into src/utils/helper-tool-releases.ts together with the new version. +import { createHash } from "node:crypto"; +import { HELPER_TOOL_RELEASES, type HelperToolId } from "../src/utils/helper-tool-releases.js"; + +const SUPPORTED_TARGETS: Array<[platform: string, architecture: string]> = [ + ["darwin", "arm64"], + ["darwin", "x64"], + ["linux", "arm64"], + ["linux", "x64"], + ["win32", "arm64"], + ["win32", "x64"], +]; + +async function fetchBytes(url: string): Promise { + const response = await fetch(url); + if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); + return new Uint8Array(await response.arrayBuffer()); +} + +async function main(): Promise { + const [tool, version] = process.argv.slice(2); + if (!tool || !version || !(tool in HELPER_TOOL_RELEASES)) { + console.error("usage: npx tsx scripts/pin-helper-tools.ts "); + process.exit(2); + } + const current = HELPER_TOOL_RELEASES[tool as HelperToolId]; + const tag = current.tag.startsWith("v") ? `v${version}` : version; + const lines: string[] = []; + let failed = false; + for (const [platform, architecture] of SUPPORTED_TARGETS) { + const assetName = current.assetName(platform, architecture)?.replaceAll(current.version, version); + if (!assetName) continue; + const url = `https://github.com/${current.repo}/releases/download/${tag}/${assetName}`; + const digest = createHash("sha256").update(await fetchBytes(url)).digest("hex"); + let note = "no upstream checksum published"; + try { + const published = new TextDecoder().decode(await fetchBytes(`${url}.sha256`)); + const match = published.match(/[0-9a-f]{64}/i)?.[0].toLowerCase(); + note = match === digest ? "matches upstream .sha256" : `MISMATCH with upstream .sha256 (${match ?? "unparseable"})`; + if (match !== digest) failed = true; + } catch { + // Only ripgrep and uv publish per-asset checksum files. + } + lines.push(`\t\t\t"${assetName}": "${digest}", // ${note}`); + } + console.log(`// ${current.repo} ${tag}\n${lines.join("\n")}`); + if (failed) { + console.error("Upstream checksum mismatch detected; do not pin this release."); + process.exit(1); + } +} + +await main(); diff --git a/packages/coding-agent/src/core/kernel/bootstrap.ts b/packages/coding-agent/src/core/kernel/bootstrap.ts index 5ab93cc64e..c6daa7dd85 100644 --- a/packages/coding-agent/src/core/kernel/bootstrap.ts +++ b/packages/coding-agent/src/core/kernel/bootstrap.ts @@ -7,9 +7,11 @@ import { stderr, stdin } from "node:process"; import { createInterface } from "node:readline/promises"; import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; -import { getPackageDir } from "../../config.js"; +import { getBinDir, getPackageDir } from "../../config.js"; import { isProcessAlive, spawnHidden } from "../../utils/child-process.js"; import { tryAcquireDirLock } from "../../utils/dir-lock.js"; +import { installPinnedHelperTool } from "../../utils/helper-tool-install.js"; +import { HELPER_TOOL_RELEASES } from "../../utils/helper-tool-releases.js"; import type { PythonSkillRuntimeInfo } from "../skills.js"; const BOOTSTRAP_SCHEMA = 9; @@ -71,7 +73,9 @@ export function buildBatchShimInvocation( }; } -const UV_INSTALL_COMMAND = "curl -LsSf https://astral.sh/uv/install.sh | sh"; +const UV_RELEASE = HELPER_TOOL_RELEASES.uv; +const UV_INSTALL_DOCS_URL = "https://docs.astral.sh/uv/getting-started/installation/"; +const UV_DOWNLOAD_TIMEOUT_MS = 120_000; const REQUIRED_HARNESS_METHODS = [ "create_memory", "update_memory", @@ -542,35 +546,57 @@ async function findExecutable(name: string): Promise { return null; } +function uvBinaryFileName(): string { + return process.platform === "win32" ? "uv.exe" : "uv"; +} + +async function uvWorks(uv: string): Promise { + try { + await run(uv, ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + async function ensureUv(options: EnsureKernelPythonOptions): Promise { const fromPath = await findExecutable("uv"); if (fromPath) return fromPath; - const localUv = path.join(os.homedir(), ".local", "bin", process.platform === "win32" ? "uv.exe" : "uv"); + const managedUv = path.join(getBinDir(), uvBinaryFileName()); + if (await isExecutable(managedUv)) return managedUv; + + const localUv = path.join(os.homedir(), ".local", "bin", uvBinaryFileName()); if (await isExecutable(localUv)) return localUv; const shouldInstallUv = process.env.PRIME_AGENT_INSTALL_UV === "1" || (!options.onProgress && (await confirmUvInstall())); if (!shouldInstallUv) { throw new Error( - `uv is required to set up the Python kernel. Install uv yourself: ${UV_INSTALL_COMMAND}, ` + - "or set PRIME_AGENT_INSTALL_UV=1 to let prime-agent run that installer.", + `uv is required to set up the Python kernel. Install uv yourself (${UV_INSTALL_DOCS_URL}), ` + + `or set PRIME_AGENT_INSTALL_UV=1 to let prime-agent download uv ${UV_RELEASE.version} from GitHub releases with SHA-256 verification.`, ); } - reportProgress(options, "› installing uv (one-time)…"); + reportProgress(options, `› installing uv ${UV_RELEASE.version} (one-time)…`); try { - await run("sh", ["-c", UV_INSTALL_COMMAND], { stdio: options.onProgress ? "ignore" : "inherit" }); + // Downloads the pinned release archive from GitHub and checks its bundled SHA-256 + // before extraction; nothing from the network is ever piped into a shell. + return await installPinnedHelperTool({ + tool: "uv", + platform: process.platform, + architecture: process.arch, + destDir: getBinDir(), + binaryFileName: uvBinaryFileName(), + verifyBinary: uvWorks, + timeoutMs: UV_DOWNLOAD_TIMEOUT_MS, + }); } catch (error) { throw new Error( - `couldn't install uv from astral.sh; install it yourself: ${UV_INSTALL_COMMAND}, then re-run prime-agent. ${errorMessage(error)}`, + `couldn't install uv ${UV_RELEASE.version} from ${UV_RELEASE.repo} releases: ${errorMessage(error)}. ` + + `Install uv yourself (${UV_INSTALL_DOCS_URL}), then re-run prime-agent.`, ); } - - if (await isExecutable(localUv)) return localUv; - const installedFromPath = await findExecutable("uv"); - if (installedFromPath) return installedFromPath; - throw new Error("uv install completed but binary not found at ~/.local/bin/uv"); } async function confirmUvInstall(): Promise { @@ -579,7 +605,11 @@ async function confirmUvInstall(): Promise { const rl = createInterface({ input: stdin, output: stderr }); try { - const answer = (await rl.question("Prime Agent needs uv to set up Python. Install uv from astral.sh now? [Y/n] ")) + const answer = ( + await rl.question( + `Prime Agent needs uv to set up Python. Download uv ${UV_RELEASE.version} from GitHub releases (SHA-256 verified) now? [Y/n] `, + ) + ) .trim() .toLowerCase(); return answer !== "n" && answer !== "no"; diff --git a/packages/coding-agent/src/utils/helper-tool-install.ts b/packages/coding-agent/src/utils/helper-tool-install.ts new file mode 100644 index 0000000000..aaa570d1cb --- /dev/null +++ b/packages/coding-agent/src/utils/helper-tool-install.ts @@ -0,0 +1,240 @@ +import { createHash } from "node:crypto"; +import { + chmodSync, + createWriteStream, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + renameSync, + rmSync, +} from "node:fs"; +import { basename, join, sep } from "node:path"; +import { Readable, Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream } from "node:stream/web"; +import extractZip from "extract-zip"; +import { spawnSyncHidden } from "./child-process.js"; +import { HELPER_TOOL_RELEASES, type HelperToolId, helperToolDownloadUrl } from "./helper-tool-releases.js"; + +const DEFAULT_DOWNLOAD_TIMEOUT_MS = 120_000; +const MAX_ASSET_BYTES = 256 * 1024 * 1024; +const TAR_LIST_MAX_BUFFER = 16 * 1024 * 1024; +const S_IFMT = 0o170000; +const S_IFLNK = 0o120000; + +export class UnsupportedHelperPlatformError extends Error {} +export class HelperIntegrityError extends Error {} +export class UnsafeArchiveMemberError extends Error {} + +export interface DownloadVerifiedOptions { + timeoutMs?: number; + maxBytes?: number; +} + +/** Download `url` to `dest`, hashing the stream; `dest` is removed unless the SHA-256 matches. */ +export async function downloadVerified( + url: string, + dest: string, + expectedSha256: string, + options: DownloadVerifiedOptions = {}, +): Promise { + if (!/^[0-9a-f]{64}$/.test(expectedSha256)) { + throw new HelperIntegrityError(`No valid pinned SHA-256 for ${basename(dest)}; refusing to download`); + } + if (!url.startsWith("https://")) { + throw new HelperIntegrityError(`Refusing to download ${basename(dest)} over a non-HTTPS URL`); + } + const maxBytes = options.maxBytes ?? MAX_ASSET_BYTES; + + const response = await fetch(url, { + signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`Failed to download ${basename(dest)}: HTTP ${response.status}`); + } + if (!response.body) { + throw new Error(`Failed to download ${basename(dest)}: empty response body`); + } + + const hash = createHash("sha256"); + let received = 0; + const hasher = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + received += chunk.length; + if (received > maxBytes) { + callback(new HelperIntegrityError(`Download of ${basename(dest)} exceeded ${maxBytes} bytes`)); + return; + } + hash.update(chunk); + callback(null, chunk); + }, + }); + + try { + await pipeline( + Readable.fromWeb(response.body as unknown as ReadableStream), + hasher, + createWriteStream(dest, { flags: "wx", mode: 0o600 }), + ); + const actual = hash.digest("hex"); + if (actual !== expectedSha256) { + throw new HelperIntegrityError( + `SHA-256 mismatch for ${basename(dest)}: expected ${expectedSha256}, got ${actual}`, + ); + } + } catch (error) { + rmSync(dest, { force: true }); + throw error; + } +} + +/** Reject archive members that could write outside the extraction directory. */ +export function assertSafeArchiveMemberPath(memberPath: string): void { + if (memberPath.length === 0 || memberPath.includes("\0")) { + throw new UnsafeArchiveMemberError(`Archive member has an empty or NUL-containing path`); + } + if (/^[\\/]/.test(memberPath) || /^[A-Za-z]:/.test(memberPath)) { + throw new UnsafeArchiveMemberError(`Archive member uses an absolute path: ${memberPath}`); + } + if (memberPath.split(/[\\/]+/).includes("..")) { + throw new UnsafeArchiveMemberError(`Archive member escapes the extraction directory: ${memberPath}`); + } +} + +function listTarMembers(archivePath: string): string[] { + const result = spawnSyncHidden("tar", ["tzPf", archivePath], { + stdio: "pipe", + encoding: "utf8", + maxBuffer: TAR_LIST_MAX_BUFFER, + }); + if (result.error || result.status !== 0) { + const detail = result.error?.message ?? result.stderr?.trim() ?? "unknown error"; + throw new Error(`Failed to list ${basename(archivePath)}: ${detail}`); + } + return result.stdout.split(/\r?\n/).filter((line) => line.length > 0); +} + +/** Extract a `.tar.gz` or `.zip` into `extractDir` after validating every member path. */ +export async function extractArchiveSafely(archivePath: string, extractDir: string): Promise { + mkdirSync(extractDir, { recursive: true }); + if (archivePath.endsWith(".tar.gz")) { + for (const member of listTarMembers(archivePath)) { + assertSafeArchiveMemberPath(member); + } + const result = spawnSyncHidden("tar", ["xzf", archivePath, "-C", extractDir], { stdio: "pipe" }); + if (result.error || result.status !== 0) { + const detail = result.error?.message ?? result.stderr?.toString().trim() ?? "unknown error"; + throw new Error(`Failed to extract ${basename(archivePath)}: ${detail}`); + } + return; + } + if (archivePath.endsWith(".zip")) { + await extractZip(archivePath, { + dir: extractDir, + onEntry: (entry) => { + assertSafeArchiveMemberPath(entry.fileName); + const mode = (entry.externalFileAttributes >> 16) & 0xffff; + if ((mode & S_IFMT) === S_IFLNK) { + throw new UnsafeArchiveMemberError(`Archive member is a symbolic link: ${entry.fileName}`); + } + }, + }); + return; + } + throw new Error(`Unsupported archive format: ${basename(archivePath)}`); +} + +function findFileRecursively(rootDir: string, fileName: string): string | null { + const stack: string[] = [rootDir]; + while (stack.length > 0) { + const currentDir = stack.pop(); + if (!currentDir) continue; + for (const entry of readdirSync(currentDir, { withFileTypes: true })) { + const fullPath = join(currentDir, entry.name); + if (entry.isFile() && entry.name === fileName) return fullPath; + if (entry.isDirectory()) stack.push(fullPath); + } + } + return null; +} + +function locateExtractedBinary(extractDir: string, archiveName: string, binaryFileName: string): string { + const nested = join(extractDir, archiveName.replace(/\.(tar\.gz|zip)$/, ""), binaryFileName); + const flat = join(extractDir, binaryFileName); + const candidate = + [nested, flat].find((path) => { + try { + return lstatSync(path).isFile(); + } catch { + return false; + } + }) ?? findFileRecursively(extractDir, binaryFileName); + if (!candidate) { + throw new Error(`Binary not found in archive: expected ${binaryFileName} in ${archiveName}`); + } + if (!lstatSync(candidate).isFile()) { + throw new UnsafeArchiveMemberError(`Extracted ${binaryFileName} is not a regular file`); + } + const root = realpathSync(extractDir); + if (!realpathSync(candidate).startsWith(root + sep)) { + throw new UnsafeArchiveMemberError(`Extracted ${binaryFileName} resolves outside the extraction directory`); + } + return candidate; +} + +export interface InstallPinnedHelperToolOptions { + tool: HelperToolId; + platform: string; + architecture: string; + /** Directory the verified binary is moved into; staging happens in a sibling temp dir. */ + destDir: string; + binaryFileName: string; + /** Must return true for the staged binary (for example a `--version` run) before it is installed. */ + verifyBinary: (binaryPath: string) => Promise | boolean; + timeoutMs?: number; +} + +/** + * Download the pinned release asset for `tool`, verify its SHA-256, extract it in a + * staging directory, check the binary works, then move it to `destDir`. Nothing is left + * in `destDir` when any step fails. + */ +export async function installPinnedHelperTool(options: InstallPinnedHelperToolOptions): Promise { + const release = HELPER_TOOL_RELEASES[options.tool]; + const assetName = release.assetName(options.platform, options.architecture); + if (!assetName) { + throw new UnsupportedHelperPlatformError(`Unsupported platform: ${options.platform}/${options.architecture}`); + } + const expectedSha256 = release.sha256[assetName]; + if (!expectedSha256) { + throw new HelperIntegrityError(`No pinned SHA-256 for ${assetName}; refusing to install ${options.tool}`); + } + + mkdirSync(options.destDir, { recursive: true }); + const stagingDir = mkdtempSync(join(options.destDir, `.staging-${options.binaryFileName}-`)); + try { + const archivePath = join(stagingDir, assetName); + await downloadVerified(helperToolDownloadUrl(release, assetName), archivePath, expectedSha256, { + timeoutMs: options.timeoutMs, + }); + + const extractDir = join(stagingDir, "extract"); + await extractArchiveSafely(archivePath, extractDir); + const stagedBinary = locateExtractedBinary(extractDir, assetName, options.binaryFileName); + if (options.platform !== "win32") { + chmodSync(stagedBinary, 0o755); + } + if (!(await options.verifyBinary(stagedBinary))) { + throw new Error(`Downloaded ${options.binaryFileName} ${release.version} failed its version check`); + } + + const installedPath = join(options.destDir, options.binaryFileName); + rmSync(installedPath, { force: true }); + renameSync(stagedBinary, installedPath); + return installedPath; + } finally { + rmSync(stagingDir, { recursive: true, force: true }); + } +} diff --git a/packages/coding-agent/src/utils/helper-tool-releases.ts b/packages/coding-agent/src/utils/helper-tool-releases.ts new file mode 100644 index 0000000000..586596edf8 --- /dev/null +++ b/packages/coding-agent/src/utils/helper-tool-releases.ts @@ -0,0 +1,114 @@ +// Pinned upstream releases for the helper binaries Prime Agent downloads on demand. +// +// Every asset is verified against the SHA-256 recorded here before it is extracted, +// so a compromised release page or CDN cannot substitute a binary. Bump a version with +// `npx tsx scripts/pin-helper-tools.ts ` from packages/coding-agent, +// which downloads the pinned assets, cross-checks upstream `.sha256` files where the +// project publishes them (ripgrep, uv; fd publishes none), and prints the new table. + +export type HelperToolId = "fd" | "rg" | "uv"; + +export interface HelperToolRelease { + /** GitHub repository in `owner/name` form. */ + repo: string; + version: string; + /** Git tag of the release (fd prefixes versions with `v`). */ + tag: string; + /** Release asset for a platform/architecture, or null when unsupported. */ + assetName: (platform: string, architecture: string) => string | null; + /** Lowercase hex SHA-256 of each supported asset. */ + sha256: Readonly>; +} + +function rustTarget(platform: string, architecture: string): string | null { + const cpu = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; + if (!cpu) return null; + switch (platform) { + case "darwin": + return `${cpu}-apple-darwin`; + case "linux": + return `${cpu}-unknown-linux-gnu`; + case "win32": + return `${cpu}-pc-windows-msvc`; + default: + return null; + } +} + +function archiveExtension(platform: string): string { + return platform === "win32" ? "zip" : "tar.gz"; +} + +const FD_VERSION = "10.5.0"; +const RIPGREP_VERSION = "15.2.0"; +const UV_VERSION = "0.12.9"; + +export const HELPER_TOOL_RELEASES: Readonly> = { + fd: { + repo: "sharkdp/fd", + version: FD_VERSION, + tag: `v${FD_VERSION}`, + assetName: (platform, architecture) => { + const target = rustTarget(platform, architecture); + return target ? `fd-v${FD_VERSION}-${target}.${archiveExtension(platform)}` : null; + }, + sha256: { + "fd-v10.5.0-aarch64-apple-darwin.tar.gz": "b67e1836c468e42e411984b56e52fa7abec08c2bd22c867398e7cc134aac5e12", + "fd-v10.5.0-x86_64-apple-darwin.tar.gz": "7e31028c62c6955877735d0406807aa484c2a5e6f86235a59e26c29c301da590", + "fd-v10.5.0-aarch64-unknown-linux-gnu.tar.gz": + "c0ee43802e3313a317c5af2f4eabd6ba13eeedd595af9775f05e18a13ac4f52c", + "fd-v10.5.0-x86_64-unknown-linux-gnu.tar.gz": + "a1259cd129636efbc3fef123525c1b49e88fe5088c012630983c310e52fdfa95", + "fd-v10.5.0-aarch64-pc-windows-msvc.zip": "a2bcddcfd259b05357a77bbc6cd671fdb30f63fd266a0e748305890a8c5ceaa6", + "fd-v10.5.0-x86_64-pc-windows-msvc.zip": "a227701b8551c35a9931d9f6da75503cf86d88e182d71fb849a70864c5d57cd7", + }, + }, + rg: { + repo: "BurntSushi/ripgrep", + version: RIPGREP_VERSION, + tag: RIPGREP_VERSION, + assetName: (platform, architecture) => { + // ripgrep ships a static musl build for x86_64 Linux and only a glibc build for aarch64. + const target = + platform === "linux" && architecture === "x64" + ? "x86_64-unknown-linux-musl" + : rustTarget(platform, architecture); + return target ? `ripgrep-${RIPGREP_VERSION}-${target}.${archiveExtension(platform)}` : null; + }, + sha256: { + "ripgrep-15.2.0-aarch64-apple-darwin.tar.gz": + "3750b2e93f37e0c692657da574d7019a101c0084da05a790c83fd335bad973e4", + "ripgrep-15.2.0-x86_64-apple-darwin.tar.gz": + "af7825fcc69a2afc7a7aea55fc9af90e26421d8f20fe59df32e233c0b8a231c1", + "ripgrep-15.2.0-aarch64-unknown-linux-gnu.tar.gz": + "a740b91c82eaf9914cfedd353572f2791cbe0162c84101ee0951058f4dcbc90d", + "ripgrep-15.2.0-x86_64-unknown-linux-musl.tar.gz": + "33e15bcf1624b25cdd2a55813a47a2f95dbe126268203e76aa6a585d1e7b149c", + "ripgrep-15.2.0-aarch64-pc-windows-msvc.zip": + "e4abca10c3a64ebea742667dd7009449d49403db5460dd6873e389fa2945360f", + "ripgrep-15.2.0-x86_64-pc-windows-msvc.zip": + "71b2fef860abe467217a538ff31de02f5258807c0129f771846f87bd029aafc5", + }, + }, + uv: { + repo: "astral-sh/uv", + version: UV_VERSION, + tag: UV_VERSION, + assetName: (platform, architecture) => { + const target = rustTarget(platform, architecture); + return target ? `uv-${target}.${archiveExtension(platform)}` : null; + }, + sha256: { + "uv-aarch64-apple-darwin.tar.gz": "301f72afaf54060f92da7016cb0115bd077f43a9c8e39c1d8170a0bac80fd398", + "uv-x86_64-apple-darwin.tar.gz": "e1ca175824f1056589ce9908f7631879ebc3c36535b5e63dc06510beb370b4c1", + "uv-aarch64-unknown-linux-gnu.tar.gz": "c36fe17937ff6bd16dc42fc13854b5465999fcab2efe0af559381e945e3c6001", + "uv-x86_64-unknown-linux-gnu.tar.gz": "ec7a99cd05e0cd7f80243f135ce1361c76835cb0ee60055d14d20eba8eba1460", + "uv-aarch64-pc-windows-msvc.zip": "d3360363a3cb671f2c854f4ef48cf4a57fe8664f8ec6a248076d68b797a8acc0", + "uv-x86_64-pc-windows-msvc.zip": "ddbfcee1ac615a0499f6aa97b5ec8ebdf3ee4a7714a48055ec2ba0030e3cf810", + }, + }, +}; + +export function helperToolDownloadUrl(release: HelperToolRelease, assetName: string): string { + return `https://github.com/${release.repo}/releases/download/${release.tag}/${assetName}`; +} diff --git a/packages/coding-agent/src/utils/tools-manager.ts b/packages/coding-agent/src/utils/tools-manager.ts index 57aa77e373..67f17ac34e 100644 --- a/packages/coding-agent/src/utils/tools-manager.ts +++ b/packages/coding-agent/src/utils/tools-manager.ts @@ -1,15 +1,13 @@ import chalk from "chalk"; -import extractZip from "extract-zip"; -import { chmodSync, createWriteStream, existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "fs"; +import { existsSync } from "fs"; import { arch, platform } from "os"; import { join } from "path"; -import { Readable } from "stream"; -import { pipeline } from "stream/promises"; -import { APP_NAME, getBinDir } from "../config.js"; +import { getBinDir } from "../config.js"; import { spawnSyncHidden } from "./child-process.js"; +import { installPinnedHelperTool, UnsupportedHelperPlatformError } from "./helper-tool-install.js"; +import { HELPER_TOOL_RELEASES } from "./helper-tool-releases.js"; const TOOLS_DIR = getBinDir(); -const NETWORK_TIMEOUT_MS = 10_000; const DOWNLOAD_TIMEOUT_MS = 120_000; const COMMAND_TIMEOUT_MS = 5_000; const RIPGREP_INSTALL_URL = "https://github.com/BurntSushi/ripgrep#installation"; @@ -41,59 +39,19 @@ function isOfflineModeEnabled(): boolean { interface ToolConfig { name: string; - repo: string; // GitHub repo (e.g., "sharkdp/fd") binaryName: string; // Name of the binary inside the archive systemBinaryNames?: string[]; // Alternative system command names to try before downloading - tagPrefix: string; // Prefix for tags (e.g., "v" for v1.0.0, "" for 1.0.0) - getAssetName: (version: string, plat: string, architecture: string) => string | null; } -const TOOLS: Record = { +const TOOLS: Record = { fd: { name: "fd", - repo: "sharkdp/fd", binaryName: "fd", systemBinaryNames: ["fd", "fdfind"], - tagPrefix: "v", - getAssetName: (version, plat, architecture) => { - if (plat === "darwin") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `fd-v${version}-${archStr}-apple-darwin.tar.gz`; - } else if (plat === "linux") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `fd-v${version}-${archStr}-unknown-linux-gnu.tar.gz`; - } else if (plat === "win32") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `fd-v${version}-${archStr}-pc-windows-msvc.zip`; - } - return null; - }, }, rg: { name: "ripgrep", - repo: "BurntSushi/ripgrep", binaryName: "rg", - tagPrefix: "", - getAssetName: (version, plat, architecture) => { - if (plat === "darwin") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `ripgrep-${version}-${archStr}-apple-darwin.tar.gz`; - } else if (plat === "linux") { - if (architecture === "arm64") { - return `ripgrep-${version}-aarch64-unknown-linux-gnu.tar.gz`; - } - return architecture === "x64" ? `ripgrep-${version}-x86_64-unknown-linux-musl.tar.gz` : null; - } else if (plat === "win32") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `ripgrep-${version}-${archStr}-pc-windows-msvc.zip`; - } - return null; - }, }, }; @@ -129,145 +87,22 @@ export function getToolPath(tool: ManagedTool): string | null { return null; } -// Fetch latest release version from GitHub -async function getLatestVersion(repo: string): Promise { - const response = await fetch(`https://api.github.com/repos/${repo}/releases/latest`, { - headers: { "User-Agent": `${APP_NAME}-coding-agent` }, - signal: AbortSignal.timeout(NETWORK_TIMEOUT_MS), - }); - - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status}`); - } - - const data = (await response.json()) as { tag_name: string }; - return data.tag_name.replace(/^v/, ""); -} - -// Download a file from URL -async function downloadFile(url: string, dest: string): Promise { - const response = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), - }); - - if (!response.ok) { - throw new Error(`Failed to download: ${response.status}`); - } - - if (!response.body) { - throw new Error("No response body"); - } - - const fileStream = createWriteStream(dest); - await pipeline(Readable.fromWeb(response.body as any), fileStream); -} - -function findBinaryRecursively(rootDir: string, binaryFileName: string): string | null { - const stack: string[] = [rootDir]; - - while (stack.length > 0) { - const currentDir = stack.pop(); - if (!currentDir) continue; - - const entries = readdirSync(currentDir, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = join(currentDir, entry.name); - if (entry.isFile() && entry.name === binaryFileName) { - return fullPath; - } - if (entry.isDirectory()) { - stack.push(fullPath); - } - } - } - - return null; -} - -// Download and install a tool -class UnsupportedToolPlatformError extends Error {} - +// Download the pinned release, verify its digest, and install it into TOOLS_DIR. async function downloadTool(tool: ManagedTool): Promise { const config = TOOLS[tool]; if (!config) throw new Error(`Unknown tool: ${tool}`); const plat = platform(); - const architecture = arch(); - - if (!config.getAssetName("VERSION", plat, architecture)) { - throw new UnsupportedToolPlatformError(`Unsupported platform: ${plat}/${architecture}`); - } - - // Get latest version and the matching platform asset. - const version = await getLatestVersion(config.repo); - const assetName = config.getAssetName(version, plat, architecture); - if (!assetName) throw new UnsupportedToolPlatformError(`Unsupported platform: ${plat}/${architecture}`); - - // Create tools directory - mkdirSync(TOOLS_DIR, { recursive: true }); - - const downloadUrl = `https://github.com/${config.repo}/releases/download/${config.tagPrefix}${version}/${assetName}`; - const archivePath = join(TOOLS_DIR, assetName); - const binaryExt = plat === "win32" ? ".exe" : ""; - const binaryPath = join(TOOLS_DIR, config.binaryName + binaryExt); - - // Download - await downloadFile(downloadUrl, archivePath); - - // Extract into a unique temp directory. fd and rg downloads can run concurrently - // during startup, so sharing a fixed directory causes races. - const extractDir = join( - TOOLS_DIR, - `extract_tmp_${config.binaryName}_${process.pid}_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`, - ); - mkdirSync(extractDir, { recursive: true }); - - try { - if (assetName.endsWith(".tar.gz")) { - const extractResult = spawnSyncHidden("tar", ["xzf", archivePath, "-C", extractDir], { stdio: "pipe" }); - if (extractResult.error || extractResult.status !== 0) { - const errMsg = extractResult.error?.message ?? extractResult.stderr?.toString().trim() ?? "unknown error"; - throw new Error(`Failed to extract ${assetName}: ${errMsg}`); - } - } else if (assetName.endsWith(".zip")) { - await extractZip(archivePath, { dir: extractDir }); - } else { - throw new Error(`Unsupported archive format: ${assetName}`); - } - - // Find the binary in extracted files. Some archives contain files directly - // at root, others nest under a versioned subdirectory. - const binaryFileName = config.binaryName + binaryExt; - const extractedDir = join(extractDir, assetName.replace(/\.(tar\.gz|zip)$/, "")); - const extractedBinaryCandidates = [join(extractedDir, binaryFileName), join(extractDir, binaryFileName)]; - let extractedBinary = extractedBinaryCandidates.find((candidate) => existsSync(candidate)); - - if (!extractedBinary) { - extractedBinary = findBinaryRecursively(extractDir, binaryFileName) ?? undefined; - } - - if (extractedBinary) { - rmSync(binaryPath, { force: true }); - renameSync(extractedBinary, binaryPath); - } else { - throw new Error(`Binary not found in archive: expected ${binaryFileName} under ${extractDir}`); - } - - // Make executable (Unix only) - if (plat !== "win32") { - chmodSync(binaryPath, 0o755); - } - if (!commandWorks(binaryPath)) { - rmSync(binaryPath, { force: true }); - throw new Error(`Installed ${config.name} binary failed its version check`); - } - } finally { - // Cleanup - rmSync(archivePath, { force: true }); - rmSync(extractDir, { recursive: true, force: true }); - } - - return binaryPath; + const binaryFileName = config.binaryName + (plat === "win32" ? ".exe" : ""); + return installPinnedHelperTool({ + tool, + platform: plat, + architecture: arch(), + destDir: TOOLS_DIR, + binaryFileName, + verifyBinary: commandWorks, + timeoutMs: DOWNLOAD_TIMEOUT_MS, + }); } // Termux package names for tools @@ -352,9 +187,9 @@ export async function ensureToolWithStatus(tool: ManagedTool, silent: boolean = }; } - // Tool not found - download it + // Tool not found - download the pinned release if (!silent) { - console.log(chalk.dim(`${config.name} not found. Downloading...`)); + console.log(chalk.dim(`${config.name} not found. Downloading ${HELPER_TOOL_RELEASES[tool].version}...`)); } try { @@ -369,7 +204,7 @@ export async function ensureToolWithStatus(tool: ManagedTool, silent: boolean = } return { status: "unavailable", - reason: e instanceof UnsupportedToolPlatformError ? "unsupported_platform" : "download_failed", + reason: e instanceof UnsupportedHelperPlatformError ? "unsupported_platform" : "download_failed", platform: platformName, architecture, detail: e instanceof Error ? e.message : String(e), diff --git a/packages/coding-agent/test/archive-fixtures.ts b/packages/coding-agent/test/archive-fixtures.ts new file mode 100644 index 0000000000..b4bc5283d5 --- /dev/null +++ b/packages/coding-agent/test/archive-fixtures.ts @@ -0,0 +1,111 @@ +// Minimal tar.gz and zip writers for helper-install tests. Real archivers strip or +// refuse `..` and absolute member names, so hostile fixtures have to be hand-built. +import { crc32, gzipSync } from "node:zlib"; + +export interface ArchiveEntry { + name: string; + content?: string; + mode?: number; + /** Symbolic link target; the entry becomes a symlink when set. */ + linkTarget?: string; +} + +function octal(value: number, width: number): Buffer { + return Buffer.from(`${value.toString(8).padStart(width - 1, "0")}\0`, "latin1"); +} + +function tarHeader(entry: ArchiveEntry, size: number): Buffer { + const header = Buffer.alloc(512); + header.write(entry.name, 0, 100, "utf8"); + octal(entry.mode ?? (entry.linkTarget ? 0o777 : 0o644), 8).copy(header, 100); + octal(0, 8).copy(header, 108); + octal(0, 8).copy(header, 116); + octal(size, 12).copy(header, 124); + octal(0, 12).copy(header, 136); + header.fill(" ", 148, 156); + header.write(entry.linkTarget ? "2" : "0", 156, 1, "latin1"); + if (entry.linkTarget) header.write(entry.linkTarget, 157, 100, "utf8"); + header.write("ustar\0", 257, 6, "latin1"); + header.write("00", 263, 2, "latin1"); + let sum = 0; + for (const byte of header) sum += byte; + Buffer.from(`${sum.toString(8).padStart(6, "0")}\0 `, "latin1").copy(header, 148); + return header; +} + +export function makeTarGz(entries: ArchiveEntry[]): Buffer { + const blocks: Buffer[] = []; + for (const entry of entries) { + const data = Buffer.from(entry.linkTarget ? "" : (entry.content ?? ""), "utf8"); + blocks.push(tarHeader(entry, data.length)); + blocks.push(data); + const padding = (512 - (data.length % 512)) % 512; + if (padding) blocks.push(Buffer.alloc(padding)); + } + blocks.push(Buffer.alloc(1024)); + return gzipSync(Buffer.concat(blocks)); +} + +function u16(value: number): Buffer { + const buffer = Buffer.alloc(2); + buffer.writeUInt16LE(value); + return buffer; +} + +function u32(value: number): Buffer { + const buffer = Buffer.alloc(4); + buffer.writeUInt32LE(value >>> 0); + return buffer; +} + +export function makeZip(entries: ArchiveEntry[]): Buffer { + const local: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const data = Buffer.from(entry.linkTarget ?? entry.content ?? "", "utf8"); + const crc = crc32(data); + const mode = entry.linkTarget ? 0o120777 : (entry.mode ?? 0o644); + const fixed = Buffer.concat([ + u16(20), + u16(0), + u16(0), + u16(0), + u16(0), + u32(crc), + u32(data.length), + u32(data.length), + ]); + const localHeader = Buffer.concat([u32(0x04034b50), fixed, u16(name.length), u16(0), name]); + local.push(localHeader, data); + central.push( + Buffer.concat([ + u32(0x02014b50), + u16((3 << 8) | 20), + fixed, + u16(name.length), + u16(0), + u16(0), + u16(0), + u16(0), + u32(mode << 16), + u32(offset), + name, + ]), + ); + offset += localHeader.length + data.length; + } + const centralDirectory = Buffer.concat(central); + const end = Buffer.concat([ + u32(0x06054b50), + u16(0), + u16(0), + u16(entries.length), + u16(entries.length), + u32(centralDirectory.length), + u32(offset), + u16(0), + ]); + return Buffer.concat([...local, centralDirectory, end]); +} diff --git a/packages/coding-agent/test/helper-tool-install.test.ts b/packages/coding-agent/test/helper-tool-install.test.ts new file mode 100644 index 0000000000..f38607b3a1 --- /dev/null +++ b/packages/coding-agent/test/helper-tool-install.test.ts @@ -0,0 +1,304 @@ +// ENG-5343: fd/rg helper provisioning must verify pinned digests, contain archive +// members, and leave nothing behind when a download fails. +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { makeTarGz, makeZip } from "./archive-fixtures.js"; + +// getBinDir() is read once at tools-manager import time, so the path must exist before imports run. +const toolState = vi.hoisted(() => ({ + work: `/tmp/eng5343-helper-${process.pid}`, + toolsDir: `/tmp/eng5343-helper-${process.pid}/bin`, + platform: process.platform as string, + architecture: process.arch as string, + sha256: { fd: {} as Record, rg: {} as Record }, +})); + +vi.mock("../src/config.js", () => ({ + APP_NAME: "prime-agent", + getBinDir: () => toolState.toolsDir, +})); + +vi.mock("os", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { ...actual, arch: () => toolState.architecture, platform: () => toolState.platform }, + arch: () => toolState.architecture, + platform: () => toolState.platform, + }; +}); + +vi.mock("../src/utils/helper-tool-releases.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + HELPER_TOOL_RELEASES: { + ...actual.HELPER_TOOL_RELEASES, + fd: { + ...actual.HELPER_TOOL_RELEASES.fd, + get sha256() { + return toolState.sha256.fd; + }, + }, + rg: { + ...actual.HELPER_TOOL_RELEASES.rg, + get sha256() { + return toolState.sha256.rg; + }, + }, + }, + }; +}); + +import { assertSafeArchiveMemberPath } from "../src/utils/helper-tool-install.js"; +import { HELPER_TOOL_RELEASES, helperToolDownloadUrl } from "../src/utils/helper-tool-releases.js"; +import { ensureToolWithStatus } from "../src/utils/tools-manager.js"; + +const FD_SCRIPT = "#!/bin/sh\necho 'fd 10.5.0 MARKER-5343'\nexit 0\n"; +const originalPath = process.env.PATH; +const work = toolState.work; +let requests: string[] = []; + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function fdAsset(): string { + const asset = HELPER_TOOL_RELEASES.fd.assetName(toolState.platform, toolState.architecture); + if (!asset) throw new Error("unsupported test platform"); + return asset; +} + +function fdArchiveDir(): string { + return fdAsset().replace(/\.(tar\.gz|zip)$/, ""); +} + +function pinFd(bytes: Uint8Array): void { + toolState.sha256.fd = { [fdAsset()]: sha256(bytes) }; +} + +function serve(body: Uint8Array | (() => Response)): void { + vi.stubGlobal("fetch", async (input: unknown) => { + requests.push(String(input)); + return typeof body === "function" ? body() : new Response(body, { status: 200 }); + }); +} + +function leftovers(): string[] { + return existsSync(toolState.toolsDir) ? readdirSync(toolState.toolsDir) : []; +} + +describe("ENG-5343 helper tool provisioning", () => { + beforeEach(() => { + rmSync(work, { recursive: true, force: true }); + toolState.platform = process.platform; + toolState.architecture = process.arch; + toolState.sha256 = { fd: {}, rg: {} }; + mkdirSync(toolState.toolsDir, { recursive: true }); + process.env.PATH = "/usr/bin:/bin"; + requests = []; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + if (originalPath === undefined) delete process.env.PATH; + else process.env.PATH = originalPath; + rmSync(work, { recursive: true, force: true }); + }); + + it.skipIf(process.platform === "win32")("installs a pinned asset whose digest matches", async () => { + const archive = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + pinFd(archive); + serve(archive); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toEqual({ status: "available", path: join(toolState.toolsDir, "fd") }); + expect(readFileSync(join(toolState.toolsDir, "fd"), "utf8")).toBe(FD_SCRIPT); + expect(requests).toEqual([helperToolDownloadUrl(HELPER_TOOL_RELEASES.fd, fdAsset())]); + expect(leftovers()).toEqual(["fd"]); + }); + + it.skipIf(process.platform === "win32")("rejects a tampered archive before extracting it", async () => { + const genuine = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + const tampered = makeTarGz([ + { name: `${fdArchiveDir()}/fd`, content: "#!/bin/sh\necho tampered-fd MARKER-5343\n", mode: 0o755 }, + ]); + pinFd(genuine); + serve(tampered); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("SHA-256 mismatch"), + }); + expect(requests).toHaveLength(1); + expect(leftovers()).toEqual([]); + }); + + it("refuses to download when the release has no pinned digest for this platform", async () => { + toolState.sha256.fd = {}; + serve(new Uint8Array([1])); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("No pinned SHA-256"), + }); + expect(requests).toEqual([]); + expect(leftovers()).toEqual([]); + }); + + it.skipIf(process.platform === "win32")("rejects tar members that escape the extraction directory", async () => { + for (const hostile of [ + "../escaped-5343.txt", + `${fdArchiveDir()}/../../escaped-5343.txt`, + "/tmp/absolute-5343.txt", + ]) { + const archive = makeTarGz([ + { name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }, + { name: hostile, content: "escaped\n" }, + ]); + pinFd(archive); + serve(archive); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringMatching(/escapes the extraction directory|absolute path/), + }); + expect(existsSync(join(work, "escaped-5343.txt"))).toBe(false); + expect(existsSync("/tmp/absolute-5343.txt")).toBe(false); + expect(leftovers()).toEqual([]); + } + }); + + it.skipIf(process.platform === "win32")("rejects a binary delivered as a symlink", async () => { + const archive = makeTarGz([{ name: `${fdArchiveDir()}/fd`, linkTarget: "/bin/sh" }]); + pinFd(archive); + serve(archive); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ status: "unavailable", reason: "download_failed" }); + expect(leftovers()).toEqual([]); + }); + + it.skipIf(process.platform === "win32")("leaves nothing behind when the download is interrupted", async () => { + const archive = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + pinFd(archive); + serve(() => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(archive.subarray(0, 16)); + controller.error(new Error("connection reset")); + }, + }); + return new Response(stream, { status: 200 }); + }); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ status: "unavailable", reason: "download_failed" }); + expect(leftovers()).toEqual([]); + }); + + it.skipIf(process.platform === "win32")("keeps a working binary when a re-download fails verification", async () => { + const genuine = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + pinFd(genuine); + serve(genuine); + await expect(ensureToolWithStatus("fd")).resolves.toMatchObject({ status: "available" }); + + // Break the installed binary so the next call re-downloads, then serve a tampered asset. + rmSync(join(toolState.toolsDir, "fd")); + serve(makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: "#!/bin/sh\nexit 0\n", mode: 0o755 }])); + + await expect(ensureToolWithStatus("fd")).resolves.toMatchObject({ status: "unavailable" }); + expect(leftovers()).toEqual([]); + }); + + describe("zip assets", () => { + const rgAsset = () => { + const asset = HELPER_TOOL_RELEASES.rg.assetName("win32", "x64"); + if (!asset) throw new Error("unsupported test platform"); + return asset; + }; + const RG_SCRIPT = "#!/bin/sh\nexit 0\n"; + + beforeEach(() => { + toolState.platform = "win32"; + toolState.architecture = "x64"; + }); + + function pinRg(bytes: Uint8Array): void { + toolState.sha256.rg = { [rgAsset()]: sha256(bytes) }; + } + + it.skipIf(process.platform === "win32")("installs a verified zip asset", async () => { + const archive = makeZip([ + { name: `${rgAsset().replace(/\.zip$/, "")}/rg.exe`, content: RG_SCRIPT, mode: 0o755 }, + ]); + pinRg(archive); + serve(archive); + + await expect(ensureToolWithStatus("rg")).resolves.toEqual({ + status: "available", + path: join(toolState.toolsDir, "rg.exe"), + }); + expect(leftovers()).toEqual(["rg.exe"]); + }); + + it("rejects zip members with traversal or absolute paths", async () => { + for (const hostile of ["../escaped-5343.txt", "/tmp/absolute-5343.txt", "C:\\escaped-5343.txt"]) { + const archive = makeZip([ + { name: "rg.exe", content: RG_SCRIPT, mode: 0o755 }, + { name: hostile, content: "escaped\n" }, + ]); + pinRg(archive); + serve(archive); + + await expect(ensureToolWithStatus("rg")).resolves.toMatchObject({ + status: "unavailable", + reason: "download_failed", + }); + expect(existsSync(join(work, "escaped-5343.txt"))).toBe(false); + expect(existsSync("/tmp/absolute-5343.txt")).toBe(false); + expect(leftovers()).toEqual([]); + } + }); + + it("rejects zip members that are symbolic links", async () => { + const archive = makeZip([{ name: "rg.exe", linkTarget: "/bin/sh" }]); + pinRg(archive); + serve(archive); + + await expect(ensureToolWithStatus("rg")).resolves.toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("symbolic link"), + }); + expect(leftovers()).toEqual([]); + }); + }); + + it("validates archive member paths", () => { + expect(() => assertSafeArchiveMemberPath("fd-v10.5.0/fd")).not.toThrow(); + expect(() => assertSafeArchiveMemberPath("dir/../ok/../file")).toThrow(/escapes/); + expect(() => assertSafeArchiveMemberPath("..")).toThrow(/escapes/); + expect(() => assertSafeArchiveMemberPath("..\\evil")).toThrow(/escapes/); + expect(() => assertSafeArchiveMemberPath("/etc/passwd")).toThrow(/absolute/); + expect(() => assertSafeArchiveMemberPath("\\\\server\\share")).toThrow(/absolute/); + expect(() => assertSafeArchiveMemberPath("C:/Windows/evil")).toThrow(/absolute/); + expect(() => assertSafeArchiveMemberPath("")).toThrow(/empty/); + expect(() => assertSafeArchiveMemberPath("a\0b")).toThrow(/NUL/); + expect(() => assertSafeArchiveMemberPath("..hidden/file")).not.toThrow(); + }); +}); diff --git a/packages/coding-agent/test/kernel-bootstrap-uv.test.ts b/packages/coding-agent/test/kernel-bootstrap-uv.test.ts new file mode 100644 index 0000000000..9e2bd33a94 --- /dev/null +++ b/packages/coding-agent/test/kernel-bootstrap-uv.test.ts @@ -0,0 +1,197 @@ +// ENG-5343: the kernel bootstrap must install uv from the pinned, digest-verified +// release archive and never pipe a remote script into a shell. +import { createHash } from "node:crypto"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { makeTarGz } from "./archive-fixtures.js"; + +const uvState = vi.hoisted(() => ({ sha256: {} as Record })); + +vi.mock("../src/utils/helper-tool-releases.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + HELPER_TOOL_RELEASES: { + ...actual.HELPER_TOOL_RELEASES, + uv: { + ...actual.HELPER_TOOL_RELEASES.uv, + get sha256() { + return uvState.sha256; + }, + }, + }, + }; +}); + +import { DEFAULT_RLM_EXTRA_IMPORT_NAMES, ensureKernelPython } from "../src/core/kernel/bootstrap.js"; +import { HELPER_TOOL_RELEASES, helperToolDownloadUrl } from "../src/utils/helper-tool-releases.js"; + +let home = ""; +let originalEnv: NodeJS.ProcessEnv; +let requests: string[] = []; + +function uvAsset(): string { + const asset = HELPER_TOOL_RELEASES.uv.assetName(process.platform, process.arch); + if (!asset) throw new Error("unsupported test platform"); + return asset; +} + +function fakeUvScript(logPath: string): string { + return [ + "#!/bin/sh", + `printf '%s\\n' "$*" >> "${logPath}"`, + 'if [ "$1" = "--version" ]; then echo "uv 0.0.0-fake"; exit 0; fi', + 'if [ "$1" = "python" ]; then exit 0; fi', + 'if [ "$1" = "venv" ]; then', + ' venv="$2"; mkdir -p "$venv/bin"', + ` cat > "$venv/bin/python" <<'PY'`, + "#!/bin/sh", + 'if [ "$1" = "-c" ]; then', + ' case "$2" in', + ' "import rlm") exit 0 ;;', + ...DEFAULT_RLM_EXTRA_IMPORT_NAMES.map((name) => ` "import ${name}") exit 0 ;;`), + ' *"_harness_methods"*) exit 0 ;;', + " *) exit 1 ;;", + " esac", + "fi", + "exit 0", + "PY", + ' chmod +x "$venv/bin/python"; exit 0', + "fi", + 'if [ "$1" = "pip" ]; then exit 0; fi', + "exit 2", + "", + ].join("\n"); +} + +function uvArchive(logPath: string): Buffer { + const dir = uvAsset().replace(/\.tar\.gz$/, ""); + return makeTarGz([ + { name: `${dir}/uv`, content: fakeUvScript(logPath), mode: 0o755 }, + { name: `${dir}/uvx`, content: "#!/bin/sh\nexit 0\n", mode: 0o755 }, + ]); +} + +function serve(body: Uint8Array | (() => Response)): void { + vi.stubGlobal("fetch", async (input: unknown) => { + requests.push(String(input)); + return typeof body === "function" ? body() : new Response(body, { status: 200 }); + }); +} + +function binDir(): string { + return join(home, ".prime", "agent", "bin"); +} + +describe.skipIf(process.platform === "win32")("ENG-5343 uv bootstrap", () => { + beforeEach(() => { + originalEnv = { ...process.env }; + home = mkdtempSync(join(tmpdir(), "eng5343-uv-")); + requests = []; + uvState.sha256 = {}; + + // No uv anywhere; fake sh/curl record any attempt to run an installer script. + const fakeBin = join(home, "fake-bin"); + mkdirSync(fakeBin, { recursive: true }); + for (const name of ["sh", "curl"]) { + writeFileSync( + join(fakeBin, name), + `#!/bin/bash\nprintf '%s\\n' "$*" >> "${join(home, `${name}.log`)}"\nexit 99\n`, + ); + chmodSync(join(fakeBin, name), 0o755); + } + process.env.PATH = `${fakeBin}:/usr/bin:/bin`; + process.env.HOME = home; + process.env.PRIME_AGENT_KERNEL_VENV = join(home, "kernel-venv"); + process.env.PRIME_AGENT_INSTALL_UV = "1"; + delete process.env.PRIME_AGENT_KERNEL_PYTHON; + delete process.env.PRIME_AGENT_CODING_AGENT_DIR; + delete process.env.XDG_DATA_HOME; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + process.env = originalEnv; + rmSync(home, { recursive: true, force: true }); + }); + + it("downloads the pinned uv release, verifies it, and installs it without running a shell", async () => { + const uvLog = join(home, "uv.log"); + const archive = uvArchive(uvLog); + uvState.sha256 = { [uvAsset()]: createHash("sha256").update(archive).digest("hex") }; + serve(archive); + const progress: string[] = []; + + const python = await ensureKernelPython({ onProgress: (message) => progress.push(message) }); + + expect(python).toBe(join(home, "kernel-venv", "bin", "python")); + expect(requests).toEqual([helperToolDownloadUrl(HELPER_TOOL_RELEASES.uv, uvAsset())]); + expect(requests[0]).toMatch(/^https:\/\/github\.com\/astral-sh\/uv\/releases\/download\/\d+\.\d+\.\d+\//); + expect(readdirSync(binDir())).toEqual(["uv"]); + expect(readFileSync(uvLog, "utf8")).toContain("python install 3.11"); + expect(existsSync(join(home, "sh.log"))).toBe(false); + expect(existsSync(join(home, "curl.log"))).toBe(false); + expect(progress).toContain(`› installing uv ${HELPER_TOOL_RELEASES.uv.version} (one-time)…`); + }); + + it("rejects a uv archive whose digest does not match the pinned release", async () => { + const uvLog = join(home, "uv.log"); + uvState.sha256 = { [uvAsset()]: "0".repeat(64) }; + serve(uvArchive(uvLog)); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/SHA-256 mismatch/); + + expect(existsSync(uvLog)).toBe(false); + expect(existsSync(binDir()) ? readdirSync(binDir()) : []).toEqual([]); + expect(existsSync(join(home, "sh.log"))).toBe(false); + }); + + it("refuses to install uv when no digest is pinned for this platform", async () => { + serve(new Uint8Array([1])); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/No pinned SHA-256/); + + expect(requests).toEqual([]); + expect(existsSync(binDir()) ? readdirSync(binDir()) : []).toEqual([]); + }); + + it("leaves nothing behind when the uv download is interrupted", async () => { + const archive = uvArchive(join(home, "uv.log")); + uvState.sha256 = { [uvAsset()]: createHash("sha256").update(archive).digest("hex") }; + serve(() => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(archive.subarray(0, 32)); + controller.error(new Error("connection reset")); + }, + }); + return new Response(stream, { status: 200 }); + }); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/connection reset/); + + expect(existsSync(binDir()) ? readdirSync(binDir()) : []).toEqual([]); + expect(existsSync(join(home, "sh.log"))).toBe(false); + }); + + it("does not download anything when installation is refused", async () => { + process.env.PRIME_AGENT_INSTALL_UV = "0"; + serve(new Uint8Array([1])); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/PRIME_AGENT_INSTALL_UV=1/); + + expect(requests).toEqual([]); + expect(existsSync(join(home, "sh.log"))).toBe(false); + }); +}); diff --git a/packages/coding-agent/test/tools-manager.test.ts b/packages/coding-agent/test/tools-manager.test.ts index 0db6218bb1..916be89e36 100644 --- a/packages/coding-agent/test/tools-manager.test.ts +++ b/packages/coding-agent/test/tools-manager.test.ts @@ -1,4 +1,5 @@ -import { chmodSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -7,6 +8,7 @@ const toolState = vi.hoisted(() => ({ platform: "linux", architecture: "x64", extractZip: async (_source: string, _options: { dir: string }): Promise => {}, + rgSha256: {} as Record, })); vi.mock("../src/config.js", () => ({ @@ -14,6 +16,22 @@ vi.mock("../src/config.js", () => ({ getBinDir: () => toolState.toolsDir, })); +vi.mock("../src/utils/helper-tool-releases.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + HELPER_TOOL_RELEASES: { + ...actual.HELPER_TOOL_RELEASES, + rg: { + ...actual.HELPER_TOOL_RELEASES.rg, + get sha256() { + return toolState.rgSha256; + }, + }, + }, + }; +}); + vi.mock("os", () => ({ arch: () => toolState.architecture, platform: () => toolState.platform, @@ -23,6 +41,7 @@ vi.mock("extract-zip", () => ({ default: (source: string, options: { dir: string }) => toolState.extractZip(source, options), })); +import { HELPER_TOOL_RELEASES } from "../src/utils/helper-tool-releases.js"; import { ensureToolWithStatus, formatMissingRipgrepMessage, @@ -30,6 +49,13 @@ import { type ToolUnavailableResult, } from "../src/utils/tools-manager.js"; +const RG_WINDOWS_ASSET = `ripgrep-${HELPER_TOOL_RELEASES.rg.version}-x86_64-pc-windows-msvc.zip`; +const ASSET_BYTES = new Uint8Array([1]); + +function pinRgWindowsAsset(bytes: Uint8Array): void { + toolState.rgSha256 = { [RG_WINDOWS_ASSET]: createHash("sha256").update(bytes).digest("hex") }; +} + const originalPath = process.env.PATH; const originalOffline = process.env.PI_OFFLINE; const pathDir = join(toolState.toolsDir, "path"); @@ -55,6 +81,7 @@ describe("tools manager", () => { toolState.platform = "linux"; toolState.architecture = "x64"; toolState.extractZip = async () => {}; + toolState.rgSha256 = {}; }); afterEach(() => { @@ -105,6 +132,7 @@ describe("tools manager", () => { }); toolState.platform = "linux"; + toolState.rgSha256 = { [HELPER_TOOL_RELEASES.rg.assetName("linux", "x64") ?? ""]: "0".repeat(64) }; vi.stubGlobal( "fetch", vi.fn(async () => Promise.reject(new Error("network unavailable"))), @@ -119,15 +147,8 @@ describe("tools manager", () => { it("validates a downloaded binary before reporting it available", async () => { toolState.platform = "win32"; writeExecutable(join(toolState.toolsDir, "rg.exe"), 1); - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - new Response(JSON.stringify({ tag_name: "15.1.0" }), { - status: 200, - headers: { "Content-Type": "application/json" }, - }), - ) - .mockResolvedValueOnce(new Response(new Uint8Array([1]), { status: 200 })); + pinRgWindowsAsset(ASSET_BYTES); + const fetchMock = vi.fn().mockResolvedValueOnce(new Response(ASSET_BYTES, { status: 200 })); vi.stubGlobal("fetch", fetchMock); toolState.extractZip = async (_source, options) => { writeExecutable(join(options.dir, "rg.exe")); @@ -137,18 +158,17 @@ describe("tools manager", () => { status: "available", path: join(toolState.toolsDir, "rg.exe"), }); - expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][0]).toBe( + `https://github.com/BurntSushi/ripgrep/releases/download/${HELPER_TOOL_RELEASES.rg.tag}/${RG_WINDOWS_ASSET}`, + ); + expect(readdirSync(toolState.toolsDir).sort()).toEqual(["path", "rg.exe"]); }); it("removes a downloaded binary that fails its version check", async () => { toolState.platform = "win32"; - vi.stubGlobal( - "fetch", - vi - .fn() - .mockResolvedValueOnce(new Response(JSON.stringify({ tag_name: "15.1.0" }), { status: 200 })) - .mockResolvedValueOnce(new Response(new Uint8Array([1]), { status: 200 })), - ); + pinRgWindowsAsset(ASSET_BYTES); + vi.stubGlobal("fetch", vi.fn().mockResolvedValueOnce(new Response(ASSET_BYTES, { status: 200 }))); toolState.extractZip = async (_source, options) => { writeExecutable(join(options.dir, "rg.exe"), 1); }; @@ -158,6 +178,23 @@ describe("tools manager", () => { reason: "download_failed", }); expect(existsSync(join(toolState.toolsDir, "rg.exe"))).toBe(false); + expect(readdirSync(toolState.toolsDir)).toEqual(["path"]); + }); + + it("rejects a downloaded asset whose digest differs from the pinned release", async () => { + toolState.platform = "win32"; + pinRgWindowsAsset(new Uint8Array([2])); + vi.stubGlobal("fetch", vi.fn().mockResolvedValueOnce(new Response(ASSET_BYTES, { status: 200 }))); + const extractZip = vi.fn(async () => {}); + toolState.extractZip = extractZip; + + await expect(ensureToolWithStatus("rg")).resolves.toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("SHA-256 mismatch"), + }); + expect(extractZip).not.toHaveBeenCalled(); + expect(readdirSync(toolState.toolsDir)).toEqual(["path"]); }); it("formats actionable platform-specific ripgrep warnings", () => {