From 2b698944b8e43dd61dc81102ee0c82fbbc0099e2 Mon Sep 17 00:00:00 2001 From: Kevin Thomas Date: Wed, 9 Sep 2026 16:19:57 -0700 Subject: [PATCH] fix(coding-agent): verify pinned fd, ripgrep, and uv downloads Helper binaries were resolved from the latest GitHub release and installed without any digest check, partial downloads stayed in ~/.prime/agent/bin, and archive containment relied on the system tar. uv was installed by piping https://astral.sh/uv/install.sh into sh. Pin fd 10.5.0, ripgrep 15.2.0, and uv 0.12.9 with bundled SHA-256 digests, download into a staging directory, verify the digest before extraction, validate tar and zip member paths, run the version check on the staged binary, and only then move it into place. uv now comes from the pinned GitHub release archive through the same path; PRIME_AGENT_INSTALL_UV and the interactive confirmation keep their semantics. Linear: ENG-5343 --- .../eng-5343-helper-uv-verification.md | 1 + packages/coding-agent/docs/rlm-runtime.md | 2 + .../coding-agent/scripts/pin-helper-tools.ts | 59 ++++ .../coding-agent/src/core/kernel/bootstrap.ts | 58 +++- .../src/utils/helper-tool-install.ts | 240 ++++++++++++++ .../src/utils/helper-tool-releases.ts | 114 +++++++ .../coding-agent/src/utils/tools-manager.ts | 203 ++---------- .../coding-agent/test/archive-fixtures.ts | 111 +++++++ .../test/helper-tool-install.test.ts | 304 ++++++++++++++++++ .../test/kernel-bootstrap-uv.test.ts | 197 ++++++++++++ .../coding-agent/test/tools-manager.test.ts | 73 +++-- 11 files changed, 1146 insertions(+), 216 deletions(-) create mode 100644 packages/coding-agent/.changes/eng-5343-helper-uv-verification.md create mode 100644 packages/coding-agent/scripts/pin-helper-tools.ts create mode 100644 packages/coding-agent/src/utils/helper-tool-install.ts create mode 100644 packages/coding-agent/src/utils/helper-tool-releases.ts create mode 100644 packages/coding-agent/test/archive-fixtures.ts create mode 100644 packages/coding-agent/test/helper-tool-install.test.ts create mode 100644 packages/coding-agent/test/kernel-bootstrap-uv.test.ts diff --git a/packages/coding-agent/.changes/eng-5343-helper-uv-verification.md b/packages/coding-agent/.changes/eng-5343-helper-uv-verification.md new file mode 100644 index 0000000000..d1d6ee3735 --- /dev/null +++ b/packages/coding-agent/.changes/eng-5343-helper-uv-verification.md @@ -0,0 +1 @@ +- Changed helper provisioning to download pinned fd, ripgrep, and uv releases, verify their SHA-256 and archive contents before installing into `~/.prime/agent/bin`, and clean up partial downloads; uv is no longer installed by piping a remote script into `sh`. diff --git a/packages/coding-agent/docs/rlm-runtime.md b/packages/coding-agent/docs/rlm-runtime.md index 63625ccbc9..285fa72280 100644 --- a/packages/coding-agent/docs/rlm-runtime.md +++ b/packages/coding-agent/docs/rlm-runtime.md @@ -81,6 +81,8 @@ The kernel is created lazily on first Python REPL use. Python resolution is: The managed environment includes Python 3.11, `prime-agent-runtime`, `dill`, and the default Python packages. A bootstrap marker detects stale environments. +When no `uv` is found on `PATH`, in `~/.prime/agent/bin`, or in `~/.local/bin`, the bootstrap offers to install one. Set `PRIME_AGENT_INSTALL_UV=1` to accept without a prompt (the installer does this) or `PRIME_AGENT_INSTALL_UV=0` to refuse. The install downloads the pinned `uv` release archive from GitHub, verifies it against the SHA-256 bundled in `src/utils/helper-tool-releases.ts`, checks that the extracted binary runs, and only then places it in `~/.prime/agent/bin/uv`; no remote script is executed. The optional `fd` and `rg` search helpers are provisioned into the same directory the same way. Pinned versions are bumped with `scripts/pin-helper-tools.ts`. + Startup spawns `python -m rlm.repl` and exchanges newline-delimited JSON over stdio: the runtime announces itself with a single `ready` event, then requests and events flow one JSON object per line (see `prime-agent-runtime/src/rlm/repl.md`). The manager owns the child process and a bounded stderr tail. Shutdown sends a `shutdown` request, waits for the process to exit, and terminates it as a fallback. Persistent sessions may snapshot the kernel namespace into their session artifact directory for revival. diff --git a/packages/coding-agent/scripts/pin-helper-tools.ts b/packages/coding-agent/scripts/pin-helper-tools.ts new file mode 100644 index 0000000000..c2bf8e7345 --- /dev/null +++ b/packages/coding-agent/scripts/pin-helper-tools.ts @@ -0,0 +1,59 @@ +// Recompute the pinned SHA-256 table for a helper tool release. +// +// Usage (from packages/coding-agent): npx tsx scripts/pin-helper-tools.ts +// +// Downloads every supported asset of that release from GitHub, prints its SHA-256, and +// cross-checks the upstream `.sha256` file when the project publishes one. Paste +// the printed table into src/utils/helper-tool-releases.ts together with the new version. +import { createHash } from "node:crypto"; +import { HELPER_TOOL_RELEASES, type HelperToolId } from "../src/utils/helper-tool-releases.js"; + +const SUPPORTED_TARGETS: Array<[platform: string, architecture: string]> = [ + ["darwin", "arm64"], + ["darwin", "x64"], + ["linux", "arm64"], + ["linux", "x64"], + ["win32", "arm64"], + ["win32", "x64"], +]; + +async function fetchBytes(url: string): Promise { + const response = await fetch(url); + if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); + return new Uint8Array(await response.arrayBuffer()); +} + +async function main(): Promise { + const [tool, version] = process.argv.slice(2); + if (!tool || !version || !(tool in HELPER_TOOL_RELEASES)) { + console.error("usage: npx tsx scripts/pin-helper-tools.ts "); + process.exit(2); + } + const current = HELPER_TOOL_RELEASES[tool as HelperToolId]; + const tag = current.tag.startsWith("v") ? `v${version}` : version; + const lines: string[] = []; + let failed = false; + for (const [platform, architecture] of SUPPORTED_TARGETS) { + const assetName = current.assetName(platform, architecture)?.replaceAll(current.version, version); + if (!assetName) continue; + const url = `https://github.com/${current.repo}/releases/download/${tag}/${assetName}`; + const digest = createHash("sha256").update(await fetchBytes(url)).digest("hex"); + let note = "no upstream checksum published"; + try { + const published = new TextDecoder().decode(await fetchBytes(`${url}.sha256`)); + const match = published.match(/[0-9a-f]{64}/i)?.[0].toLowerCase(); + note = match === digest ? "matches upstream .sha256" : `MISMATCH with upstream .sha256 (${match ?? "unparseable"})`; + if (match !== digest) failed = true; + } catch { + // Only ripgrep and uv publish per-asset checksum files. + } + lines.push(`\t\t\t"${assetName}": "${digest}", // ${note}`); + } + console.log(`// ${current.repo} ${tag}\n${lines.join("\n")}`); + if (failed) { + console.error("Upstream checksum mismatch detected; do not pin this release."); + process.exit(1); + } +} + +await main(); diff --git a/packages/coding-agent/src/core/kernel/bootstrap.ts b/packages/coding-agent/src/core/kernel/bootstrap.ts index 5ab93cc64e..c6daa7dd85 100644 --- a/packages/coding-agent/src/core/kernel/bootstrap.ts +++ b/packages/coding-agent/src/core/kernel/bootstrap.ts @@ -7,9 +7,11 @@ import { stderr, stdin } from "node:process"; import { createInterface } from "node:readline/promises"; import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; -import { getPackageDir } from "../../config.js"; +import { getBinDir, getPackageDir } from "../../config.js"; import { isProcessAlive, spawnHidden } from "../../utils/child-process.js"; import { tryAcquireDirLock } from "../../utils/dir-lock.js"; +import { installPinnedHelperTool } from "../../utils/helper-tool-install.js"; +import { HELPER_TOOL_RELEASES } from "../../utils/helper-tool-releases.js"; import type { PythonSkillRuntimeInfo } from "../skills.js"; const BOOTSTRAP_SCHEMA = 9; @@ -71,7 +73,9 @@ export function buildBatchShimInvocation( }; } -const UV_INSTALL_COMMAND = "curl -LsSf https://astral.sh/uv/install.sh | sh"; +const UV_RELEASE = HELPER_TOOL_RELEASES.uv; +const UV_INSTALL_DOCS_URL = "https://docs.astral.sh/uv/getting-started/installation/"; +const UV_DOWNLOAD_TIMEOUT_MS = 120_000; const REQUIRED_HARNESS_METHODS = [ "create_memory", "update_memory", @@ -542,35 +546,57 @@ async function findExecutable(name: string): Promise { return null; } +function uvBinaryFileName(): string { + return process.platform === "win32" ? "uv.exe" : "uv"; +} + +async function uvWorks(uv: string): Promise { + try { + await run(uv, ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + async function ensureUv(options: EnsureKernelPythonOptions): Promise { const fromPath = await findExecutable("uv"); if (fromPath) return fromPath; - const localUv = path.join(os.homedir(), ".local", "bin", process.platform === "win32" ? "uv.exe" : "uv"); + const managedUv = path.join(getBinDir(), uvBinaryFileName()); + if (await isExecutable(managedUv)) return managedUv; + + const localUv = path.join(os.homedir(), ".local", "bin", uvBinaryFileName()); if (await isExecutable(localUv)) return localUv; const shouldInstallUv = process.env.PRIME_AGENT_INSTALL_UV === "1" || (!options.onProgress && (await confirmUvInstall())); if (!shouldInstallUv) { throw new Error( - `uv is required to set up the Python kernel. Install uv yourself: ${UV_INSTALL_COMMAND}, ` + - "or set PRIME_AGENT_INSTALL_UV=1 to let prime-agent run that installer.", + `uv is required to set up the Python kernel. Install uv yourself (${UV_INSTALL_DOCS_URL}), ` + + `or set PRIME_AGENT_INSTALL_UV=1 to let prime-agent download uv ${UV_RELEASE.version} from GitHub releases with SHA-256 verification.`, ); } - reportProgress(options, "› installing uv (one-time)…"); + reportProgress(options, `› installing uv ${UV_RELEASE.version} (one-time)…`); try { - await run("sh", ["-c", UV_INSTALL_COMMAND], { stdio: options.onProgress ? "ignore" : "inherit" }); + // Downloads the pinned release archive from GitHub and checks its bundled SHA-256 + // before extraction; nothing from the network is ever piped into a shell. + return await installPinnedHelperTool({ + tool: "uv", + platform: process.platform, + architecture: process.arch, + destDir: getBinDir(), + binaryFileName: uvBinaryFileName(), + verifyBinary: uvWorks, + timeoutMs: UV_DOWNLOAD_TIMEOUT_MS, + }); } catch (error) { throw new Error( - `couldn't install uv from astral.sh; install it yourself: ${UV_INSTALL_COMMAND}, then re-run prime-agent. ${errorMessage(error)}`, + `couldn't install uv ${UV_RELEASE.version} from ${UV_RELEASE.repo} releases: ${errorMessage(error)}. ` + + `Install uv yourself (${UV_INSTALL_DOCS_URL}), then re-run prime-agent.`, ); } - - if (await isExecutable(localUv)) return localUv; - const installedFromPath = await findExecutable("uv"); - if (installedFromPath) return installedFromPath; - throw new Error("uv install completed but binary not found at ~/.local/bin/uv"); } async function confirmUvInstall(): Promise { @@ -579,7 +605,11 @@ async function confirmUvInstall(): Promise { const rl = createInterface({ input: stdin, output: stderr }); try { - const answer = (await rl.question("Prime Agent needs uv to set up Python. Install uv from astral.sh now? [Y/n] ")) + const answer = ( + await rl.question( + `Prime Agent needs uv to set up Python. Download uv ${UV_RELEASE.version} from GitHub releases (SHA-256 verified) now? [Y/n] `, + ) + ) .trim() .toLowerCase(); return answer !== "n" && answer !== "no"; diff --git a/packages/coding-agent/src/utils/helper-tool-install.ts b/packages/coding-agent/src/utils/helper-tool-install.ts new file mode 100644 index 0000000000..aaa570d1cb --- /dev/null +++ b/packages/coding-agent/src/utils/helper-tool-install.ts @@ -0,0 +1,240 @@ +import { createHash } from "node:crypto"; +import { + chmodSync, + createWriteStream, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + renameSync, + rmSync, +} from "node:fs"; +import { basename, join, sep } from "node:path"; +import { Readable, Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream } from "node:stream/web"; +import extractZip from "extract-zip"; +import { spawnSyncHidden } from "./child-process.js"; +import { HELPER_TOOL_RELEASES, type HelperToolId, helperToolDownloadUrl } from "./helper-tool-releases.js"; + +const DEFAULT_DOWNLOAD_TIMEOUT_MS = 120_000; +const MAX_ASSET_BYTES = 256 * 1024 * 1024; +const TAR_LIST_MAX_BUFFER = 16 * 1024 * 1024; +const S_IFMT = 0o170000; +const S_IFLNK = 0o120000; + +export class UnsupportedHelperPlatformError extends Error {} +export class HelperIntegrityError extends Error {} +export class UnsafeArchiveMemberError extends Error {} + +export interface DownloadVerifiedOptions { + timeoutMs?: number; + maxBytes?: number; +} + +/** Download `url` to `dest`, hashing the stream; `dest` is removed unless the SHA-256 matches. */ +export async function downloadVerified( + url: string, + dest: string, + expectedSha256: string, + options: DownloadVerifiedOptions = {}, +): Promise { + if (!/^[0-9a-f]{64}$/.test(expectedSha256)) { + throw new HelperIntegrityError(`No valid pinned SHA-256 for ${basename(dest)}; refusing to download`); + } + if (!url.startsWith("https://")) { + throw new HelperIntegrityError(`Refusing to download ${basename(dest)} over a non-HTTPS URL`); + } + const maxBytes = options.maxBytes ?? MAX_ASSET_BYTES; + + const response = await fetch(url, { + signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`Failed to download ${basename(dest)}: HTTP ${response.status}`); + } + if (!response.body) { + throw new Error(`Failed to download ${basename(dest)}: empty response body`); + } + + const hash = createHash("sha256"); + let received = 0; + const hasher = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + received += chunk.length; + if (received > maxBytes) { + callback(new HelperIntegrityError(`Download of ${basename(dest)} exceeded ${maxBytes} bytes`)); + return; + } + hash.update(chunk); + callback(null, chunk); + }, + }); + + try { + await pipeline( + Readable.fromWeb(response.body as unknown as ReadableStream), + hasher, + createWriteStream(dest, { flags: "wx", mode: 0o600 }), + ); + const actual = hash.digest("hex"); + if (actual !== expectedSha256) { + throw new HelperIntegrityError( + `SHA-256 mismatch for ${basename(dest)}: expected ${expectedSha256}, got ${actual}`, + ); + } + } catch (error) { + rmSync(dest, { force: true }); + throw error; + } +} + +/** Reject archive members that could write outside the extraction directory. */ +export function assertSafeArchiveMemberPath(memberPath: string): void { + if (memberPath.length === 0 || memberPath.includes("\0")) { + throw new UnsafeArchiveMemberError(`Archive member has an empty or NUL-containing path`); + } + if (/^[\\/]/.test(memberPath) || /^[A-Za-z]:/.test(memberPath)) { + throw new UnsafeArchiveMemberError(`Archive member uses an absolute path: ${memberPath}`); + } + if (memberPath.split(/[\\/]+/).includes("..")) { + throw new UnsafeArchiveMemberError(`Archive member escapes the extraction directory: ${memberPath}`); + } +} + +function listTarMembers(archivePath: string): string[] { + const result = spawnSyncHidden("tar", ["tzPf", archivePath], { + stdio: "pipe", + encoding: "utf8", + maxBuffer: TAR_LIST_MAX_BUFFER, + }); + if (result.error || result.status !== 0) { + const detail = result.error?.message ?? result.stderr?.trim() ?? "unknown error"; + throw new Error(`Failed to list ${basename(archivePath)}: ${detail}`); + } + return result.stdout.split(/\r?\n/).filter((line) => line.length > 0); +} + +/** Extract a `.tar.gz` or `.zip` into `extractDir` after validating every member path. */ +export async function extractArchiveSafely(archivePath: string, extractDir: string): Promise { + mkdirSync(extractDir, { recursive: true }); + if (archivePath.endsWith(".tar.gz")) { + for (const member of listTarMembers(archivePath)) { + assertSafeArchiveMemberPath(member); + } + const result = spawnSyncHidden("tar", ["xzf", archivePath, "-C", extractDir], { stdio: "pipe" }); + if (result.error || result.status !== 0) { + const detail = result.error?.message ?? result.stderr?.toString().trim() ?? "unknown error"; + throw new Error(`Failed to extract ${basename(archivePath)}: ${detail}`); + } + return; + } + if (archivePath.endsWith(".zip")) { + await extractZip(archivePath, { + dir: extractDir, + onEntry: (entry) => { + assertSafeArchiveMemberPath(entry.fileName); + const mode = (entry.externalFileAttributes >> 16) & 0xffff; + if ((mode & S_IFMT) === S_IFLNK) { + throw new UnsafeArchiveMemberError(`Archive member is a symbolic link: ${entry.fileName}`); + } + }, + }); + return; + } + throw new Error(`Unsupported archive format: ${basename(archivePath)}`); +} + +function findFileRecursively(rootDir: string, fileName: string): string | null { + const stack: string[] = [rootDir]; + while (stack.length > 0) { + const currentDir = stack.pop(); + if (!currentDir) continue; + for (const entry of readdirSync(currentDir, { withFileTypes: true })) { + const fullPath = join(currentDir, entry.name); + if (entry.isFile() && entry.name === fileName) return fullPath; + if (entry.isDirectory()) stack.push(fullPath); + } + } + return null; +} + +function locateExtractedBinary(extractDir: string, archiveName: string, binaryFileName: string): string { + const nested = join(extractDir, archiveName.replace(/\.(tar\.gz|zip)$/, ""), binaryFileName); + const flat = join(extractDir, binaryFileName); + const candidate = + [nested, flat].find((path) => { + try { + return lstatSync(path).isFile(); + } catch { + return false; + } + }) ?? findFileRecursively(extractDir, binaryFileName); + if (!candidate) { + throw new Error(`Binary not found in archive: expected ${binaryFileName} in ${archiveName}`); + } + if (!lstatSync(candidate).isFile()) { + throw new UnsafeArchiveMemberError(`Extracted ${binaryFileName} is not a regular file`); + } + const root = realpathSync(extractDir); + if (!realpathSync(candidate).startsWith(root + sep)) { + throw new UnsafeArchiveMemberError(`Extracted ${binaryFileName} resolves outside the extraction directory`); + } + return candidate; +} + +export interface InstallPinnedHelperToolOptions { + tool: HelperToolId; + platform: string; + architecture: string; + /** Directory the verified binary is moved into; staging happens in a sibling temp dir. */ + destDir: string; + binaryFileName: string; + /** Must return true for the staged binary (for example a `--version` run) before it is installed. */ + verifyBinary: (binaryPath: string) => Promise | boolean; + timeoutMs?: number; +} + +/** + * Download the pinned release asset for `tool`, verify its SHA-256, extract it in a + * staging directory, check the binary works, then move it to `destDir`. Nothing is left + * in `destDir` when any step fails. + */ +export async function installPinnedHelperTool(options: InstallPinnedHelperToolOptions): Promise { + const release = HELPER_TOOL_RELEASES[options.tool]; + const assetName = release.assetName(options.platform, options.architecture); + if (!assetName) { + throw new UnsupportedHelperPlatformError(`Unsupported platform: ${options.platform}/${options.architecture}`); + } + const expectedSha256 = release.sha256[assetName]; + if (!expectedSha256) { + throw new HelperIntegrityError(`No pinned SHA-256 for ${assetName}; refusing to install ${options.tool}`); + } + + mkdirSync(options.destDir, { recursive: true }); + const stagingDir = mkdtempSync(join(options.destDir, `.staging-${options.binaryFileName}-`)); + try { + const archivePath = join(stagingDir, assetName); + await downloadVerified(helperToolDownloadUrl(release, assetName), archivePath, expectedSha256, { + timeoutMs: options.timeoutMs, + }); + + const extractDir = join(stagingDir, "extract"); + await extractArchiveSafely(archivePath, extractDir); + const stagedBinary = locateExtractedBinary(extractDir, assetName, options.binaryFileName); + if (options.platform !== "win32") { + chmodSync(stagedBinary, 0o755); + } + if (!(await options.verifyBinary(stagedBinary))) { + throw new Error(`Downloaded ${options.binaryFileName} ${release.version} failed its version check`); + } + + const installedPath = join(options.destDir, options.binaryFileName); + rmSync(installedPath, { force: true }); + renameSync(stagedBinary, installedPath); + return installedPath; + } finally { + rmSync(stagingDir, { recursive: true, force: true }); + } +} diff --git a/packages/coding-agent/src/utils/helper-tool-releases.ts b/packages/coding-agent/src/utils/helper-tool-releases.ts new file mode 100644 index 0000000000..586596edf8 --- /dev/null +++ b/packages/coding-agent/src/utils/helper-tool-releases.ts @@ -0,0 +1,114 @@ +// Pinned upstream releases for the helper binaries Prime Agent downloads on demand. +// +// Every asset is verified against the SHA-256 recorded here before it is extracted, +// so a compromised release page or CDN cannot substitute a binary. Bump a version with +// `npx tsx scripts/pin-helper-tools.ts ` from packages/coding-agent, +// which downloads the pinned assets, cross-checks upstream `.sha256` files where the +// project publishes them (ripgrep, uv; fd publishes none), and prints the new table. + +export type HelperToolId = "fd" | "rg" | "uv"; + +export interface HelperToolRelease { + /** GitHub repository in `owner/name` form. */ + repo: string; + version: string; + /** Git tag of the release (fd prefixes versions with `v`). */ + tag: string; + /** Release asset for a platform/architecture, or null when unsupported. */ + assetName: (platform: string, architecture: string) => string | null; + /** Lowercase hex SHA-256 of each supported asset. */ + sha256: Readonly>; +} + +function rustTarget(platform: string, architecture: string): string | null { + const cpu = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; + if (!cpu) return null; + switch (platform) { + case "darwin": + return `${cpu}-apple-darwin`; + case "linux": + return `${cpu}-unknown-linux-gnu`; + case "win32": + return `${cpu}-pc-windows-msvc`; + default: + return null; + } +} + +function archiveExtension(platform: string): string { + return platform === "win32" ? "zip" : "tar.gz"; +} + +const FD_VERSION = "10.5.0"; +const RIPGREP_VERSION = "15.2.0"; +const UV_VERSION = "0.12.9"; + +export const HELPER_TOOL_RELEASES: Readonly> = { + fd: { + repo: "sharkdp/fd", + version: FD_VERSION, + tag: `v${FD_VERSION}`, + assetName: (platform, architecture) => { + const target = rustTarget(platform, architecture); + return target ? `fd-v${FD_VERSION}-${target}.${archiveExtension(platform)}` : null; + }, + sha256: { + "fd-v10.5.0-aarch64-apple-darwin.tar.gz": "b67e1836c468e42e411984b56e52fa7abec08c2bd22c867398e7cc134aac5e12", + "fd-v10.5.0-x86_64-apple-darwin.tar.gz": "7e31028c62c6955877735d0406807aa484c2a5e6f86235a59e26c29c301da590", + "fd-v10.5.0-aarch64-unknown-linux-gnu.tar.gz": + "c0ee43802e3313a317c5af2f4eabd6ba13eeedd595af9775f05e18a13ac4f52c", + "fd-v10.5.0-x86_64-unknown-linux-gnu.tar.gz": + "a1259cd129636efbc3fef123525c1b49e88fe5088c012630983c310e52fdfa95", + "fd-v10.5.0-aarch64-pc-windows-msvc.zip": "a2bcddcfd259b05357a77bbc6cd671fdb30f63fd266a0e748305890a8c5ceaa6", + "fd-v10.5.0-x86_64-pc-windows-msvc.zip": "a227701b8551c35a9931d9f6da75503cf86d88e182d71fb849a70864c5d57cd7", + }, + }, + rg: { + repo: "BurntSushi/ripgrep", + version: RIPGREP_VERSION, + tag: RIPGREP_VERSION, + assetName: (platform, architecture) => { + // ripgrep ships a static musl build for x86_64 Linux and only a glibc build for aarch64. + const target = + platform === "linux" && architecture === "x64" + ? "x86_64-unknown-linux-musl" + : rustTarget(platform, architecture); + return target ? `ripgrep-${RIPGREP_VERSION}-${target}.${archiveExtension(platform)}` : null; + }, + sha256: { + "ripgrep-15.2.0-aarch64-apple-darwin.tar.gz": + "3750b2e93f37e0c692657da574d7019a101c0084da05a790c83fd335bad973e4", + "ripgrep-15.2.0-x86_64-apple-darwin.tar.gz": + "af7825fcc69a2afc7a7aea55fc9af90e26421d8f20fe59df32e233c0b8a231c1", + "ripgrep-15.2.0-aarch64-unknown-linux-gnu.tar.gz": + "a740b91c82eaf9914cfedd353572f2791cbe0162c84101ee0951058f4dcbc90d", + "ripgrep-15.2.0-x86_64-unknown-linux-musl.tar.gz": + "33e15bcf1624b25cdd2a55813a47a2f95dbe126268203e76aa6a585d1e7b149c", + "ripgrep-15.2.0-aarch64-pc-windows-msvc.zip": + "e4abca10c3a64ebea742667dd7009449d49403db5460dd6873e389fa2945360f", + "ripgrep-15.2.0-x86_64-pc-windows-msvc.zip": + "71b2fef860abe467217a538ff31de02f5258807c0129f771846f87bd029aafc5", + }, + }, + uv: { + repo: "astral-sh/uv", + version: UV_VERSION, + tag: UV_VERSION, + assetName: (platform, architecture) => { + const target = rustTarget(platform, architecture); + return target ? `uv-${target}.${archiveExtension(platform)}` : null; + }, + sha256: { + "uv-aarch64-apple-darwin.tar.gz": "301f72afaf54060f92da7016cb0115bd077f43a9c8e39c1d8170a0bac80fd398", + "uv-x86_64-apple-darwin.tar.gz": "e1ca175824f1056589ce9908f7631879ebc3c36535b5e63dc06510beb370b4c1", + "uv-aarch64-unknown-linux-gnu.tar.gz": "c36fe17937ff6bd16dc42fc13854b5465999fcab2efe0af559381e945e3c6001", + "uv-x86_64-unknown-linux-gnu.tar.gz": "ec7a99cd05e0cd7f80243f135ce1361c76835cb0ee60055d14d20eba8eba1460", + "uv-aarch64-pc-windows-msvc.zip": "d3360363a3cb671f2c854f4ef48cf4a57fe8664f8ec6a248076d68b797a8acc0", + "uv-x86_64-pc-windows-msvc.zip": "ddbfcee1ac615a0499f6aa97b5ec8ebdf3ee4a7714a48055ec2ba0030e3cf810", + }, + }, +}; + +export function helperToolDownloadUrl(release: HelperToolRelease, assetName: string): string { + return `https://github.com/${release.repo}/releases/download/${release.tag}/${assetName}`; +} diff --git a/packages/coding-agent/src/utils/tools-manager.ts b/packages/coding-agent/src/utils/tools-manager.ts index 57aa77e373..67f17ac34e 100644 --- a/packages/coding-agent/src/utils/tools-manager.ts +++ b/packages/coding-agent/src/utils/tools-manager.ts @@ -1,15 +1,13 @@ import chalk from "chalk"; -import extractZip from "extract-zip"; -import { chmodSync, createWriteStream, existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "fs"; +import { existsSync } from "fs"; import { arch, platform } from "os"; import { join } from "path"; -import { Readable } from "stream"; -import { pipeline } from "stream/promises"; -import { APP_NAME, getBinDir } from "../config.js"; +import { getBinDir } from "../config.js"; import { spawnSyncHidden } from "./child-process.js"; +import { installPinnedHelperTool, UnsupportedHelperPlatformError } from "./helper-tool-install.js"; +import { HELPER_TOOL_RELEASES } from "./helper-tool-releases.js"; const TOOLS_DIR = getBinDir(); -const NETWORK_TIMEOUT_MS = 10_000; const DOWNLOAD_TIMEOUT_MS = 120_000; const COMMAND_TIMEOUT_MS = 5_000; const RIPGREP_INSTALL_URL = "https://github.com/BurntSushi/ripgrep#installation"; @@ -41,59 +39,19 @@ function isOfflineModeEnabled(): boolean { interface ToolConfig { name: string; - repo: string; // GitHub repo (e.g., "sharkdp/fd") binaryName: string; // Name of the binary inside the archive systemBinaryNames?: string[]; // Alternative system command names to try before downloading - tagPrefix: string; // Prefix for tags (e.g., "v" for v1.0.0, "" for 1.0.0) - getAssetName: (version: string, plat: string, architecture: string) => string | null; } -const TOOLS: Record = { +const TOOLS: Record = { fd: { name: "fd", - repo: "sharkdp/fd", binaryName: "fd", systemBinaryNames: ["fd", "fdfind"], - tagPrefix: "v", - getAssetName: (version, plat, architecture) => { - if (plat === "darwin") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `fd-v${version}-${archStr}-apple-darwin.tar.gz`; - } else if (plat === "linux") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `fd-v${version}-${archStr}-unknown-linux-gnu.tar.gz`; - } else if (plat === "win32") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `fd-v${version}-${archStr}-pc-windows-msvc.zip`; - } - return null; - }, }, rg: { name: "ripgrep", - repo: "BurntSushi/ripgrep", binaryName: "rg", - tagPrefix: "", - getAssetName: (version, plat, architecture) => { - if (plat === "darwin") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `ripgrep-${version}-${archStr}-apple-darwin.tar.gz`; - } else if (plat === "linux") { - if (architecture === "arm64") { - return `ripgrep-${version}-aarch64-unknown-linux-gnu.tar.gz`; - } - return architecture === "x64" ? `ripgrep-${version}-x86_64-unknown-linux-musl.tar.gz` : null; - } else if (plat === "win32") { - const archStr = architecture === "arm64" ? "aarch64" : architecture === "x64" ? "x86_64" : null; - if (!archStr) return null; - return `ripgrep-${version}-${archStr}-pc-windows-msvc.zip`; - } - return null; - }, }, }; @@ -129,145 +87,22 @@ export function getToolPath(tool: ManagedTool): string | null { return null; } -// Fetch latest release version from GitHub -async function getLatestVersion(repo: string): Promise { - const response = await fetch(`https://api.github.com/repos/${repo}/releases/latest`, { - headers: { "User-Agent": `${APP_NAME}-coding-agent` }, - signal: AbortSignal.timeout(NETWORK_TIMEOUT_MS), - }); - - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status}`); - } - - const data = (await response.json()) as { tag_name: string }; - return data.tag_name.replace(/^v/, ""); -} - -// Download a file from URL -async function downloadFile(url: string, dest: string): Promise { - const response = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), - }); - - if (!response.ok) { - throw new Error(`Failed to download: ${response.status}`); - } - - if (!response.body) { - throw new Error("No response body"); - } - - const fileStream = createWriteStream(dest); - await pipeline(Readable.fromWeb(response.body as any), fileStream); -} - -function findBinaryRecursively(rootDir: string, binaryFileName: string): string | null { - const stack: string[] = [rootDir]; - - while (stack.length > 0) { - const currentDir = stack.pop(); - if (!currentDir) continue; - - const entries = readdirSync(currentDir, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = join(currentDir, entry.name); - if (entry.isFile() && entry.name === binaryFileName) { - return fullPath; - } - if (entry.isDirectory()) { - stack.push(fullPath); - } - } - } - - return null; -} - -// Download and install a tool -class UnsupportedToolPlatformError extends Error {} - +// Download the pinned release, verify its digest, and install it into TOOLS_DIR. async function downloadTool(tool: ManagedTool): Promise { const config = TOOLS[tool]; if (!config) throw new Error(`Unknown tool: ${tool}`); const plat = platform(); - const architecture = arch(); - - if (!config.getAssetName("VERSION", plat, architecture)) { - throw new UnsupportedToolPlatformError(`Unsupported platform: ${plat}/${architecture}`); - } - - // Get latest version and the matching platform asset. - const version = await getLatestVersion(config.repo); - const assetName = config.getAssetName(version, plat, architecture); - if (!assetName) throw new UnsupportedToolPlatformError(`Unsupported platform: ${plat}/${architecture}`); - - // Create tools directory - mkdirSync(TOOLS_DIR, { recursive: true }); - - const downloadUrl = `https://github.com/${config.repo}/releases/download/${config.tagPrefix}${version}/${assetName}`; - const archivePath = join(TOOLS_DIR, assetName); - const binaryExt = plat === "win32" ? ".exe" : ""; - const binaryPath = join(TOOLS_DIR, config.binaryName + binaryExt); - - // Download - await downloadFile(downloadUrl, archivePath); - - // Extract into a unique temp directory. fd and rg downloads can run concurrently - // during startup, so sharing a fixed directory causes races. - const extractDir = join( - TOOLS_DIR, - `extract_tmp_${config.binaryName}_${process.pid}_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`, - ); - mkdirSync(extractDir, { recursive: true }); - - try { - if (assetName.endsWith(".tar.gz")) { - const extractResult = spawnSyncHidden("tar", ["xzf", archivePath, "-C", extractDir], { stdio: "pipe" }); - if (extractResult.error || extractResult.status !== 0) { - const errMsg = extractResult.error?.message ?? extractResult.stderr?.toString().trim() ?? "unknown error"; - throw new Error(`Failed to extract ${assetName}: ${errMsg}`); - } - } else if (assetName.endsWith(".zip")) { - await extractZip(archivePath, { dir: extractDir }); - } else { - throw new Error(`Unsupported archive format: ${assetName}`); - } - - // Find the binary in extracted files. Some archives contain files directly - // at root, others nest under a versioned subdirectory. - const binaryFileName = config.binaryName + binaryExt; - const extractedDir = join(extractDir, assetName.replace(/\.(tar\.gz|zip)$/, "")); - const extractedBinaryCandidates = [join(extractedDir, binaryFileName), join(extractDir, binaryFileName)]; - let extractedBinary = extractedBinaryCandidates.find((candidate) => existsSync(candidate)); - - if (!extractedBinary) { - extractedBinary = findBinaryRecursively(extractDir, binaryFileName) ?? undefined; - } - - if (extractedBinary) { - rmSync(binaryPath, { force: true }); - renameSync(extractedBinary, binaryPath); - } else { - throw new Error(`Binary not found in archive: expected ${binaryFileName} under ${extractDir}`); - } - - // Make executable (Unix only) - if (plat !== "win32") { - chmodSync(binaryPath, 0o755); - } - if (!commandWorks(binaryPath)) { - rmSync(binaryPath, { force: true }); - throw new Error(`Installed ${config.name} binary failed its version check`); - } - } finally { - // Cleanup - rmSync(archivePath, { force: true }); - rmSync(extractDir, { recursive: true, force: true }); - } - - return binaryPath; + const binaryFileName = config.binaryName + (plat === "win32" ? ".exe" : ""); + return installPinnedHelperTool({ + tool, + platform: plat, + architecture: arch(), + destDir: TOOLS_DIR, + binaryFileName, + verifyBinary: commandWorks, + timeoutMs: DOWNLOAD_TIMEOUT_MS, + }); } // Termux package names for tools @@ -352,9 +187,9 @@ export async function ensureToolWithStatus(tool: ManagedTool, silent: boolean = }; } - // Tool not found - download it + // Tool not found - download the pinned release if (!silent) { - console.log(chalk.dim(`${config.name} not found. Downloading...`)); + console.log(chalk.dim(`${config.name} not found. Downloading ${HELPER_TOOL_RELEASES[tool].version}...`)); } try { @@ -369,7 +204,7 @@ export async function ensureToolWithStatus(tool: ManagedTool, silent: boolean = } return { status: "unavailable", - reason: e instanceof UnsupportedToolPlatformError ? "unsupported_platform" : "download_failed", + reason: e instanceof UnsupportedHelperPlatformError ? "unsupported_platform" : "download_failed", platform: platformName, architecture, detail: e instanceof Error ? e.message : String(e), diff --git a/packages/coding-agent/test/archive-fixtures.ts b/packages/coding-agent/test/archive-fixtures.ts new file mode 100644 index 0000000000..b4bc5283d5 --- /dev/null +++ b/packages/coding-agent/test/archive-fixtures.ts @@ -0,0 +1,111 @@ +// Minimal tar.gz and zip writers for helper-install tests. Real archivers strip or +// refuse `..` and absolute member names, so hostile fixtures have to be hand-built. +import { crc32, gzipSync } from "node:zlib"; + +export interface ArchiveEntry { + name: string; + content?: string; + mode?: number; + /** Symbolic link target; the entry becomes a symlink when set. */ + linkTarget?: string; +} + +function octal(value: number, width: number): Buffer { + return Buffer.from(`${value.toString(8).padStart(width - 1, "0")}\0`, "latin1"); +} + +function tarHeader(entry: ArchiveEntry, size: number): Buffer { + const header = Buffer.alloc(512); + header.write(entry.name, 0, 100, "utf8"); + octal(entry.mode ?? (entry.linkTarget ? 0o777 : 0o644), 8).copy(header, 100); + octal(0, 8).copy(header, 108); + octal(0, 8).copy(header, 116); + octal(size, 12).copy(header, 124); + octal(0, 12).copy(header, 136); + header.fill(" ", 148, 156); + header.write(entry.linkTarget ? "2" : "0", 156, 1, "latin1"); + if (entry.linkTarget) header.write(entry.linkTarget, 157, 100, "utf8"); + header.write("ustar\0", 257, 6, "latin1"); + header.write("00", 263, 2, "latin1"); + let sum = 0; + for (const byte of header) sum += byte; + Buffer.from(`${sum.toString(8).padStart(6, "0")}\0 `, "latin1").copy(header, 148); + return header; +} + +export function makeTarGz(entries: ArchiveEntry[]): Buffer { + const blocks: Buffer[] = []; + for (const entry of entries) { + const data = Buffer.from(entry.linkTarget ? "" : (entry.content ?? ""), "utf8"); + blocks.push(tarHeader(entry, data.length)); + blocks.push(data); + const padding = (512 - (data.length % 512)) % 512; + if (padding) blocks.push(Buffer.alloc(padding)); + } + blocks.push(Buffer.alloc(1024)); + return gzipSync(Buffer.concat(blocks)); +} + +function u16(value: number): Buffer { + const buffer = Buffer.alloc(2); + buffer.writeUInt16LE(value); + return buffer; +} + +function u32(value: number): Buffer { + const buffer = Buffer.alloc(4); + buffer.writeUInt32LE(value >>> 0); + return buffer; +} + +export function makeZip(entries: ArchiveEntry[]): Buffer { + const local: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const data = Buffer.from(entry.linkTarget ?? entry.content ?? "", "utf8"); + const crc = crc32(data); + const mode = entry.linkTarget ? 0o120777 : (entry.mode ?? 0o644); + const fixed = Buffer.concat([ + u16(20), + u16(0), + u16(0), + u16(0), + u16(0), + u32(crc), + u32(data.length), + u32(data.length), + ]); + const localHeader = Buffer.concat([u32(0x04034b50), fixed, u16(name.length), u16(0), name]); + local.push(localHeader, data); + central.push( + Buffer.concat([ + u32(0x02014b50), + u16((3 << 8) | 20), + fixed, + u16(name.length), + u16(0), + u16(0), + u16(0), + u16(0), + u32(mode << 16), + u32(offset), + name, + ]), + ); + offset += localHeader.length + data.length; + } + const centralDirectory = Buffer.concat(central); + const end = Buffer.concat([ + u32(0x06054b50), + u16(0), + u16(0), + u16(entries.length), + u16(entries.length), + u32(centralDirectory.length), + u32(offset), + u16(0), + ]); + return Buffer.concat([...local, centralDirectory, end]); +} diff --git a/packages/coding-agent/test/helper-tool-install.test.ts b/packages/coding-agent/test/helper-tool-install.test.ts new file mode 100644 index 0000000000..f38607b3a1 --- /dev/null +++ b/packages/coding-agent/test/helper-tool-install.test.ts @@ -0,0 +1,304 @@ +// ENG-5343: fd/rg helper provisioning must verify pinned digests, contain archive +// members, and leave nothing behind when a download fails. +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { makeTarGz, makeZip } from "./archive-fixtures.js"; + +// getBinDir() is read once at tools-manager import time, so the path must exist before imports run. +const toolState = vi.hoisted(() => ({ + work: `/tmp/eng5343-helper-${process.pid}`, + toolsDir: `/tmp/eng5343-helper-${process.pid}/bin`, + platform: process.platform as string, + architecture: process.arch as string, + sha256: { fd: {} as Record, rg: {} as Record }, +})); + +vi.mock("../src/config.js", () => ({ + APP_NAME: "prime-agent", + getBinDir: () => toolState.toolsDir, +})); + +vi.mock("os", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + default: { ...actual, arch: () => toolState.architecture, platform: () => toolState.platform }, + arch: () => toolState.architecture, + platform: () => toolState.platform, + }; +}); + +vi.mock("../src/utils/helper-tool-releases.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + HELPER_TOOL_RELEASES: { + ...actual.HELPER_TOOL_RELEASES, + fd: { + ...actual.HELPER_TOOL_RELEASES.fd, + get sha256() { + return toolState.sha256.fd; + }, + }, + rg: { + ...actual.HELPER_TOOL_RELEASES.rg, + get sha256() { + return toolState.sha256.rg; + }, + }, + }, + }; +}); + +import { assertSafeArchiveMemberPath } from "../src/utils/helper-tool-install.js"; +import { HELPER_TOOL_RELEASES, helperToolDownloadUrl } from "../src/utils/helper-tool-releases.js"; +import { ensureToolWithStatus } from "../src/utils/tools-manager.js"; + +const FD_SCRIPT = "#!/bin/sh\necho 'fd 10.5.0 MARKER-5343'\nexit 0\n"; +const originalPath = process.env.PATH; +const work = toolState.work; +let requests: string[] = []; + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function fdAsset(): string { + const asset = HELPER_TOOL_RELEASES.fd.assetName(toolState.platform, toolState.architecture); + if (!asset) throw new Error("unsupported test platform"); + return asset; +} + +function fdArchiveDir(): string { + return fdAsset().replace(/\.(tar\.gz|zip)$/, ""); +} + +function pinFd(bytes: Uint8Array): void { + toolState.sha256.fd = { [fdAsset()]: sha256(bytes) }; +} + +function serve(body: Uint8Array | (() => Response)): void { + vi.stubGlobal("fetch", async (input: unknown) => { + requests.push(String(input)); + return typeof body === "function" ? body() : new Response(body, { status: 200 }); + }); +} + +function leftovers(): string[] { + return existsSync(toolState.toolsDir) ? readdirSync(toolState.toolsDir) : []; +} + +describe("ENG-5343 helper tool provisioning", () => { + beforeEach(() => { + rmSync(work, { recursive: true, force: true }); + toolState.platform = process.platform; + toolState.architecture = process.arch; + toolState.sha256 = { fd: {}, rg: {} }; + mkdirSync(toolState.toolsDir, { recursive: true }); + process.env.PATH = "/usr/bin:/bin"; + requests = []; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + if (originalPath === undefined) delete process.env.PATH; + else process.env.PATH = originalPath; + rmSync(work, { recursive: true, force: true }); + }); + + it.skipIf(process.platform === "win32")("installs a pinned asset whose digest matches", async () => { + const archive = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + pinFd(archive); + serve(archive); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toEqual({ status: "available", path: join(toolState.toolsDir, "fd") }); + expect(readFileSync(join(toolState.toolsDir, "fd"), "utf8")).toBe(FD_SCRIPT); + expect(requests).toEqual([helperToolDownloadUrl(HELPER_TOOL_RELEASES.fd, fdAsset())]); + expect(leftovers()).toEqual(["fd"]); + }); + + it.skipIf(process.platform === "win32")("rejects a tampered archive before extracting it", async () => { + const genuine = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + const tampered = makeTarGz([ + { name: `${fdArchiveDir()}/fd`, content: "#!/bin/sh\necho tampered-fd MARKER-5343\n", mode: 0o755 }, + ]); + pinFd(genuine); + serve(tampered); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("SHA-256 mismatch"), + }); + expect(requests).toHaveLength(1); + expect(leftovers()).toEqual([]); + }); + + it("refuses to download when the release has no pinned digest for this platform", async () => { + toolState.sha256.fd = {}; + serve(new Uint8Array([1])); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("No pinned SHA-256"), + }); + expect(requests).toEqual([]); + expect(leftovers()).toEqual([]); + }); + + it.skipIf(process.platform === "win32")("rejects tar members that escape the extraction directory", async () => { + for (const hostile of [ + "../escaped-5343.txt", + `${fdArchiveDir()}/../../escaped-5343.txt`, + "/tmp/absolute-5343.txt", + ]) { + const archive = makeTarGz([ + { name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }, + { name: hostile, content: "escaped\n" }, + ]); + pinFd(archive); + serve(archive); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringMatching(/escapes the extraction directory|absolute path/), + }); + expect(existsSync(join(work, "escaped-5343.txt"))).toBe(false); + expect(existsSync("/tmp/absolute-5343.txt")).toBe(false); + expect(leftovers()).toEqual([]); + } + }); + + it.skipIf(process.platform === "win32")("rejects a binary delivered as a symlink", async () => { + const archive = makeTarGz([{ name: `${fdArchiveDir()}/fd`, linkTarget: "/bin/sh" }]); + pinFd(archive); + serve(archive); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ status: "unavailable", reason: "download_failed" }); + expect(leftovers()).toEqual([]); + }); + + it.skipIf(process.platform === "win32")("leaves nothing behind when the download is interrupted", async () => { + const archive = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + pinFd(archive); + serve(() => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(archive.subarray(0, 16)); + controller.error(new Error("connection reset")); + }, + }); + return new Response(stream, { status: 200 }); + }); + + const result = await ensureToolWithStatus("fd"); + + expect(result).toMatchObject({ status: "unavailable", reason: "download_failed" }); + expect(leftovers()).toEqual([]); + }); + + it.skipIf(process.platform === "win32")("keeps a working binary when a re-download fails verification", async () => { + const genuine = makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: FD_SCRIPT, mode: 0o755 }]); + pinFd(genuine); + serve(genuine); + await expect(ensureToolWithStatus("fd")).resolves.toMatchObject({ status: "available" }); + + // Break the installed binary so the next call re-downloads, then serve a tampered asset. + rmSync(join(toolState.toolsDir, "fd")); + serve(makeTarGz([{ name: `${fdArchiveDir()}/fd`, content: "#!/bin/sh\nexit 0\n", mode: 0o755 }])); + + await expect(ensureToolWithStatus("fd")).resolves.toMatchObject({ status: "unavailable" }); + expect(leftovers()).toEqual([]); + }); + + describe("zip assets", () => { + const rgAsset = () => { + const asset = HELPER_TOOL_RELEASES.rg.assetName("win32", "x64"); + if (!asset) throw new Error("unsupported test platform"); + return asset; + }; + const RG_SCRIPT = "#!/bin/sh\nexit 0\n"; + + beforeEach(() => { + toolState.platform = "win32"; + toolState.architecture = "x64"; + }); + + function pinRg(bytes: Uint8Array): void { + toolState.sha256.rg = { [rgAsset()]: sha256(bytes) }; + } + + it.skipIf(process.platform === "win32")("installs a verified zip asset", async () => { + const archive = makeZip([ + { name: `${rgAsset().replace(/\.zip$/, "")}/rg.exe`, content: RG_SCRIPT, mode: 0o755 }, + ]); + pinRg(archive); + serve(archive); + + await expect(ensureToolWithStatus("rg")).resolves.toEqual({ + status: "available", + path: join(toolState.toolsDir, "rg.exe"), + }); + expect(leftovers()).toEqual(["rg.exe"]); + }); + + it("rejects zip members with traversal or absolute paths", async () => { + for (const hostile of ["../escaped-5343.txt", "/tmp/absolute-5343.txt", "C:\\escaped-5343.txt"]) { + const archive = makeZip([ + { name: "rg.exe", content: RG_SCRIPT, mode: 0o755 }, + { name: hostile, content: "escaped\n" }, + ]); + pinRg(archive); + serve(archive); + + await expect(ensureToolWithStatus("rg")).resolves.toMatchObject({ + status: "unavailable", + reason: "download_failed", + }); + expect(existsSync(join(work, "escaped-5343.txt"))).toBe(false); + expect(existsSync("/tmp/absolute-5343.txt")).toBe(false); + expect(leftovers()).toEqual([]); + } + }); + + it("rejects zip members that are symbolic links", async () => { + const archive = makeZip([{ name: "rg.exe", linkTarget: "/bin/sh" }]); + pinRg(archive); + serve(archive); + + await expect(ensureToolWithStatus("rg")).resolves.toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("symbolic link"), + }); + expect(leftovers()).toEqual([]); + }); + }); + + it("validates archive member paths", () => { + expect(() => assertSafeArchiveMemberPath("fd-v10.5.0/fd")).not.toThrow(); + expect(() => assertSafeArchiveMemberPath("dir/../ok/../file")).toThrow(/escapes/); + expect(() => assertSafeArchiveMemberPath("..")).toThrow(/escapes/); + expect(() => assertSafeArchiveMemberPath("..\\evil")).toThrow(/escapes/); + expect(() => assertSafeArchiveMemberPath("/etc/passwd")).toThrow(/absolute/); + expect(() => assertSafeArchiveMemberPath("\\\\server\\share")).toThrow(/absolute/); + expect(() => assertSafeArchiveMemberPath("C:/Windows/evil")).toThrow(/absolute/); + expect(() => assertSafeArchiveMemberPath("")).toThrow(/empty/); + expect(() => assertSafeArchiveMemberPath("a\0b")).toThrow(/NUL/); + expect(() => assertSafeArchiveMemberPath("..hidden/file")).not.toThrow(); + }); +}); diff --git a/packages/coding-agent/test/kernel-bootstrap-uv.test.ts b/packages/coding-agent/test/kernel-bootstrap-uv.test.ts new file mode 100644 index 0000000000..9e2bd33a94 --- /dev/null +++ b/packages/coding-agent/test/kernel-bootstrap-uv.test.ts @@ -0,0 +1,197 @@ +// ENG-5343: the kernel bootstrap must install uv from the pinned, digest-verified +// release archive and never pipe a remote script into a shell. +import { createHash } from "node:crypto"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { makeTarGz } from "./archive-fixtures.js"; + +const uvState = vi.hoisted(() => ({ sha256: {} as Record })); + +vi.mock("../src/utils/helper-tool-releases.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + HELPER_TOOL_RELEASES: { + ...actual.HELPER_TOOL_RELEASES, + uv: { + ...actual.HELPER_TOOL_RELEASES.uv, + get sha256() { + return uvState.sha256; + }, + }, + }, + }; +}); + +import { DEFAULT_RLM_EXTRA_IMPORT_NAMES, ensureKernelPython } from "../src/core/kernel/bootstrap.js"; +import { HELPER_TOOL_RELEASES, helperToolDownloadUrl } from "../src/utils/helper-tool-releases.js"; + +let home = ""; +let originalEnv: NodeJS.ProcessEnv; +let requests: string[] = []; + +function uvAsset(): string { + const asset = HELPER_TOOL_RELEASES.uv.assetName(process.platform, process.arch); + if (!asset) throw new Error("unsupported test platform"); + return asset; +} + +function fakeUvScript(logPath: string): string { + return [ + "#!/bin/sh", + `printf '%s\\n' "$*" >> "${logPath}"`, + 'if [ "$1" = "--version" ]; then echo "uv 0.0.0-fake"; exit 0; fi', + 'if [ "$1" = "python" ]; then exit 0; fi', + 'if [ "$1" = "venv" ]; then', + ' venv="$2"; mkdir -p "$venv/bin"', + ` cat > "$venv/bin/python" <<'PY'`, + "#!/bin/sh", + 'if [ "$1" = "-c" ]; then', + ' case "$2" in', + ' "import rlm") exit 0 ;;', + ...DEFAULT_RLM_EXTRA_IMPORT_NAMES.map((name) => ` "import ${name}") exit 0 ;;`), + ' *"_harness_methods"*) exit 0 ;;', + " *) exit 1 ;;", + " esac", + "fi", + "exit 0", + "PY", + ' chmod +x "$venv/bin/python"; exit 0', + "fi", + 'if [ "$1" = "pip" ]; then exit 0; fi', + "exit 2", + "", + ].join("\n"); +} + +function uvArchive(logPath: string): Buffer { + const dir = uvAsset().replace(/\.tar\.gz$/, ""); + return makeTarGz([ + { name: `${dir}/uv`, content: fakeUvScript(logPath), mode: 0o755 }, + { name: `${dir}/uvx`, content: "#!/bin/sh\nexit 0\n", mode: 0o755 }, + ]); +} + +function serve(body: Uint8Array | (() => Response)): void { + vi.stubGlobal("fetch", async (input: unknown) => { + requests.push(String(input)); + return typeof body === "function" ? body() : new Response(body, { status: 200 }); + }); +} + +function binDir(): string { + return join(home, ".prime", "agent", "bin"); +} + +describe.skipIf(process.platform === "win32")("ENG-5343 uv bootstrap", () => { + beforeEach(() => { + originalEnv = { ...process.env }; + home = mkdtempSync(join(tmpdir(), "eng5343-uv-")); + requests = []; + uvState.sha256 = {}; + + // No uv anywhere; fake sh/curl record any attempt to run an installer script. + const fakeBin = join(home, "fake-bin"); + mkdirSync(fakeBin, { recursive: true }); + for (const name of ["sh", "curl"]) { + writeFileSync( + join(fakeBin, name), + `#!/bin/bash\nprintf '%s\\n' "$*" >> "${join(home, `${name}.log`)}"\nexit 99\n`, + ); + chmodSync(join(fakeBin, name), 0o755); + } + process.env.PATH = `${fakeBin}:/usr/bin:/bin`; + process.env.HOME = home; + process.env.PRIME_AGENT_KERNEL_VENV = join(home, "kernel-venv"); + process.env.PRIME_AGENT_INSTALL_UV = "1"; + delete process.env.PRIME_AGENT_KERNEL_PYTHON; + delete process.env.PRIME_AGENT_CODING_AGENT_DIR; + delete process.env.XDG_DATA_HOME; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + process.env = originalEnv; + rmSync(home, { recursive: true, force: true }); + }); + + it("downloads the pinned uv release, verifies it, and installs it without running a shell", async () => { + const uvLog = join(home, "uv.log"); + const archive = uvArchive(uvLog); + uvState.sha256 = { [uvAsset()]: createHash("sha256").update(archive).digest("hex") }; + serve(archive); + const progress: string[] = []; + + const python = await ensureKernelPython({ onProgress: (message) => progress.push(message) }); + + expect(python).toBe(join(home, "kernel-venv", "bin", "python")); + expect(requests).toEqual([helperToolDownloadUrl(HELPER_TOOL_RELEASES.uv, uvAsset())]); + expect(requests[0]).toMatch(/^https:\/\/github\.com\/astral-sh\/uv\/releases\/download\/\d+\.\d+\.\d+\//); + expect(readdirSync(binDir())).toEqual(["uv"]); + expect(readFileSync(uvLog, "utf8")).toContain("python install 3.11"); + expect(existsSync(join(home, "sh.log"))).toBe(false); + expect(existsSync(join(home, "curl.log"))).toBe(false); + expect(progress).toContain(`› installing uv ${HELPER_TOOL_RELEASES.uv.version} (one-time)…`); + }); + + it("rejects a uv archive whose digest does not match the pinned release", async () => { + const uvLog = join(home, "uv.log"); + uvState.sha256 = { [uvAsset()]: "0".repeat(64) }; + serve(uvArchive(uvLog)); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/SHA-256 mismatch/); + + expect(existsSync(uvLog)).toBe(false); + expect(existsSync(binDir()) ? readdirSync(binDir()) : []).toEqual([]); + expect(existsSync(join(home, "sh.log"))).toBe(false); + }); + + it("refuses to install uv when no digest is pinned for this platform", async () => { + serve(new Uint8Array([1])); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/No pinned SHA-256/); + + expect(requests).toEqual([]); + expect(existsSync(binDir()) ? readdirSync(binDir()) : []).toEqual([]); + }); + + it("leaves nothing behind when the uv download is interrupted", async () => { + const archive = uvArchive(join(home, "uv.log")); + uvState.sha256 = { [uvAsset()]: createHash("sha256").update(archive).digest("hex") }; + serve(() => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(archive.subarray(0, 32)); + controller.error(new Error("connection reset")); + }, + }); + return new Response(stream, { status: 200 }); + }); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/connection reset/); + + expect(existsSync(binDir()) ? readdirSync(binDir()) : []).toEqual([]); + expect(existsSync(join(home, "sh.log"))).toBe(false); + }); + + it("does not download anything when installation is refused", async () => { + process.env.PRIME_AGENT_INSTALL_UV = "0"; + serve(new Uint8Array([1])); + + await expect(ensureKernelPython({ onProgress: () => {} })).rejects.toThrow(/PRIME_AGENT_INSTALL_UV=1/); + + expect(requests).toEqual([]); + expect(existsSync(join(home, "sh.log"))).toBe(false); + }); +}); diff --git a/packages/coding-agent/test/tools-manager.test.ts b/packages/coding-agent/test/tools-manager.test.ts index 0db6218bb1..916be89e36 100644 --- a/packages/coding-agent/test/tools-manager.test.ts +++ b/packages/coding-agent/test/tools-manager.test.ts @@ -1,4 +1,5 @@ -import { chmodSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -7,6 +8,7 @@ const toolState = vi.hoisted(() => ({ platform: "linux", architecture: "x64", extractZip: async (_source: string, _options: { dir: string }): Promise => {}, + rgSha256: {} as Record, })); vi.mock("../src/config.js", () => ({ @@ -14,6 +16,22 @@ vi.mock("../src/config.js", () => ({ getBinDir: () => toolState.toolsDir, })); +vi.mock("../src/utils/helper-tool-releases.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + HELPER_TOOL_RELEASES: { + ...actual.HELPER_TOOL_RELEASES, + rg: { + ...actual.HELPER_TOOL_RELEASES.rg, + get sha256() { + return toolState.rgSha256; + }, + }, + }, + }; +}); + vi.mock("os", () => ({ arch: () => toolState.architecture, platform: () => toolState.platform, @@ -23,6 +41,7 @@ vi.mock("extract-zip", () => ({ default: (source: string, options: { dir: string }) => toolState.extractZip(source, options), })); +import { HELPER_TOOL_RELEASES } from "../src/utils/helper-tool-releases.js"; import { ensureToolWithStatus, formatMissingRipgrepMessage, @@ -30,6 +49,13 @@ import { type ToolUnavailableResult, } from "../src/utils/tools-manager.js"; +const RG_WINDOWS_ASSET = `ripgrep-${HELPER_TOOL_RELEASES.rg.version}-x86_64-pc-windows-msvc.zip`; +const ASSET_BYTES = new Uint8Array([1]); + +function pinRgWindowsAsset(bytes: Uint8Array): void { + toolState.rgSha256 = { [RG_WINDOWS_ASSET]: createHash("sha256").update(bytes).digest("hex") }; +} + const originalPath = process.env.PATH; const originalOffline = process.env.PI_OFFLINE; const pathDir = join(toolState.toolsDir, "path"); @@ -55,6 +81,7 @@ describe("tools manager", () => { toolState.platform = "linux"; toolState.architecture = "x64"; toolState.extractZip = async () => {}; + toolState.rgSha256 = {}; }); afterEach(() => { @@ -105,6 +132,7 @@ describe("tools manager", () => { }); toolState.platform = "linux"; + toolState.rgSha256 = { [HELPER_TOOL_RELEASES.rg.assetName("linux", "x64") ?? ""]: "0".repeat(64) }; vi.stubGlobal( "fetch", vi.fn(async () => Promise.reject(new Error("network unavailable"))), @@ -119,15 +147,8 @@ describe("tools manager", () => { it("validates a downloaded binary before reporting it available", async () => { toolState.platform = "win32"; writeExecutable(join(toolState.toolsDir, "rg.exe"), 1); - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - new Response(JSON.stringify({ tag_name: "15.1.0" }), { - status: 200, - headers: { "Content-Type": "application/json" }, - }), - ) - .mockResolvedValueOnce(new Response(new Uint8Array([1]), { status: 200 })); + pinRgWindowsAsset(ASSET_BYTES); + const fetchMock = vi.fn().mockResolvedValueOnce(new Response(ASSET_BYTES, { status: 200 })); vi.stubGlobal("fetch", fetchMock); toolState.extractZip = async (_source, options) => { writeExecutable(join(options.dir, "rg.exe")); @@ -137,18 +158,17 @@ describe("tools manager", () => { status: "available", path: join(toolState.toolsDir, "rg.exe"), }); - expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][0]).toBe( + `https://github.com/BurntSushi/ripgrep/releases/download/${HELPER_TOOL_RELEASES.rg.tag}/${RG_WINDOWS_ASSET}`, + ); + expect(readdirSync(toolState.toolsDir).sort()).toEqual(["path", "rg.exe"]); }); it("removes a downloaded binary that fails its version check", async () => { toolState.platform = "win32"; - vi.stubGlobal( - "fetch", - vi - .fn() - .mockResolvedValueOnce(new Response(JSON.stringify({ tag_name: "15.1.0" }), { status: 200 })) - .mockResolvedValueOnce(new Response(new Uint8Array([1]), { status: 200 })), - ); + pinRgWindowsAsset(ASSET_BYTES); + vi.stubGlobal("fetch", vi.fn().mockResolvedValueOnce(new Response(ASSET_BYTES, { status: 200 }))); toolState.extractZip = async (_source, options) => { writeExecutable(join(options.dir, "rg.exe"), 1); }; @@ -158,6 +178,23 @@ describe("tools manager", () => { reason: "download_failed", }); expect(existsSync(join(toolState.toolsDir, "rg.exe"))).toBe(false); + expect(readdirSync(toolState.toolsDir)).toEqual(["path"]); + }); + + it("rejects a downloaded asset whose digest differs from the pinned release", async () => { + toolState.platform = "win32"; + pinRgWindowsAsset(new Uint8Array([2])); + vi.stubGlobal("fetch", vi.fn().mockResolvedValueOnce(new Response(ASSET_BYTES, { status: 200 }))); + const extractZip = vi.fn(async () => {}); + toolState.extractZip = extractZip; + + await expect(ensureToolWithStatus("rg")).resolves.toMatchObject({ + status: "unavailable", + reason: "download_failed", + detail: expect.stringContaining("SHA-256 mismatch"), + }); + expect(extractZip).not.toHaveBeenCalled(); + expect(readdirSync(toolState.toolsDir)).toEqual(["path"]); }); it("formats actionable platform-specific ripgrep warnings", () => {