feat(graph): provision tree-sitter into embed-deps so the auto-build hook works

src/cli/embeddings.ts

@@ -5,6 +5,7 @@ import { join } from "node:path";
 import { HOME, ensureDir, log, pkgRoot, symlinkForce, warn, writeJson } from "./util.js";
 import { pidPathFor, socketPathFor } from "../embeddings/protocol.js";
 import { getEmbeddingsEnabled, setEmbeddingsEnabled } from "../user-config.js";
+import { ensureGraphDeps } from "./graph-deps.js";
 
 /**
  * Shared-deps location for the embedding daemon's runtime dependencies.
@@ -107,11 +108,17 @@ function ensureSharedDeps(): void {
     log(`  Embeddings     installing ${TRANSFORMERS_PKG}@${TRANSFORMERS_RANGE} into ${SHARED_DIR}`);
     log(`                 (~600 MB; first install only — every agent will share this)`);
     ensureDir(SHARED_DIR);
-    writeJson(join(SHARED_DIR, "package.json"), {
+    // Merge, don't clobber: ensureGraphDeps may have already declared the
+    // tree-sitter parsers here. Overwriting with transformers-only would drop
+    // that declaration and let a subsequent reconcile prune the parsers.
+    const pkgPath = join(SHARED_DIR, "package.json");
+    let existing: { dependencies?: Record<string, string> } = {};
+    try { existing = JSON.parse(readFileSync(pkgPath, "utf8")); } catch { /* none yet */ }
+    writeJson(pkgPath, {
       name: "hivemind-embed-deps",
       version: "1.0.0",
       private: true,
-      dependencies: { [TRANSFORMERS_PKG]: TRANSFORMERS_RANGE },
+      dependencies: { ...(existing.dependencies ?? {}), [TRANSFORMERS_PKG]: TRANSFORMERS_RANGE },
     });
     execFileSync("npm", ["install", "--omit=dev", "--no-package-lock", "--no-audit", "--no-fund"], {
       cwd: SHARED_DIR,
@@ -170,6 +177,10 @@ function linkAgent(install: AgentInstall): void {
  */
 export function installEmbeddings(): void {
   ensureSharedDeps();
+  // Provision the code-graph parsers into the same shared dir so the
+  // graph-on-stop hook (which symlinks here) can auto-build the graph.
+  // Best-effort — a native-build failure never aborts the embeddings install.
+  ensureGraphDeps();
   const installs = findHivemindInstalls();
   if (installs.length === 0) {
     warn("  Embeddings     no hivemind installs detected — run `hivemind install` first");

src/cli/graph-deps.ts

@@ -0,0 +1,152 @@
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { ensureDir, log, pkgRoot, warn, writeJson } from "./util.js";
+import { SHARED_DIR, SHARED_NODE_MODULES } from "./embeddings.js";
+
+/**
+ * The code-graph feature resolves `tree-sitter` (+ language grammars) at
+ * runtime from the SAME shared `embed-deps/node_modules` that the graph-on-stop
+ * hook symlinks to. `ensureSharedDeps` only installs @huggingface/transformers
+ * there, so on a real agent install tree-sitter is absent and the graph never
+ * auto-builds (the hook degrades to a graceful skip). This module provisions
+ * the parsers where the hook looks.
+ *
+ * tree-sitter is a NATIVE module: symlinking the package dir alone is not
+ * enough (it needs its own node-gyp-build / node-addon-api siblings resolved),
+ * so we run a scoped `npm install` and let npm lay down the full tree.
+ */
+export function isGraphDepsInstalled(
+  sharedNodeModules: string = SHARED_NODE_MODULES,
+  specs: string[] = treeSitterSpecs(),
+): boolean {
+  // Require EVERY requested parser to be present — not just `tree-sitter` — so
+  // an interrupted or partial install counts as "needs (re)install" rather
+  // than being skipped as done. `<name>@<range>` → dir name is the part before
+  // the (unscoped) version separator.
+  if (specs.length === 0) return false;
+  return specs.every((s) => existsSync(join(sharedNodeModules, s.slice(0, s.lastIndexOf("@")))));
+}
+
+/**
+ * Build `<name>@<range>` specs for every tree-sitter* entry in the package's
+ * own optionalDependencies. Reading them from package.json (rather than a
+ * hardcoded list) keeps the embed-deps versions in lockstep with what the
+ * bundles were built against — no drift, no ABI mismatch.
+ */
+export function treeSitterSpecs(pkgJsonPath: string = join(pkgRoot(), "package.json")): string[] {
+  let opt: Record<string, string> = {};
+  try {
+    const pkg = JSON.parse(readFileSync(pkgJsonPath, "utf8"));
+    opt = pkg.optionalDependencies ?? {};
+  } catch { return []; }
+  return Object.keys(opt)
+    .filter((n) => n === "tree-sitter" || n.startsWith("tree-sitter-"))
+    .sort()
+    .map((n) => `${n}@${opt[n]}`);
+}
+
+/**
+ * Injection seam for ensureGraphDeps. Tests replace the two external boundaries
+ * (`runNpm` = the npm install, `runHeal` = the native-build heal) and point
+ * `sharedDir` / `sharedNodeModules` / `specs` at a tmp fixture, so the branch
+ * logic runs against real fs without spawning npm or compiling anything.
+ */
+export interface GraphDepsDeps {
+  sharedDir?: string;
+  sharedNodeModules?: string;
+  specs?: string[];
+  runNpm?: (specs: string[], cwd: string) => void;
+  runHeal?: (cwd: string) => void;
+  logFn?: (msg: string) => void;
+  warnFn?: (msg: string) => void;
+}
+
+/**
+ * Install the tree-sitter parsers into the shared embed-deps dir. Idempotent
+ * (a specs-hash marker + a per-package presence check skip the re-download,
+ * and catch version bumps / partial installs) and best-effort: any failure is
+ * logged and swallowed so it never aborts the caller — the graph stays
+ * disabled but nothing else breaks, and the graph-on-stop hook degrades
+ * gracefully rather than crashing.
+ *
+ * Two-phase native provisioning: `npm install --ignore-scripts` first, so a
+ * platform without a prebuild (arm64 / Node 24) doesn't fail the download,
+ * THEN scripts/ensure-tree-sitter.mjs always runs — it validates the bindings
+ * load and compiles from source where no prebuild exists (a fast no-op on
+ * healthy prebuilt installs). Running the heal unconditionally repairs a
+ * previously interrupted or ABI-mismatched install.
+ *
+ * Decoupled from the ~600 MB embeddings download: this installs ONLY the
+ * parsers (tens of MB), so the code graph can work without semantic search.
+ */
+export function ensureGraphDeps(deps: GraphDepsDeps = {}): void {
+  const {
+    sharedDir = SHARED_DIR,
+    sharedNodeModules = SHARED_NODE_MODULES,
+    specs = treeSitterSpecs(),
+    runNpm = defaultRunNpm,
+    runHeal = defaultRunHeal,
+    logFn = log,
+    warnFn = warn,
+  } = deps;
+  try {
+    if (specs.length === 0) {
+      warnFn(`  Graph          no tree-sitter optionalDependencies found in package.json — skipping`);
+      return;
+    }
+    ensureDir(sharedDir);
+    // Create a package.json if none exists yet (user never ran embeddings
+    // install). Don't clobber an existing one — it may already declare
+    // transformers; the parsers are installed alongside.
+    const pkgPath = join(sharedDir, "package.json");
+    if (!existsSync(pkgPath)) {
+      writeJson(pkgPath, { name: "hivemind-embed-deps", version: "1.0.0", private: true, dependencies: {} });
+    }
+    // Skip the (re)download only when the exact spec set was installed before
+    // AND every package is still present. The marker also catches version
+    // bumps: a changed spec set forces a reinstall.
+    const marker = join(sharedDir, ".graph-deps");
+    const wantKey = specs.join("\n");
+    const haveKey = existsSync(marker) ? readFileSync(marker, "utf8") : "";
+    if (haveKey !== wantKey || !isGraphDepsInstalled(sharedNodeModules, specs)) {
+      logFn(`  Graph          installing tree-sitter parsers into ${sharedDir} (code-graph; ~tens of MB)`);
+      runNpm(specs, sharedDir);
+      writeFileSync(marker, wantKey);
+    } else {
+      logFn(`  Graph          tree-sitter parsers already present at ${sharedDir}`);
+    }
+    // ALWAYS heal: validates bindings load + compiles from source where no
+    // prebuild exists (arm64 / Node 24). Fast no-op on healthy prebuilt
+    // installs; repairs an interrupted or ABI-mismatched one.
+    runHeal(sharedDir);
+  } catch (err) {
+    warnFn(`  Graph          tree-sitter provisioning failed (${err instanceof Error ? err.message : String(err)}); code graph stays disabled — everything else works`);
+  }
+}
+
+/**
+ * Default npm boundary. `--ignore-scripts` fetches the packages without the
+ * native build that fails on platforms lacking a prebuild (the heal does the
+ * build). The parsers ARE saved to the shared package.json (default): a plain
+ * `npm install` PRUNES unsaved packages (verified on npm 11), so an unsaved
+ * install would be wiped by ensureSharedDeps' transformers reconcile. Being
+ * declared, they're neither pruned nor (once present and version-satisfied)
+ * rebuilt by that reconcile — so ensureSharedDeps stays free of
+ * --ignore-scripts, which would break onnxruntime-node / sharp (both have real
+ * install/postinstall native steps).
+ */
+function defaultRunNpm(specs: string[], cwd: string): void {
+  execFileSync("npm", ["install", ...specs, "--omit=dev", "--no-package-lock", "--no-audit", "--no-fund", "--ignore-scripts"], {
+    cwd,
+    stdio: "inherit",
+  });
+}
+
+/** Default heal boundary: run scripts/ensure-tree-sitter.mjs if present. */
+function defaultRunHeal(cwd: string): void {
+  const heal = join(pkgRoot(), "scripts", "ensure-tree-sitter.mjs");
+  if (existsSync(heal)) {
+    execFileSync(process.execPath, [heal], { cwd, stdio: "inherit" });
+  }
+}

src/cli/index.ts

@@ -12,6 +12,7 @@ import {
   statusEmbeddings,
   uninstallEmbeddings,
 } from "./embeddings.js";
+import { ensureGraphDeps } from "./graph-deps.js";
 import { ensureLoggedIn, isLoggedIn, loginWithProvidedToken, maybeShowOrgChoice } from "./auth.js";
 import { runAuthCommand } from "../commands/auth-login.js";
 // NOTE: ../commands/graph.js is intentionally NOT imported statically. It pulls
@@ -547,6 +548,14 @@ async function main(): Promise<void> {
   }
 
   if (cmd === "graph") {
+    // `graph init` sets up the auto-build. Provision the tree-sitter parsers
+    // into the shared embed-deps dir the graph-on-stop hook resolves from, so
+    // the automatic rebuild works — not just this CLI run (which resolves
+    // tree-sitter from its own package). Idempotent + best-effort; decoupled
+    // from the ~600 MB embeddings download.
+    if (args[1] === "init") {
+      try { ensureGraphDeps(); } catch { /* best-effort — never block graph init */ }
+    }
     let runGraphCommand: (a: string[]) => Promise<void> | void;
     try {
       ({ runGraphCommand } = await import("../commands/graph.js"));

tests/cli/cli-embeddings-install.test.ts

@@ -0,0 +1,157 @@
+import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, vi } from "vitest";
+import { mkdtempSync, mkdirSync, existsSync, rmSync, lstatSync, readlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { setFakeHome, clearFakeHome } from "../shared/fake-home.js";
+
+// Mock ONLY the process boundary: ensureSharedDeps / ensureGraphDeps shell out
+// to `npm install`, which we must not actually run. spawnSync + everything else
+// stay real (importActual).
+vi.mock("node:child_process", async (orig) => ({
+  ...(await orig<typeof import("node:child_process")>()),
+  execFileSync: vi.fn(),
+}));
+
+// embeddings.ts binds `HOME`/`SHARED_DIR` at module-evaluation time, and the
+// orchestration functions (installEmbeddings/statusEmbeddings/uninstall) read
+// them (and findHivemindInstalls) with NO injectable seam. Point HOME at a tmp
+// dir BEFORE importing the module so every path resolves under the sandbox and
+// nothing touches the real ~/.hivemind or the real agent installs.
+const HOME_DIR = mkdtempSync(join(tmpdir(), "emb-install-home-"));
+
+// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+let mod: typeof import("../../src/cli/embeddings.js");
+let cfg: typeof import("../../src/user-config.js");
+
+const PREV_TABLE = process.env.HIVEMIND_TABLE;
+
+beforeAll(async () => {
+  // Sandbox the home dir on POSIX AND Windows (os.homedir reads USERPROFILE on
+  // Windows) BEFORE importing embeddings.ts, which binds HOME/SHARED_DIR at load.
+  setFakeHome(HOME_DIR);
+  delete process.env.HIVEMIND_TABLE;
+  mod = await import("../../src/cli/embeddings.js");
+  cfg = await import("../../src/user-config.js");
+});
+
+afterAll(() => {
+  clearFakeHome();
+  if (PREV_TABLE === undefined) delete process.env.HIVEMIND_TABLE;
+  else process.env.HIVEMIND_TABLE = PREV_TABLE;
+  rmSync(HOME_DIR, { recursive: true, force: true });
+});
+
+function freshHome(): void {
+  for (const sub of [".hivemind", ".deeplake", ".codex", ".cursor", ".hermes", ".claude"]) {
+    rmSync(join(HOME_DIR, sub), { recursive: true, force: true });
+  }
+  cfg._resetUserConfigForTesting();
+  cfg._setConfigPathForTesting(() => join(HOME_DIR, ".deeplake", "config.json"));
+  mkdirSync(join(HOME_DIR, ".deeplake"), { recursive: true });
+}
+
+describe("installEmbeddings (sandboxed HOME, mocked npm)", () => {
+  beforeEach(() => { freshHome(); });
+  afterEach(() => { cfg._resetUserConfigForTesting(); });
+
+  it("no agent installs → still provisions shared deps and flips the config flag on", async () => {
+    const cp = await import("node:child_process");
+    vi.mocked(cp.execFileSync).mockClear();
+    mod.installEmbeddings();
+    // ensureSharedDeps ran its npm install (mocked) into the sandbox shared dir.
+    expect(vi.mocked(cp.execFileSync).mock.calls.some((c) => c[0] === "npm")).toBe(true);
+    // A shared package.json was laid down under the tmp HOME.
+    expect(existsSync(join(HOME_DIR, ".hivemind", "embed-deps", "package.json"))).toBe(true);
+    // Config flag flipped on regardless of whether any agent was detected.
+    expect(cfg.getEmbeddingsEnabled()).toBe(true);
+  });
+
+  it("shared deps already present → skips the npm install (isSharedDepsInstalled true branch)", async () => {
+    // Pre-create <shared>/node_modules/@huggingface/transformers so
+    // ensureSharedDeps takes its "already present" skip path.
+    mkdirSync(join(mod.SHARED_NODE_MODULES, mod.TRANSFORMERS_PKG), { recursive: true });
+    const cp = await import("node:child_process");
+    vi.mocked(cp.execFileSync).mockClear();
+    mod.installEmbeddings();
+    // No transformers `npm install` — the only npm call (if any) is the graph
+    // parsers, never the transformers one.
+    const transformersInstall = vi.mocked(cp.execFileSync).mock.calls.some(
+      (c) => c[0] === "npm" && Array.isArray(c[1]) && c[1].includes(mod.TRANSFORMERS_PKG),
+    );
+    expect(transformersInstall).toBe(false);
+    expect(cfg.getEmbeddingsEnabled()).toBe(true);
+  });
+
+  it("with a detected agent install → symlinks its node_modules to the shared deps", async () => {
+    // Lay down a codex plugin bundle so findHivemindInstalls() sees it.
+    mkdirSync(join(HOME_DIR, ".codex", "hivemind", "bundle"), { recursive: true });
+    mod.installEmbeddings();
+    const link = join(HOME_DIR, ".codex", "hivemind", "node_modules");
+    // npm is mocked so the shared node_modules target is never created; assert
+    // the symlink itself was laid down and points at the canonical shared dir
+    // (existsSync would follow the link to the absent target and report false).
+    expect(lstatSync(link).isSymbolicLink()).toBe(true);
+    expect(readlinkSync(link)).toBe(mod.SHARED_NODE_MODULES);
+    expect(cfg.getEmbeddingsEnabled()).toBe(true);
+  });
+});
+
+describe("uninstallEmbeddings --prune (sandboxed HOME)", () => {
+  beforeEach(() => { freshHome(); });
+  afterEach(() => { cfg._resetUserConfigForTesting(); });
+
+  it("prune removes the shared-deps dir and flips the flag off", () => {
+    mkdirSync(mod.SHARED_DIR, { recursive: true });
+    expect(existsSync(mod.SHARED_DIR)).toBe(true);
+    mod.uninstallEmbeddings({ prune: true });
+    expect(existsSync(mod.SHARED_DIR)).toBe(false);
+    expect(cfg.getEmbeddingsEnabled()).toBe(false);
+  });
+
+  it("without prune, leaves the shared-deps dir in place but still disables", () => {
+    mkdirSync(mod.SHARED_DIR, { recursive: true });
+    mod.uninstallEmbeddings();
+    expect(existsSync(mod.SHARED_DIR)).toBe(true);
+    expect(cfg.getEmbeddingsEnabled()).toBe(false);
+  });
+});
+
+describe("statusEmbeddings (sandboxed HOME)", () => {
+  beforeEach(() => { freshHome(); });
+  afterEach(() => { cfg._resetUserConfigForTesting(); vi.restoreAllMocks(); });
+
+  it("prints config + shared-deps + per-agent link state without throwing", () => {
+    mkdirSync(join(HOME_DIR, ".codex", "hivemind", "bundle"), { recursive: true });
+    // log()/warn() write to process.stdout/stderr (not console.*), so capture
+    // the raw streams.
+    const lines: string[] = [];
+    const outSpy = vi.spyOn(process.stdout, "write").mockImplementation((c: unknown) => { lines.push(String(c)); return true; });
+    const errSpy = vi.spyOn(process.stderr, "write").mockImplementation((c: unknown) => { lines.push(String(c)); return true; });
+    expect(() => mod.statusEmbeddings()).not.toThrow();
+    outSpy.mockRestore();
+    errSpy.mockRestore();
+    const out = lines.join("\n");
+    // Shared deps absent + codex has no node_modules symlink → specific lines.
+    expect(out).toContain(`Shared deps:   ${mod.SHARED_DIR}`);
+    expect(out).toContain("Installed:     no");
+    expect(out).toMatch(/codex\s+✗ not linked/);
+  });
+
+  it("reports a linked agent + present shared deps (linked/present status arms)", () => {
+    // Shared deps present + codex bundle symlinked into them → exercises the
+    // isSharedDepsInstalled=true and linked-to-shared status branches.
+    mkdirSync(join(HOME_DIR, ".hivemind", "embed-deps", "node_modules", "@huggingface", "transformers"), { recursive: true });
+    mkdirSync(join(HOME_DIR, ".codex", "hivemind", "bundle"), { recursive: true });
+    mod._linkAgentForTesting({ id: "codex", pluginDir: join(HOME_DIR, ".codex", "hivemind") });
+    const lines: string[] = [];
+    const outSpy = vi.spyOn(process.stdout, "write").mockImplementation((c: unknown) => { lines.push(String(c)); return true; });
+    const errSpy = vi.spyOn(process.stderr, "write").mockImplementation((c: unknown) => { lines.push(String(c)); return true; });
+    expect(() => mod.statusEmbeddings()).not.toThrow();
+    outSpy.mockRestore();
+    errSpy.mockRestore();
+    const out = lines.join("\n");
+    // Shared deps present + codex symlinked → the "installed" + linked arms.
+    expect(out).toContain("Installed:     yes");
+    expect(out).toMatch(/codex\s+✓ linked → shared/);
+  });
+});