diff --git a/.changeset/phase-150-git-arg-injection.md b/.changeset/phase-150-git-arg-injection.md new file mode 100644 index 0000000..e82ba25 --- /dev/null +++ b/.changeset/phase-150-git-arg-injection.md @@ -0,0 +1,13 @@ +--- +"gitsema": patch +--- + +Security (Phase 150 / review11 §2.1 + §3.2): close the network-reachable git +argument-injection sink. A caller-supplied "ref" beginning with `-` (e.g. +`--output=/path`) was parsed by git as a *flag*, turning `git log` into an +arbitrary-file-write primitive reachable via `semantic_bisect`/`triage`. All +git call sites that take a user-influenced ref now route through a shared +`runGit()` helper that rejects leading-`-` refs before spawning git and always +inserts git's `--end-of-options` separator so a value can never be read as a +flag (`resolveRefToTimestamp`, `parseDateArg`, `getMergeBase`, +`getBranchExclusiveBlobs`). diff --git a/.changeset/phase-151-read-route-authz.md b/.changeset/phase-151-read-route-authz.md new file mode 100644 index 0000000..23ce5bb --- /dev/null +++ b/.changeset/phase-151-read-route-authz.md @@ -0,0 +1,15 @@ +--- +"gitsema": patch +--- + +Security (Phase 151 / review11 §2.2): enforce repo authorization on read +routes. The multi-tenant grant model (`repo_grants` / `resolveUserRepoAccess`) +was defined but never checked on the ~16 search/analysis/evolution/graph/ +insights routes, so any caller could read any repo's indexed content by naming +its `repoId`. A new `repoAuthMiddleware` now runs after `repoSessionMiddleware` +and, in multi-tenant mode, requires the caller to hold a `read` grant on the +addressed repo unless it is `public` (else 403). Multi-tenant mode is opt-in +via `GITSEMA_MULTI_TENANT` (defaulting to `GITSEMA_SERVE_KEY` presence); the +global serve key and legacy per-repo scoped tokens bypass the check, and a +default open single-dev server is unaffected. Repo-level only — per-branch +grant filtering is deferred to a follow-on phase. diff --git a/.changeset/phase-152-byok-ssrf-list-bounds.md b/.changeset/phase-152-byok-ssrf-list-bounds.md new file mode 100644 index 0000000..a3e2a6c --- /dev/null +++ b/.changeset/phase-152-byok-ssrf-list-bounds.md @@ -0,0 +1,15 @@ +--- +"gitsema": patch +--- + +Security (Phase 152 / review11 §3.1 + §3.3). **BYOK SSRF guard:** on +`tools serve`, a caller-supplied `byok.http_url` is now validated before the +server calls it — non-`http(s)` schemes and hosts resolving to loopback, +link-local (incl. the `169.254.169.254` cloud-metadata IP), or RFC-1918 +private ranges are rejected by default. Operators re-permit specific internal +hosts (e.g. a local model server) via the new `GITSEMA_BYOK_ALLOW_HOSTS` +allowlist. This is a behavior change for anyone pointing BYOK at a +`localhost`/private endpoint — add the host to the allowlist. **List-tool +bounds:** the network-exposed `deps` and `blast_radius` `depth` parameter is +now upper-bounded (max 64) on both the HTTP route and MCP tool, closing the +last unbounded traversal-depth input from the Phase 147/148 exposure. diff --git a/CLAUDE.md b/CLAUDE.md index 0455302..b308a9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -445,6 +445,8 @@ Configuration is via environment variables or the `gitsema config` command (pers | `GITSEMA_LOG_MAX_BYTES` | `1048576` | Log rotation threshold (1 MB) | | `GITSEMA_SERVE_PORT` | `4242` | Port for `gitsema tools serve` HTTP server | | `GITSEMA_SERVE_KEY` | *(optional)* | Bearer token required by `gitsema tools serve` | +| `GITSEMA_MULTI_TENANT` | *(unset → follows `GITSEMA_SERVE_KEY` presence)* | Opt-in read-route authorization gate (Phase 151). When multi-tenant mode is active, a request naming a `repoId` requires the caller to hold a `read` grant on that repo (or the repo to be `public`); the global key and legacy per-repo scoped tokens bypass the check. Explicit `1`/`true`/`yes`/`on` enables it, `0`/`false` disables it even when a serve key is set; when unset, enforcement follows `GITSEMA_SERVE_KEY` presence. A default open server (no key, no flag) is unaffected. | +| `GITSEMA_BYOK_ALLOW_HOSTS` | *(empty)* | Comma-separated host/CIDR allowlist that re-permits otherwise-blocked BYOK endpoint hosts (loopback/link-local/RFC-1918). BYOK narrator/guide endpoints are SSRF-guarded by default (Phase 152); add internal model-server hosts here to allow them. | | `GITSEMA_LLM_URL` | *(optional)* | OpenAI-compatible URL for `--narrate` LLM summaries | | `GITSEMA_DATA_DIR` | `~/.gitsema/data` | Root directory for `gitsema tools serve`'s persisted repo clones + index DBs (`repos//{repo,index.db}`, `registry.db`) | | `GITSEMA_STORAGE_BACKEND` | `sqlite` | `sqlite` \| `postgres` \| `qdrant` — selects the storage profile (Phase 101–103) | diff --git a/README.md b/README.md index 1e34586..96819a7 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ Configuration is read from environment variables or persisted via `gitsema confi | `GITSEMA_LOG_MAX_BYTES` | `1048576` | Log rotation threshold (1 MB) | | `GITSEMA_SERVE_PORT` | `4242` | Port for `gitsema tools serve` | | `GITSEMA_SERVE_KEY` | *(optional)* | Bearer token required by `gitsema tools serve` | +| `GITSEMA_MULTI_TENANT` | *(unset → follows `GITSEMA_SERVE_KEY`)* | Opt-in read-route authorization: when active, a request naming a `repoId` requires the caller to hold a `read` grant on that repo (or the repo to be `public`). `1`/`true`/`yes`/`on` enables, `0`/`false` disables even with a serve key; unset follows `GITSEMA_SERVE_KEY` presence. A default open server (no key, no flag) is unaffected. | +| `GITSEMA_BYOK_ALLOW_HOSTS` | *(empty)* | Comma-separated host/CIDR allowlist re-permitting otherwise-blocked BYOK endpoint hosts (loopback/link-local/RFC-1918). BYOK narrator/guide endpoints are SSRF-guarded by default. | | `GITSEMA_WEBSOCKET_KEY` | *(optional)* | Bearer token fallback for `tools mcp --websocket`/`tools lsp --websocket` when `--key` is omitted | | `GITSEMA_MCP_HTTP_KEY` | *(optional)* | Bearer token fallback for `tools mcp --http` when `--key` is omitted | | `GITSEMA_MAX_BODY_SIZE` | `1mb` | Body-size cap for `tools serve` and `tools mcp --http` (e.g. `5mb`) | @@ -347,6 +349,8 @@ Both commands are **safe-by-default**: with no narrator model configured (or wit | `--byok-max-tokens ` | — | Max tokens per BYOK call | | `--byok-temperature ` | — | Temperature for BYOK calls | +> **SSRF note (server deployments):** on `gitsema tools serve`, the server issues the request to the caller-supplied `--byok-http-url`/`byok.http_url`. To prevent it being used to reach internal/metadata endpoints, URLs whose host resolves to loopback, link-local (incl. `169.254.169.254`), or RFC-1918 private ranges are **rejected by default**. Re-permit specific internal hosts via `GITSEMA_BYOK_ALLOW_HOSTS` (comma-separated host/CIDR allowlist). This does not affect local CLI use, only what a networked server will call. + Configure a narrator model with `gitsema models add --narrator --http-url [--key ] --activate`, with a local Ollama model (`gitsema models add --narrator --provider ollama [--global-name ] --activate`, see "Ollama" below), or with a local CLI AI tool: `gitsema models add --narrator --provider cli --cli-command [--cli-args ""] --activate` (see "CLI-based AI tool backends" below). Alternatively, pass `--byok-http-url` (and optionally `--byok-api-key`/`--byok-model`) for a one-off, never-persisted request-scoped model — it bypasses the configured/allow-listed model selection entirely (Phase 130). #### `--narrate` on other commands diff --git a/docs/PLAN.md b/docs/PLAN.md index e1ded11..93b97df 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -6096,7 +6096,29 @@ rather than continuing to carry it forward. --- -### Phase 150 — Git argument-injection close-out (review11 §2.1 + §3.2) +### Phase 150 — Git argument-injection close-out (review11 §2.1 + §3.2) ✅ + +**Status:** ✅ complete. Landed a shared `runGit(subcommand, flags, refs, +options)` helper (`src/core/git/runGit.ts`) that (a) rejects any ref failing +`isSafeGitRange()` (leading `-` and non-ref characters) before spawning git, +throwing `UnsafeGitRefError`, and (b) always inserts git's `--end-of-options` +marker before the positional refs so a value can never be reparsed as a flag. +Routed the three PLAN-scoped sinks through it: `resolveRefToTimestamp` +(`clustering.ts` — the confirmed §2.1 `semantic_bisect`/`triage` sink), +`parseDateArg` (`timeSearch.ts` — §3.2 `rev-parse`/`show`), and +`getMergeBase`/`getBranchExclusiveBlobs` (`branchDiff.ts` — §3.2 `merge-base`). +Regression suite `tests/integration/gitArgInjection.test.ts` (8 tests) asserts +`runGit`/`resolveRefToTimestamp` reject a `--output=` ref and write no +file (PoC no longer reproduces), while still resolving legitimate refs and ISO +dates. **Deviations:** (1) used git's `--end-of-options` rather than a bare +`--` — empirically, a bare `--` before a ref makes `git log`/`show` treat the +ref as a *pathspec*, silently breaking valid-ref resolution; `--end-of-options` +(git ≥ 2.24) stops flag parsing without reclassifying the positional, verified +against git 2.43. (2) `regressionGate.ts`/`codeReview.ts` were left as-is: +they already gate their refs through `isSafeGitRange()` (the review10 §7 fix) +and so already satisfy the "validates the ref" acceptance criterion; they are +CLI-only, not network-exposed. (3) `isSafeGitRange` had already been relocated +to `src/core/git/refSafety.ts` in prior work, so no move was needed. **Design:** no separate design doc — scoped directly from review11 §2.1 (PoC-confirmed) and §3.2. Root cause: the review9/10 injection fixes switched @@ -6157,7 +6179,37 @@ call sites or network entry points are added. --- -### Phase 151 — Read-route repo authorization gate (review11 §2.2) +### Phase 151 — Read-route repo authorization gate (review11 §2.2) ✅ + +**Status:** ✅ complete. Added `repoAuthMiddleware` +(`src/server/middleware/repoAuth.ts`), mounted immediately after +`repoSessionMiddleware` on all ten data-route groups (search, evolution, +analysis, insights, graph, protocol, watch, projections, narrate/explain, +guide). In multi-tenant mode it requires +`roleSatisfies(resolveUserRepoAccess(getRawDb(), userId, repoId), 'read')` +unless the repo's mirror row is `visibility === 'public'`, else 403. +`resolveRequestedRepoId()` was extracted from `repoSessionMiddleware` so both +middlewares resolve the addressed repo identically. `authMiddleware` now marks +`req.globalKeyAuth` (global `GITSEMA_SERVE_KEY`) and `req.repoTokenScoped` +(legacy per-repo token); both bypass the grant check since they already imply +access. **Multi-tenant trigger (decided):** `isMultiTenantMode()` = +explicit `GITSEMA_MULTI_TENANT` (`1`/`true`/`yes`/`on` → on, `0`/`false` → +off) if set, else `GITSEMA_SERVE_KEY` presence; a default open server (no key, +no flag) is a no-op. Documented in `CLAUDE.md`/`README.md` config tables. +Tests: `tests/repoAuthMiddleware.test.ts` (15 cases) covers granted-read (pass), +un-granted/unauthenticated private (403), public (pass), global-key bypass, +scoped-token bypass, missing-mirror-defaults-private, query-string repoId, and +the `isMultiTenantMode` trigger matrix; the full server-route suite (247 tests) +confirms open-mode behavior is unchanged (no new 403s). **Deviations:** (1) +visibility is read from the addressed repo's own DB mirror row +(`getRepo(getActiveSession(), repoId)`) — the authoritative source used by the +shipped Phase 126 register path in `remote.ts` — which is why the middleware +mounts *after* `repoSessionMiddleware` (so the repo DB is active) rather than +before. (2) Enforcement is verified at the middleware unit level plus a +full-suite open-mode regression rather than a new supertest e2e, matching the +existing `repoSessionMiddleware.test.ts` unit-only convention (a real e2e would +require standing up the three-DB registry/control-plane/repo topology). +(3) Per-branch filtering remains deferred as specified below. **Design:** no separate design doc — scoped directly from review11 §2.2 (disclosed in PLAN.md as "Phase 123 deviation #1"). The Multi-Tenant Auth @@ -6224,7 +6276,32 @@ repo-level helper is added), `CLAUDE.md` (config docs), tests, --- -### Phase 152 — BYOK SSRF guard + newly-exposed list-tool bounds (review11 §3.1 + §3.3) +### Phase 152 — BYOK SSRF guard + newly-exposed list-tool bounds (review11 §3.1 + §3.3) ✅ + +**Status:** ✅ complete. **§3.1 (BYOK SSRF):** `validateByokUrl()` in +`resolveNarrator.ts` (invoked from `byokConfig()` before the provider is +constructed, on the `narrate`/`explain`/`guide` HTTP routes) requires an +`http(s)` scheme and rejects hosts resolving to loopback (`127.0.0.0/8`, +`::1`), link-local (`169.254.0.0/16` incl. the `169.254.169.254` metadata IP, +`fe80::/10`), and RFC-1918 ranges — including via DNS resolution for +non-literal hosts — unless the host matches `GITSEMA_BYOK_ALLOW_HOSTS` +(comma-separated host/CIDR allowlist, default empty). The routes surface it as +a 400 `ByokUrlValidationError`. **§3.3 (list bounds):** audited every Phase +147/148-exposed list tool for an upper bound; all `top_k`/`top`/`limit` params +already carried `.max(...)` (500 for graph tools, 50 for insights). The one +gap was the `depth` param on `deps` (whose core BFS uses `depth ?? Infinity` +with no server-side clamp) and `blast_radius`; both now carry +`.max(MAX_GRAPH_DEPTH_REQUEST)` (= 64, a new shared constant in +`storage/types.ts`) on the HTTP schema and MCP tool. **Deviations:** (1) the +§3.1 SSRF guard and its allowlist were already implemented + unit-tested in a +prior commit; this phase verified completeness, extended the tests +(`byokCredentials.test.ts`: metadata IP, RFC-1918, non-http scheme, public +pass, CIDR re-permit), and documented it in README/features/CLAUDE config +tables. (2) `blast_radius`'s traversal already clamps to +`MAX_GRAPH_TRAVERSAL_DEPTH` (3), so its 64-cap is documentation/defense-in-depth +rather than the effective limit; `deps` is the only genuinely-unbounded site +the cap constrains. (3) Guide (local, in-process) tool schemas were not +bounded — §3.3 scopes to *network-exposed* tools only. **Design:** no separate design doc — scoped directly from review11 §3.1 and §3.3. §3.1: Phase 130's BYOK uses a request-supplied `byok.http_url` @@ -6307,350 +6384,6 @@ URL-guard helper), `src/server/routes/{narrator,guide}.ts`, **Acceptance criteria:** - CLI: `gitsema search`, `first-seen`, `code-search` output includes `[blob:...]` prefix ✅ - MCP: `semantic_search`, `search_history`, `code_search` text output includes `[blob:...]` prefix ✅ - ---- - -### Phase 154 — Semantic Refs & Incremental Transfer - -**Design:** Full design in [`docs/design/semantic-federation.md`](design/semantic-federation.md). - -**Goal:** Enable semantic index snapshots and incremental sync between repositories, forming the foundation of federated semantic knowledge exchange. - -**Key design decisions:** -- **Chunk-level granularity:** Semantic objects wrap *chunks*, not whole blobs. Enables fine-grained deduplication and specific summaries. -- **Model versioning:** Every semantic object stores `profile_version` (e.g. "text-embedding-3-small:1.0") and `model_dimensions`. Cross-model federation rejected in Phase 154–158 (deferred to Phase 159). -- **Storage:** Semantic objects are permanent, content-addressed, immutable (like Git objects). - -**Scope:** - -1. **Database schema (v33 migration):** - - New `semantic_objects` table: `(object_hash, chunk_hash, blob_hash, embedding, summary, keywords, language, entities, structural_refs, profile_version, model_dimensions, signer, created_at)` - - New `sema_refs` table: `(name, target_sema_tree_hash, blob_count, created_at, updated_at)` - - New `sema_trees` table: `(tree_hash, parent_tree_hash, entries_jsonl, metadata_json)` — tracks semantic snapshot lineage - - Extend `chunk_embeddings`: add `summary`, `keywords`, `language` columns (nullable for backward compat) - -2. **CLI commands:** - - `gitsema sema push [--remote url] [--branch name]` — create semantic ref, push new semantic objects to remote - - `gitsema sema pull [--remote url] [--branch name]` — fetch semantic ref + missing semantic objects from remote - - `gitsema sema log [--ref name] [--graph]` — show semantic ref history (like `git log`) - - `gitsema sema diff [--format text|json|html]` — semantic diff (high-level concept changes between snapshots) - -3. **HTTP API:** - - `POST /api/v1/sema/push` — accept semantic tree + objects for storage - - `GET /api/v1/sema/pull?ref=` — fetch semantic tree + packfile of missing objects - - `GET /api/v1/sema/log` — semantic ref history - - `POST /api/v1/sema/diff` — compute semantic diff between two refs - -4. **MCP tools:** - - `sema_push()` — create/update semantic ref - - `sema_pull()` — fetch semantic ref and objects - - `sema_log()` — list semantic ref history - - `sema_diff()` — semantic diff between refs - -5. **Feature:** Semantic object enrichment (Layer 1 enhancement) - - Optional `--semantic-enrich` flag on `gitsema index start` (calls narrator LLM to generate `summary`, `keywords`, `entities` per chunk) - - Extends existing chunking pipeline: `chunk → embed → [enrich] → store` - - Backward compat: old indexes without semantic_objects table still index normally (opt-in feature) - - Summary generation: reuse narrator infrastructure from Phase 56 (`buildSemanticSummary()` or similar) - -6. **Tests:** - - Schema: v32 → v33 migration applies cleanly; new tables created - - Two-repo push/pull scenario: push from Repo A, pull into Repo B, verify object deduplication by `object_hash` - - Semantic diff: track concept-level changes between refs (added/removed/changed summaries) - - Backward compat: old indexes without semantic_objects still index; queries still work (semantic objects lazy-loaded) - - Cross-model rejection: `sema pull` from a peer with different model → error with helpful message - - Enrichment: `--semantic-enrich` generates summaries; summaries appear in MCP/HTTP results - -**Acceptance criteria:** -- Semantic refs created, stored persistently, survive session restarts -- `gitsema sema push` transfers only new semantic objects (content-addressed dedup verified via hash) -- `gitsema sema pull` correctly reconstructs remote semantic state -- `gitsema sema diff` shows high-level concept changes (not just raw vector diffs) -- `--semantic-enrich` produces meaningful summaries (manual QA on real repo) -- Cross-model federation rejected with clear error message -- All commands work via HTTP + MCP + CLI (parity) -- `pnpm build && pnpm test` clean; changesets added - -**Effort:** ~2.5 weeks (schema migration + enrichment pipeline adds complexity) -**Risk:** Medium (new tables + semantic enrichment pipeline; needs LLM integration testing) - -**Files (anticipated):** `src/core/db/schema.ts`, `src/core/db/sqlite.ts` (v33 migration), `src/core/federation/semanticRefs.ts`, `src/core/narrator/semanticEnrichment.ts`, `src/cli/commands/sema.ts`, `src/server/routes/federation/sema.ts`, `src/mcp/tools/federation.ts`, tests, `CLAUDE.md`, `.changeset/` - -**Design decisions deferred to Phase 159:** -- Cross-model federation (re-embedding queries for different model spaces) -- Trust/signatures (cryptographic verification of semantic objects) -- Semantic commit deltas (see Phase 158; focuses on static state here) - ---- - -### Phase 155 — Federation Discovery & Gossip - -**Design:** Covered in `docs/design/semantic-federation.md`. - -**Goal:** Enable repositories to discover and register with peers, forming a federated network of semantic services. - -**Scope:** - -1. **Peer registration & discovery:** - - `GET /api/v1/federation/info` — returns peer's semantic capabilities (centroid embeddings per cluster, model version, repo URL) - - `POST /api/v1/federation/gossip` — receive peer info, propagate to known peers - - `gitsema federation info` — show local federation metadata - - `gitsema federation peers [--list]` — list known peers + their semantic topics - -2. **Gossip protocol:** - - Simple rumor-spreading: when Peer A learns about Peer B, A propagates to C, C to D, etc. - - TTL on gossip entries (default 24 hours) — stale peers auto-expire - - Bounded peer list (default max 50 peers) — prevents broadcast storm - - Rate limiting on gossip (default 1 gossip/sec per peer pair) — prevents network saturation - -3. **Optional central registry (lightweight):** - - HTTP endpoint to list registered peers (optional, can be disabled) - - Repos can opt-in to publish to a central registry - - Registry serves as bootstrap source for P2P gossip, but isn't required - -4. **Semantic topics per peer:** - - On each `gitsema index start`, compute cluster centroids (Phase 21) - - Extract top-K clusters and their semantic topics (function labels from guide interpretation) - - Store in federation metadata + include in `GET /api/v1/federation/info` - - Used by Phase 156 routing logic to decide which peers to query - -5. **Tests:** - - 3-repo network: A ↔ B ↔ C, verify gossip reaches all nodes within 3 rounds - - Peer expiry: mark peer stale, verify it stops being returned after TTL - - Peer limit: add 100 peers, verify only top 50 (by relevance) are kept - -**Acceptance criteria:** -- Peer discovery works across 3+ repositories without manual configuration -- Gossip protocol converges within expected rounds (O(log N) proof not needed, just empirical validation) -- Peer info includes semantic topics (centroid embeddings + labels) -- `gitsema federation peers` shows current peer list with last-seen timestamps -- `pnpm build && pnpm test` clean - -**Effort:** ~1.5 weeks -**Risk:** Low (read-only; no data mutation) - -**Files (anticipated):** `src/core/federation/gossip.ts`, `src/cli/commands/federation.ts`, `src/server/routes/federation/discovery.ts`, `src/mcp/tools/federation.ts` (extended), tests, `.changeset/` - ---- - -### Phase 156 — Semantic Query Routing - -**Design:** Covered in `docs/design/semantic-federation.md`. - -**Goal:** Route queries intelligently to the most relevant peers based on semantic similarity, reducing broadcast queries and enabling users to choose how federated results are stored. - -**Key design decisions:** -- **Storage modes:** Support transient (no storage), cached (session scope), and imported (permanent) federation search results. -- **Same-model only:** Only query peers with matching `profile_version`; cross-model federation deferred to Phase 159. -- **Three-signal ranking:** Reuse existing three-signal ranking (Phase 41) for merged cross-peer results. - -**Scope:** - -1. **Semantic DHT routing table (`semanticDHT.ts`):** - - Build routing table from peer gossip + centroid embeddings (from Phase 155) - - For each known peer, store semantic topics (cluster centroids + labels) - - On each query, embed query and compute similarity to all peer centroids - - Select top-N peers (default N=5, configurable `--federation-peers `) above threshold (default 0.3) - - Filter by model match: only include peers with same `profile_version` as local index - -2. **Federated search CLI:** - - `gitsema federation search [--peers | --auto] [--top k] [--cache|--import|--no-cache] [--format text|json|html]` - - `--auto`: use gossip-discovered peers; `--peers`: explicit peer URLs - - Storage modes (mutually exclusive): - - `--cache` (default): store in temp `federation_cache` table (session scope), TTL = session - - `--import`: merge into permanent `semantic_objects` table (mark with `origin: "federation_imported"`) - - `--no-cache`: transient (show results, don't store) - - Fetch results from selected peers in parallel with timeout (default 5s/peer) - - Merge by `chunk_hash` (remove duplicates) - - Re-rank using three-signal ranking (vector similarity, recency, path relevance) - - Include provenance: show peer URL + model version for each result - -3. **HTTP API:** - - `POST /api/v1/federation/search` — accept query, storage mode (`cache|import|transient`), peer hints - - `GET /api/v1/federation/route?query=` — return routing decision (which peers selected + why) - - Streaming results as peers respond (don't wait for slowest) - - `X-Peer-Info` header on each result: `{url: "...", model: "...", response_time: ...}` - -4. **MCP tools:** - - `federation_search()` — query federated peers (supports storage mode selection) - - `federation_route()` — show routing decision (peers selected + reasoning) - -5. **Result merging & ranking:** - - De-duplicate by `chunk_hash` (if same chunk from multiple peers, keep highest-confidence result) - - Combine provenance metadata: `{peer_url: "...", peer_model: "...", local_origin: "..."}` - - Re-rank using three-signal: vector similarity (70%) + recency (20%) + path relevance (10%) - - Fallback: if peer unresponsive, drop and continue (fail-open) - -6. **Metadata tracking:** - - Extend `semantic_objects` to track `origin` (values: "local", "federation_temp", "federation_imported") + `peer_url` + `import_date` - - Cache table `federation_cache`: same schema as `semantic_objects`, but session-scoped (dropped on exit) - -7. **Tests:** - - Schema: `federation_cache` table creates/drops correctly - - 3-repo network: query "authentication", verify only auth-heavy repos selected - - Routing: `--federation-peers 3` limits to top-3 by similarity - - Storage modes: `--cache` doesn't pollute local index; `--import` persists; `--no-cache` returns 0 cached results - - Cross-model rejection: query Repo B (different model) → skipped with warning - - Result ranking: merge from 2 peers, verify no regressions vs. single-repo ranking - - Timeout: slow peer (latency > 5s) doesn't block others; partial results returned - - Provenance: results show peer URL + model; CLI and HTTP match - -**Acceptance criteria:** -- `gitsema federation search ` returns ranked results from multiple (same-model) peers -- Routing logic selects relevant peers; `--federation-peers ` respected -- Merged results de-duplicated by chunk_hash, re-ranked correctly -- `--cache` (default) doesn't persist results; `--import` does; `--no-cache` is transient -- Cross-model peers rejected with warning (not queried) -- Peer timeout doesn't block final result (fail-open) -- HTTP + MCP parity with CLI -- Provenance metadata visible in all formats (text, json, html) -- `pnpm build && pnpm test` clean - -**Effort:** ~3.5 weeks (DHT + caching + storage modes add complexity) -**Risk:** Medium (distributed ranking; needs careful testing on multi-repo setup; caching adds state management) - -**Files (anticipated):** `src/core/db/schema.ts` (federation_cache table), `src/core/federation/semanticDHT.ts`, `src/core/federation/federatedSearch.ts`, `src/cli/commands/federation.ts` (extended), `src/server/routes/federation/search.ts`, `src/mcp/tools/federation.ts` (extended), tests, `.changeset/` - -**Design decisions deferred to Phase 159:** -- Cross-model federation (re-embedding for different model spaces) -- Peer trust/signatures (verifying peer semantic objects) - ---- - -### Phase 157 — Semantic Packfiles - -**Design:** Covered in `docs/design/semantic-federation.md`. - -**Goal:** Enable efficient bulk transfer of semantic objects related to a query via a binary packfile format. - -**Scope:** - -1. **Packfile format (`semanticPackfile.ts`):** - - Binary format: `[count: varint][for each object: hash_len, hash, embedding, summary_len, summary, metadata][checksum: SHA-256]` - - Gzip compression by default - - Optional signing: include Ed25519 signature (for future Phase 155+ trust model) - - Fast serialization: stream-based, no full buffer load - -2. **CLI commands:** - - `gitsema sema pack --query [--output file.sema] [--sign]` — create packfile locally (for sharing) - - `gitsema sema fetch --query [--remote url] [--output file.sema]` — fetch packfile from peer - - `gitsema sema unpack [--input file.sema] [--merge]` — import packfile into local index (dedup by blob hash) - -3. **HTTP API:** - - `POST /api/v1/federation/pack` — client specifies query, server returns packfile - - `GET /api/v1/federation/fetch?packfile_id=` — retrieve pre-computed packfile (streaming) - -4. **MCP tools:** - - `sema_pack()` — create packfile - - `sema_fetch()` — fetch packfile from peer - - (No `sema_unpack()` on MCP — unpacking happens locally) - -5. **Packfile integration:** - - On `gitsema sema fetch --query `, automatically unpack + merge into local index - - Deduplication by blob hash: if local index already has the blob, skip re-storing - - Bandwidth savings: packfiles compress ~50%+ over individual fetch requests (tested empirically) - -6. **Tests:** - - Pack/unpack round-trip: create packfile, unpack, verify blob hashes match - - Compression: measure packfile size vs. raw objects (expect 50%+ reduction) - - Streaming: packfile for large query (1000s of objects) doesn't allocate full buffer - - Deduplication: fetch same packfile twice, verify second fetch doesn't re-store blobs - -**Acceptance criteria:** -- Packfiles serialize + deserialize correctly -- Compression reduces size by 40%+ (tunable via zlib level) -- `gitsema sema pack` + `gitsema sema fetch` work end-to-end -- Unpacking deduplicates by blob hash (second unpack adds 0 new blobs) -- Packfiles can be shared offline (email, S3, etc.) -- `pnpm build && pnpm test` clean - -**Effort:** ~2.5 weeks -**Risk:** Medium (new binary format; needs robust error handling + fuzzing) - -**Files (anticipated):** `src/core/federation/semanticPackfile.ts`, `src/cli/commands/sema.ts` (extended), `src/server/routes/federation/pack.ts`, `src/mcp/tools/federation.ts` (extended), tests, `.changeset/` - ---- - -### Phase 158 — Semantic Commits & Deltas - -**Design:** Covered in `docs/design/semantic-federation.md`. - -**Goal:** Track semantic changes per Git commit, enabling concept-level causality analysis and change-driven queries. - -**Scope:** - -1. **Database schema:** Add `semantic_commits` table (v34 migration): - - `semantic_commits(id, git_commit_hash, semantic_commit_hash, parent_semantic_commit_hash, summary, added_concepts_json, removed_concepts_json, changed_concepts_json, author, timestamp, provenance_json)` - - Added/removed/changed concepts include: concept text, embedding, confidence score, affected blob hashes - -2. **Semantic delta computation:** - - On each `gitsema index start`, compare current semantic state to previous state (via `sema_refs`) - - For each Git commit, compute: - - **Added concepts:** new embeddings not in parent commit - - **Removed concepts:** embeddings in parent but not current - - **Changed concepts:** embeddings in both but with large cosine distance (> threshold, default 0.3) - - Store deltas in `semantic_commits` with git commit hash linkage - -3. **CLI commands:** - - `gitsema semantic-commits [--ref] [--since ] [--until ] [--format text|json|html]` — show semantic commit log - - `gitsema semantic-blame [--file path] [--since ] [--format text|json]` — blame a concept by semantic change (not line change) - - `gitsema concept-lifecycle [--format text|json|html]` — show lifecycle stages of a concept (born → growing → mature → declining → dead) - -4. **HTTP API:** - - `GET /api/v1/semantic-commits` — fetch semantic commit log with filtering - - `POST /api/v1/semantic-blame` — semantic blame endpoint - - `GET /api/v1/concept-lifecycle` — concept lifecycle endpoint - -5. **MCP tools:** - - `semantic_commits()` — query semantic commit log - - `semantic_blame()` — blame by semantic change - - `concept_lifecycle()` — trace concept evolution stages - -6. **Queries enabled:** - - "Show me repos that recently improved OAuth" → search semantic commit summaries + concepts - - "Who introduced this security pattern?" → semantic blame on semantic commits - - "When did caching become stale?" → concept-lifecycle detection (declining stage) - - "What breaks if I remove this function?" → semantic blast radius + concept lifecycle - -7. **Tests:** - - Semantic commit creation on index: verify deltas computed correctly - - Concept lifecycle detection: manually create a multi-commit arc (add → grow → mature → decline), verify classification - - Semantic blame accuracy: compare semantic-blame results to `gitsema blame` (should show similar patterns) - -**Acceptance criteria:** -- Semantic commits created on every indexing run (one per Git commit) -- Deltas correctly identify added/removed/changed concepts -- Lifecycle detection classifies concept stages accurately (tested on synthetic + real repo) -- `gitsema semantic-blame ` produces meaningful results (compared to line-level blame) -- All commands work via HTTP + MCP + CLI -- `pnpm build && pnpm test` clean - -**Effort:** ~2 weeks -**Risk:** Medium (delta computation is complex; needs extensive validation on real repos with semantic drift) - -**Files (anticipated):** `src/core/db/schema.ts`, `src/core/db/sqlite.ts` (v34 migration), `src/core/federation/semanticCommits.ts`, `src/core/search/semanticBlame.ts`, `src/cli/commands/semanticCommits.ts`, `src/server/routes/federation/semanticCommits.ts`, `src/mcp/tools/federation.ts` (extended), tests, `CLAUDE.md`, `.changeset/` - ---- - -## Semantic Federation Summary - -**Phases 154–158 implement the three-layer federation architecture from `docs/design/semantic-federation.md`:** - -| Phase | Layer | Goal | -|---|---|---| -| 154 | Layer 1 + 2 | Semantic objects + snapshots; incremental sync | -| 155 | Layer 3 | Peer discovery & gossip protocol | -| 156 | Layer 3 | Intelligent query routing (DHT) | -| 157 | Layer 3 | Efficient bulk transfer (packfiles) | -| 158 | Layer 1 | Semantic deltas + concept causality | - -**Key insight:** These phases transform gitsema from a single-repository tool into a network of federated semantic services. By Phase 158, repositories can answer questions about their own knowledge, share insights with peers, and collectively answer complex semantic queries that no single repository could answer alone. - -**Post-Phase-158 roadmap** (future phases): -- **Phase 159:** Semantic signatures & trust model (Phase 155 follow-on) — enable signed semantic objects for supply-chain provenance -- **Phase 160:** Web UI for federation — dashboard showing peer network, popular topics, cross-repo insights -- **Phase 161:** Semantic LLM grounding — use federated semantic knowledge to improve LLM context in `guide` and `narrate` commands -- **Phase 162:** Integration with package managers — expose gitsema federation API as a plugin for npm/pip/cargo semantic search - HTTP: `/search`, `/first-seen` text rendering includes `[blob:...]` prefix ✅ - HTML: search results show "Blob Hash" column or `blob:` prefix ✅ - Tests: new hash-labeling tests added ✅ @@ -6663,6 +6396,25 @@ URL-guard helper), `src/server/routes/{narrator,guide}.ts`, --- +## Withdrawn: Semantic Federation (Phases 154–158) + +Phases 154–158 (semantic objects & refs, federation discovery & gossip, +semantic query routing, semantic packfiles, semantic commits & deltas) were +scheduled here on 2026-07-08 from `docs/design/semantic-federation.md` and +**withdrawn on 2026-07-09, before any implementation began**. The design was +judged premature: it centered on a speculative P2P/gossip network rather than +the actual driver (agent-scale read load on hosted repo endpoints), and it was +written without reconciling against the shipped multi-tenant auth (Phases +122–126), storage-abstraction (101–103), and locked-model-set (128) layers. + +The design doc is retained and marked withdrawn. The salvageable kernels — +chunk-level semantic enrichment, a SKOS-style concept vocabulary, and +prebuilt-index distribution for agent-scale serving — moved back to +`docs/feature-ideas.md`. Phase numbers 154–158 are retired to keep git history +unambiguous; the next scheduled phase starts at 159. + +--- + ## Deployment scenarios & usage envisioning The architecture of gitsema supports three distinct deployment scenarios, each with different operational models and target users. This section clarifies the intended usage patterns and the infrastructure requirements for each. diff --git a/docs/design/concept-vocabulary.md b/docs/design/concept-vocabulary.md new file mode 100644 index 0000000..d8bd996 --- /dev/null +++ b/docs/design/concept-vocabulary.md @@ -0,0 +1,720 @@ +# Design Doc — SKOS-Style Concept Vocabulary (model-independent semantic layer) + +> Status: **draft — refined from `docs/feature-ideas.md`, not yet scheduled.** +> Target phases: **unassigned** (labeled C1–C5 below; real numbers are assigned +> when this is promoted into `docs/PLAN.md`). +> Scope: a lightweight, curated, **model-independent** controlled vocabulary +> (SKOS-subset semantics in plain relational tables) mapped onto gitsema's +> existing artifacts — blobs, chunks, symbols, clusters — with a faceted query +> surface at CLI/MCP/HTTP parity. +> +> Provenance: salvaged from the withdrawn semantic-federation design's +> keyword/SKOS thread (`docs/design/semantic-federation.md`, ⛔ withdrawn — +> background reading only). This document stands alone and does **not** depend +> on any part of that design. + +This document nails down the **conceptual model**, the **SKOS subset**, the +**schema**, the **curation workflow**, the **assignment mechanics and +confidence model**, the **query surface**, and the **phase boundaries** before +any code lands. PLAN.md phase entries should link here rather than restating +the design (same convention as `docs/knowledge-graph.md`). + +--- + +## 1. Motivation & relationship to what exists + +Everything semantic in gitsema today lives in **one embedding model's vector +space**. Vectors from different models are incomparable — which is exactly why +the withdrawn federation design had to forbid cross-model exchange and defer +"cross-space similarity" to open research, and why the locked-model-set work +(Phase 128, `repos.profile_name`) pins a repo to the profile it was first +indexed with. Meanwhile, every human-readable *label* gitsema produces is +ad-hoc and unstable: + +- **Cluster labels** (`blob_clusters.label`) are derived from term frequency + + path prefixes (`src/core/search/clustering/clustering.ts`, + `labelEnhancer.ts` TF-IDF). They are **not stable across runs**: `kMeansInit` + seeds with `Math.random()`, and each `computeClusters` run does + `DELETE FROM cluster_assignments; DELETE FROM blob_clusters` and re-inserts + under fresh autoincrement IDs. The same codebase clustered twice yields + different cluster identities and possibly different labels. +- **"Concept" is not a persisted entity anywhere.** The word appears zero + times in `src/core/db/schema.ts`. `concept-evolution`, `dead-concepts`, + `lifecycle`, `change-points`, `bisect`, `author` all define a "concept" as + an **ephemeral query string** embedded at call time and compared by cosine + similarity. Nothing survives the call. +- **Keywords** (`blob_clusters.top_keywords`, FTS tokens) are freeform strings + with no relations, no synonyms, no hierarchy. + +What is missing is a **model-independent semantic layer**: a small, curated +vocabulary of *concepts* with SKOS-ish structure (`broader` / `narrower` / +`related`, preferred + alternative labels), mapped onto the artifacts gitsema +already indexes. The defining property: **concepts are identified by curated +keys and labels, never by vectors**, so two repos indexed with *different +embedding models* — or the same repo re-indexed under a new model — still +interoperate at the concept layer ("both have material under +`auth > token-refresh`"). The cross-model problem that is unsolvable at the +vector layer simply dissolves one level up. This is the principled foundation +for any future cross-repo/cross-model feature. + +Concretely, the vocabulary enables: + +- **Faceted search** — `gitsema search "refresh handler" --concept auth/jwt` + restricts results to material assigned to a concept (and its narrower + descendants). +- **Stable topic labels** — clusters come and go per run; a cluster can be + *mapped to* concepts, giving a stable name that survives re-clustering and + re-indexing. +- **Concept-level diffs** — "between v1.0 and v2.0, material under + `payments/webhooks` doubled; `legacy/soap` disappeared" (a future phase + builds this on assignments × `blob_commits`, the same join `first-seen` + uses). +- **A curated map of what the codebase is about** — browsable, exportable, + versionable alongside the code. + +### Constraints this design must respect (from `CLAUDE.md`) + +1. **Git is the source of truth** for code facts. The vocabulary is *human/LLM + knowledge about* the code — the one legitimately non-Git-derived layer — + but assignments to code always target Git-native identities (blob hashes). +2. **Blob-first.** Assignments pivot on `blob_hash` (and blob-derived chunk / + symbol occurrence identities), never on paths or commits. A useful + consequence: blob-level assignments are **content-addressed** — the same + file content in two repos (or at two paths) carries its concept assignments + with it for free. +3. **Streaming.** Automated assignment passes iterate the index in bounded + batches; never load all content into memory. +4. **CLI-first; MCP/HTTP thin.** All logic in `src/core/concepts/`; MCP tools + and HTTP routes wrap the same functions. +5. **Storage seam.** The vocabulary lives in the **relational** store, behind + a new `ConceptStore` interface on `StorageProfile` — following the + `GraphStore` precedent, including the fail-loud + `UnsupportedConceptStore` for non-relational profiles + (cf. `src/core/storage/unsupportedGraphStore.ts`). + +--- + +## 2. Grounding: the SKOS subset we adopt (and what we skip) + +The design borrows real [SKOS](https://www.w3.org/TR/skos-reference/) +semantics so the vocabulary is interoperable-by-construction with external +taxonomies, but the implementation is **a set of SQLite tables, not an RDF +triple store**. The mapping: + +| SKOS construct | Adopted as | Notes | +|---|---|---| +| `skos:ConceptScheme` | `concept_schemes` row | A named vocabulary. One default scheme per index; imported schemes are additional rows. Scheme `uri` gives global identity for cross-repo interop. | +| `skos:Concept` | `concepts` row | Identified by `(scheme, slug)` — see §3.1. | +| `skos:prefLabel` | `concepts.pref_label` | Exactly one per concept (we do not model per-language labels in v1 — see §10). | +| `skos:altLabel`, `skos:hiddenLabel` | `concept_labels` rows | Alternative/hidden labels power **lexical assignment** (§5.2) — the model-independent signal. | +| `skos:notation` | `concepts.slug` | A stable, scheme-unique machine key. | +| `skos:definition`, `skos:scopeNote` | `concepts.definition`, `concepts.scope_note` | Free text; also feeds the centroid assigner's embedding text (§5.3). | +| `skos:broader` / `skos:narrower` | `concept_relations` rows, **stored one direction only** (`broader`) | `narrower` is derived as the inverse, per SKOS convention. Hierarchy is a DAG (a concept may have multiple broader concepts); cycles are rejected on write. | +| `skos:related` | `concept_relations` rows (`related`) | Symmetric (stored once, queried both ways). Enforce SKOS's disjointness: `related` may not connect a concept to its own broader-transitive ancestor/descendant. | +| `skos:hasTopConcept` | derived | A scheme's top concepts = concepts with no `broader` relation in that scheme. Not stored. | +| `skos:exactMatch`, `skos:closeMatch` | `concept_relations` rows (`exact_match`, `close_match`) | **Cross-scheme mapping** — the interop mechanism between an imported external taxonomy and the local scheme, and between two repos' schemes. Deferred to phase C5 but in the schema from day one. | +| Deprecation (`owl:deprecated` idiom) | `concepts.status = 'deprecated'` + `concepts.replaced_by` | Concepts are never hard-deleted — see §7 (scheme evolution). | + +**Explicitly skipped** (lightweight by design): RDF serialization as the +native format, `skos:Collection`/`OrderedCollection`, per-language label tags, +`broadMatch`/`narrowMatch`/`relatedMatch`, SKOS-XL. A best-effort Turtle +import/export can be added later without schema changes (§10); the native +interchange format is JSON (§6.1). + +--- + +## 3. Conceptual model & options considered + +### 3.1 Concept identity + +The central design question, exactly parallel to the knowledge-graph's +node-key decision: concepts need **stable identity** that survives renames, +re-parenting, re-indexing, and travel between repos. + +**Decision: `concept_key = ":"`, with the slug +flat (not hierarchical).** e.g. `main:jwt-validation`, `owasp:a07`. + +- The slug does **not** encode the broader-chain. A display path like + `auth > token-refresh > jwt-validation` is *derived* at read time from + `broader` relations. Re-parenting a concept (the most common curation edit) + therefore never changes its identity, its assignments, or its history — + this is what makes "scheme evolution without invalidating history" (§7) + cheap. +- `pref_label` is freely editable; the slug is fixed at creation (changing it + means deprecate + replace, §7). +- CLI/MCP accept friendly references — a bare slug, a label, or a `/`-joined + path (`auth/jwt` walks labels down the hierarchy) — and resolve them to the + key; ambiguity is an error listing candidates. + +### 3.2 Where concepts live relative to existing structures — options + +**Option A — concepts as `graph_nodes` + `edges`** (reuse the knowledge-graph +tables; `concept:` node kind, `broader` edge type, `about` edges to +file/symbol nodes). +*Rejected.* `graph_nodes`/`edges` are **truncate-and-rebuilt wholesale** by +`gitsema graph build` (schema.ts comment: "Rebuilt wholesale… like +`blob_clusters`"). Curated vocabulary must never be wiped by a derived-data +rebuild. Mixing durable curated rows into a recomputable table would force +every rebuild to carve around them — a standing bug factory. The graph also +carries structural semantics (`calls`, `imports`) that don't apply. + +**Option B — a real RDF/triple store or embedded quad-store.** +*Rejected.* Massive dependency for a vocabulary that will typically hold tens +to low hundreds of concepts. The storage seam is relational; SKOS's subset +maps to 4 small tables losslessly for our needs. + +**Option C — flat tags** (a `tags` table, no relations, no schemes). +*Rejected as an endpoint, but it is the v1 kernel.* Flat labels can't express +`auth > token-refresh`, synonyms, or cross-scheme mapping — which are exactly +the properties that make the layer model-independent *and useful*. The +incremental path (C1 ships vocabulary + manual tagging before any automation) +gives us the flat-tag utility on the way. + +**Option D (chosen) — dedicated relational SKOS-subset tables** with a +**hard durability split**, mirroring the repo's established +immutable-vs-recomputable discipline: + +| Layer | Tables | Lifecycle | Precedent | +|---|---|---|---| +| **Curated vocabulary** | `concept_schemes`, `concepts`, `concept_labels`, `concept_relations` | Durable. Edited by explicit commands; survives re-index, re-cluster, `graph build`, model changes. Never truncate-rebuilt. | `embed_config` + `settings` (durable config-like rows) | +| **Assignments** | `concept_assignments` | Split by method: `manual` rows are durable; automated rows (`lexical`, `centroid`, `llm`, `cluster`) are recomputable and replaced per-method by re-runs. | `graph_nodes`/`edges` (recomputable, confidence-carrying), `blob_clusters` (replaced per run) | + +### 3.3 Relationship to clustering and the knowledge graph + +- **Clusters are evidence, not concepts.** A cluster is an unstable, + model-dependent grouping; a concept is a stable, model-independent name. + The bridge (phase C4): map each cluster to concepts by lexical overlap + between the cluster's `top_keywords`/label (already TF-IDF-enhanced via + `enhanceClusters`) and concept labels, then (a) show concept names in + `clusters` output — stable labels across re-clustering — and (b) optionally + propagate the cluster's blob assignments to the matched concept at + discounted confidence (§5.5). +- **The graph stays structural.** No concept rows in `graph_nodes`. Read-time + enrichment (e.g. showing concepts of a node's underlying blob in + `graph relate` output) is a later, purely presentational join. If a future + phase wants concept edges materialized for traversal, that is an explicit + derived projection *into* the graph rebuild, never the source of truth. + +--- + +## 4. Schema + +Two migrations, following the one-file-per-version pattern +(`src/core/db/migrations/NNN_name.ts`, appended to `runner.ts`, bump +`CURRENT_SCHEMA_VERSION`, mirror in `schema.ts` + `initTables`). Numbers below +assume v32 is still current when C1 lands; renumber if other work ships first +(note: the also-unscheduled `docs/semantic-enrichment-plan.md` pencils in v33 +for its `enrichments` table — whichever design lands first takes v33). + +### 4.1 Vocabulary tables — migration v32 → v33 (phase C1) + +```ts +concept_schemes = sqliteTable('concept_schemes', { + slug: text('slug').primaryKey(), // 'main', 'owasp' + title: text('title').notNull(), + /** Global identity for cross-repo interop (skos:ConceptScheme URI). Optional; */ + /** defaults to a tag-URI derived from slug on export. */ + uri: text('uri').unique(), + /** 'manual' | 'imported' — provenance of the scheme itself. */ + source: text('source').notNull().default('manual'), + createdAt: integer('created_at').notNull(), + updatedAt: integer('updated_at').notNull(), +}) + +concepts = sqliteTable('concepts', { + /** ':' — the stable key (§3.1). */ + conceptKey: text('concept_key').primaryKey(), + schemeSlug: text('scheme_slug').notNull().references(() => conceptSchemes.slug), + slug: text('slug').notNull(), // skos:notation; flat, scheme-unique + prefLabel: text('pref_label').notNull(), // skos:prefLabel + definition: text('definition'), // skos:definition + scopeNote: text('scope_note'), // skos:scopeNote + /** 'proposed' | 'approved' | 'deprecated' — curation state (§6). */ + status: text('status').notNull().default('approved'), + /** 'manual' | 'llm' | 'imported' — who created it (§6). */ + source: text('source').notNull().default('manual'), + /** Deprecation forwarding (§7): concept_key of the replacement, if any. */ + replacedBy: text('replaced_by'), + createdAt: integer('created_at').notNull(), + updatedAt: integer('updated_at').notNull(), +}, (t) => ({ schemeSlugUnique: uniqueIndex(...).on(t.schemeSlug, t.slug) })) + +concept_labels = sqliteTable('concept_labels', { + id: integer('id').primaryKey({ autoIncrement: true }), + conceptKey: text('concept_key').notNull().references(() => concepts.conceptKey), + label: text('label').notNull(), + /** 'alt' | 'hidden' (prefLabel lives on the concept row). */ + kind: text('kind').notNull().default('alt'), +}, (t) => ({ perConcept: uniqueIndex(...).on(t.conceptKey, t.label) })) + +concept_relations = sqliteTable('concept_relations', { + srcKey: text('src_key').notNull().references(() => concepts.conceptKey), + dstKey: text('dst_key').notNull().references(() => concepts.conceptKey), + /** 'broader' (src's broader is dst; narrower derived as inverse) | + 'related' (symmetric; stored once with srcKey < dstKey) | + 'exact_match' | 'close_match' (cross-scheme mapping, §2). */ + relation: text('relation').notNull(), +}, (t) => ({ pk: primaryKey({ columns: [t.srcKey, t.dstKey, t.relation] }) })) +``` + +Write-time integrity (enforced in `ConceptStore`, not SQL triggers): +`broader` must stay acyclic; `related`/matches must not duplicate a +broader-transitive link; relations may not target `deprecated` concepts +(existing relations to a concept being deprecated are retargeted or dropped +per §7). + +### 4.2 Assignments — migration v33 → v34 (phase C2) + +```ts +concept_assignments = sqliteTable('concept_assignments', { + id: integer('id').primaryKey({ autoIncrement: true }), + conceptKey: text('concept_key').notNull().references(() => concepts.conceptKey), + /** 'blob' | 'chunk' | 'symbol' | 'cluster' — what is being tagged. */ + targetKind: text('target_kind').notNull(), + /** Path-free, content-addressed identity (§5.1): + blob: + chunk: #- + symbol: ## (occurrence identity, Phase 105) + cluster: @ (run-scoped; C4 only) */ + targetKey: text('target_key').notNull(), + /** 'manual' | 'lexical' | 'centroid' | 'llm' | 'cluster' (§5). */ + method: text('method').notNull(), + confidence: real('confidence').notNull(), // 0..1 (§5.6) + /** Embedding model that produced this assignment — REQUIRED for 'centroid' + (it is model-dependent), null for model-independent methods. */ + model: text('model'), + createdAt: integer('created_at').notNull(), +}, (t) => ({ + uniq: uniqueIndex(...).on(t.conceptKey, t.targetKind, t.targetKey, t.method), + byTarget: index(...).on(t.targetKind, t.targetKey), + byConcept: index(...).on(t.conceptKey, t.method), +})) +``` + +No FK from `target_key` to `blobs` — assignments may be imported ahead of +content (e.g. a shared scheme file tags blob hashes this clone hasn't indexed +yet), and `index gc` must not cascade into curated data. `concepts doctor` +(folded into `index doctor --extended`) reports dangling targets instead. + +### 4.3 Storage seam — `ConceptStore` + +A fifth store on `StorageProfile` (alongside `metadata` / `vectors` / `fts` / +`graph`), **relational-only**, exactly following the `GraphStore` precedent: + +```ts +interface ConceptStore { + // vocabulary (durable) + upsertScheme(s: ConceptSchemeRecord): Promise + createConcept(c: ConceptRecord): Promise + updateConcept(key: string, patch: Partial): Promise + getConcept(key: string): Promise + resolveConcept(ref: string): Promise // slug | label | path → candidates + listConcepts(opts?: { scheme?: string; status?: ConceptStatus }): Promise + setLabels(key: string, labels: ConceptLabelRecord[]): Promise + relate(src: string, dst: string, relation: ConceptRelation): Promise // validates §4.1 integrity + unrelate(src: string, dst: string, relation: ConceptRelation): Promise + /** Narrower-transitive closure of `key`, depth-capped (§8.1 faceted-search expansion). */ + narrowerClosure(key: string, maxDepth?: number): Promise + + // assignments (manual durable; automated replaced per method) + assign(a: ConceptAssignmentRecord): Promise + unassign(conceptKey: string, targetKind: TargetKind, targetKey: string, method?: AssignMethod): Promise + replaceMethodAssignments(method: AssignMethod, rows: ConceptAssignmentRecord[], opts?: { conceptKey?: string; model?: string }): Promise + assignmentsFor(targetKind: TargetKind, targetKeys: string[]): Promise> + targetsFor(conceptKeys: string[], opts?: { kinds?: TargetKind[]; minConfidence?: number }): Promise + coverage(opts?: { scheme?: string }): Promise +} +``` + +- **sqlite** implements it in C1/C2. +- **postgres** implements it in C5 (plain relational DDL, no pgvector needed). +- **qdrant profile**: `UnsupportedConceptStore` throwing + `"concept vocabulary requires a relational backend"` on every method — + same fail-loud rule as `UnsupportedGraphStore` (review9 §4). Note the + qdrant profile does have a Postgres metadata companion; once the Postgres + `ConceptStore` exists (C5), the qdrant profile wires concepts through that + companion and the unsupported stub disappears for it. +- `storage migrate` gains the five tables in its copy path (C5); `storage + info`/`doctor` report concept availability per backend. + +--- + +## 5. Assignment mechanics & confidence + +### 5.1 Targets are content-addressed and path-free + +All assignment targets use blob-intrinsic identities (§4.2 `target_key`), +mirroring the knowledge-graph occurrence discipline. Consequences: + +- Assignments survive renames/moves for free (same blob hash). +- An edited file gets a new blob hash and **starts unassigned** — automated + assigners re-cover it on the next run; manual assignments are re-attached + via a helper (`concepts tag --carry-forward ` copies the previous + blob-version's manual tags to HEAD's blob after review) rather than + silently guessed. +- History queries come free: joining assignments × `blob_commits` yields + "when did material under this concept first appear" with the exact + mechanism `first-seen` already uses. +- **Blob-level (`targetKind='blob'`) is the primary kind in C2**; `chunk` and + `symbol` are in the schema from day one but only populated once an assigner + produces them (LLM/centroid at chunk granularity is a natural C3 extension). + +### 5.2 Method: `lexical` (model-independent — the backbone) + +Matches concept labels against index-side text signals. Entirely +string-based, therefore **valid across embedding models and repos** — this is +the method that realizes the feature's defining property, so it ships first +(C2) and is the default `concepts assign` mode. + +Signals, reusing existing machinery: + +1. **Path tokens** — `splitIdentifier` + `normalizeToken` from + `labelEnhancer.ts` over each blob's paths. +2. **FTS content** — BM25 lookups (`FtsStore.search`) per concept label + (pref + alt; `hidden` labels participate in matching but are never + displayed, per SKOS). +3. **Cluster keywords** — `blob_clusters.top_keywords` for the blob's current + cluster (when a clustering run exists). + +Score = weighted combination (path hit > FTS hit > cluster-keyword hit), +normalized to 0..1; assignments below `concepts.assignThreshold` (default +0.35) are dropped. Multi-word labels must match as phrases in FTS and as +adjacent tokens in paths. Runs are **streaming** (batch over blobs, bounded +memory) and **replace prior `lexical` rows wholesale** +(`replaceMethodAssignments`), like a cluster run. + +### 5.3 Method: `centroid` (model-dependent, explicitly marked) + +Embeds each concept's text (`pref_label + altLabels + definition`) with the +**active text model** and assigns blobs whose embedding cosine similarity +exceeds the threshold (via the existing `VectorStore.search` path). +Confidence = calibrated similarity. **The assignment row records `model`** — +these rows are only meaningful within that model's space, and the query layer +ignores centroid rows whose model doesn't match the active profile. The +*concept itself* stays model-independent; only this evidence layer is scoped. +Ships in C3. + +### 5.4 Method: `llm` (optional, safe-by-default) + +Batch classification through the existing narrator plumbing +(`resolveNarrator.ts` configs in `embed_config` kind rows, active selection in +`settings`): "given this scheme's concepts (labels + definitions) and this +chunk, which concepts apply?" Follows every established narrator convention: + +- **Safe-by-default**: no configured narrator model → the command reports + "LLM assignment requires a configured narrator model" and exits cleanly + (`createDisabledProvider` pattern). Never a hard failure. +- **Redaction is mandatory**: all outbound content passes `redact.ts` + (`redact`/`redactAll`), same as `narrate`/`explain`. +- Confidence = LLM-reported, **capped at 0.9** (only `manual` reaches 1.0). +- Cost-bounded: `--max-targets ` and concept/paths filters; resumable by + skipping (concept, target, method='llm') rows that already exist. + +Ships in C3, after the model-independent backbone is proven. + +### 5.5 Method: `cluster` (bridge, C4) + +For each cluster in the latest run: lexically match its +`top_keywords` + label against concept labels; on a match above threshold, +(a) record the cluster→concept mapping +(`targetKind='cluster'`, `targetKey='@'`) so `clusters` +output can display stable concept names, and (b) optionally +(`--propagate`) assign the cluster's member blobs to the concept at +`0.6 × match-score` confidence. Cluster assignments are wiped with each +re-cluster (they reference run-scoped IDs); propagated blob rows are +replaced on each bridge run like other automated methods. + +### 5.6 Confidence & aggregation model + +- One row per `(concept, target, method)`; methods accumulate as independent + evidence rather than overwriting each other. +- **Effective confidence** at query time = `max` over the target's valid rows + (centroid rows filtered by active model, §5.3). Shown with its winning + method in output (`0.82 via lexical`), so users can calibrate trust — + the same transparency rule as graph edge confidence. +- Method priors: `manual` = 1.0 · `llm` ≤ 0.9 · `centroid` ≤ ~0.9 (calibrated + cosine) · `lexical` ≤ 0.8 · `cluster` ≤ 0.6. +- Query-side floor: `concepts.minConfidence` (default 0.5) — an assignment + below the floor exists as evidence but doesn't make a blob answer for the + concept in filtered search. + +--- + +## 6. Curation model + +**Decision: hybrid — three sources with provenance, human approval gating +automated proposals.** Every concept row carries `source` +(`manual` | `llm` | `imported`) and `status` +(`proposed` | `approved` | `deprecated`). + +1. **Manual (primary path, C1).** `gitsema concepts add/label/relate/…` + create `approved` concepts directly. The vocabulary is small (tens of + concepts) and high-leverage — hand-curation is the realistic default, and + it works with zero LLM configuration. +2. **LLM-proposed (C3).** `gitsema concepts propose` feeds cluster keyword + digests + top paths through the narrator (redacted, safe-by-default, as + §5.4) and inserts results as `status='proposed'`. Proposed concepts are + visible in `concepts list --proposed` but **excluded from search + filtering and assignment runs** until a human runs + `concepts approve ` (or `reject`, which deletes — proposals carry no + history worth preserving). `concepts.autoApprove=true` opts out of the + gate for throwaway/experimental use. +3. **Imported (C1 for JSON; Turtle later).** `gitsema concepts import + ` loads a scheme file (§6.1) as `source='imported'`, + `status='approved'` (an external taxonomy is presumed curated). Re-import + **upserts by `(scheme, slug)`** and prints an added/updated/now-missing + diff; missing concepts are flagged, not auto-deprecated. + +### 6.1 Interchange format + +Native format: a **JSON scheme file** with a documented 1:1 SKOS mapping +(kept lightweight; no RDF dependency): + +```jsonc +{ + "scheme": { "slug": "main", "title": "gitsema main vocabulary", + "uri": "https://example.com/schemes/main" }, + "concepts": [ + { "slug": "jwt-validation", "prefLabel": "JWT validation", + "altLabels": ["jwt verify", "token validation"], + "definition": "Verification of JSON Web Token signatures and claims.", + "broader": ["token-refresh"], + "related": ["session-management"] } + ] +} +``` + +`concepts export [--scheme s] [--include-assignments]` writes it; +`--include-assignments` adds blob-hash-keyed assignment rows, which transfer +meaningfully to any repo sharing content (content-addressing, §5.1). The file +is deliberately git-friendly — teams can commit it (e.g. +`.gitsema/concepts.json` by convention) and treat the vocabulary as reviewed +code. This file, plus scheme URIs and `exact_match`/`close_match` mappings, +is the entire cross-repo interop story in v1: **no network protocol, no +federation** — a scheme travels as a file. + +--- + +## 7. Scheme evolution over time + +Rules, all following the "never invalidate history" requirement: + +- **Rename** — `pref_label` and labels are freely mutable; the key never + changes. No assignment impact. +- **Re-parent** — edit `broader` relations; identity unaffected (§3.1). +- **Deprecate / merge** — `concepts deprecate [--replaced-by ]` + sets `status='deprecated'` + `replaced_by`. Merging A into B = + deprecate A with `--replaced-by B`. Assignments to A are **left in place** + (they are historical fact); the query layer follows `replaced_by` chains + (cycle-safe, depth-capped), so `--concept B` transparently includes + material assigned to A. Deprecated concepts are hidden from default + listings, excluded from new assignment runs, and warn when used directly. +- **Split** — create the new narrower concepts, re-run assigners (their + labels now attract the material), optionally re-tag manual rows, then + deprecate the parent or keep it as the broader hub. No special machinery. +- **Slug change** — not supported in place; it's create-new + deprecate-old + with `replaced_by`, preserving both histories. +- **Hard delete** — only `concepts reject` on `proposed` rows, and a + `concepts gc --deprecated --before ` escape hatch that refuses to run + while assignments reference the concept unless `--force`. + +--- + +## 8. Query surface (CLI / MCP / HTTP parity) + +Per `docs/parity.md` conventions (and CLAUDE.md's "parity over response-shape +stability" rule), each phase ships its surface across CLI + MCP + HTTP +together; guide tools + `interpretations.ts` + `pnpm gen:skill` follow in C4 +(the `docsSync` test enforces the pairing once guide entries exist). + +### 8.1 CLI — new `concepts` command group + +Registered as `src/cli/register/concepts.ts` (`registerConcepts(program)` +called from `all.ts`; entries added to `COMMAND_GROUPS` under a new +"Concepts" help bucket). + +| Command | Phase | Description | +|---|---|---| +| `concepts list [--scheme s] [--tree] [--proposed]` | C1 | List/browse concepts; `--tree` renders the broader hierarchy | +| `concepts show ` | C1 | One concept: labels, relations, definition, (C2+) assignment counts | +| `concepts add --label [--scheme s] [--broader ref] [--definition d]` | C1 | Create an approved concept | +| `concepts label --alt \| --hidden \| --remove ` | C1 | Manage alt/hidden labels | +| `concepts relate --as broader\|related\|exact-match\|close-match` / `unrelate` | C1 | Manage relations (integrity-checked) | +| `concepts deprecate [--replaced-by ref2]` | C1 | Deprecation/merge (§7) | +| `concepts import ` / `concepts export [--out f] [--include-assignments]` | C1 | JSON scheme interchange (§6.1) | +| `concepts tag [--carry-forward]` / `untag` | C2 | Manual assignment (resolves a path at HEAD to its blob hash) | +| `concepts assign [--method lexical\|centroid\|llm\|cluster] [--concept ref] [--dry-run]` | C2 (lexical) / C3 (centroid, llm) / C4 (cluster) | Run an automated assigner; replaces that method's rows | +| `concepts of ` | C2 | Reverse lookup: which concepts cover this file/blob | +| `concepts coverage [--scheme s]` | C2 | Vocabulary health: per-concept target counts, % of HEAD blobs covered, unassigned top clusters | +| `concepts propose [--scheme s]` / `approve ` / `reject ` | C3 | LLM proposal workflow (§6) | +| `concepts diff ` | C4 | Concept-level diff between git refs (assignments × `blob_commits`) | +| `search … --concept [--no-concept-expand]` | C2 | Faceted search: restrict to targets of the concept **and its narrower-transitive closure** (expansion on by default, depth-capped like `MAX_GRAPH_TRAVERSAL_DEPTH`; effective-confidence ≥ `concepts.minConfidence`). Implemented as a blob-hash filter into the existing `vectorSearch` options — same mechanism as `--branch`. | + +`first-seen` gains `--concept` alongside `search` in C2 (it shares the search +pipeline). Broader flag rollout across `evolution`/`dead-concepts`/etc. is +left to the parity sweep in C4. + +### 8.2 MCP tools — `src/mcp/tools/concepts.ts` + +`registerConceptsTools(server)` wired in `src/mcp/server.ts`: + +| Tool | Phase | Mirrors | +|---|---|---| +| `concept_list` | C1 | `concepts list` (scheme/status filters, tree flag) | +| `concept_show` | C1 | `concepts show` | +| `concept_of` | C2 | `concepts of` | +| `concept_coverage` | C2 | `concepts coverage` | +| `concept_assign` | C2/C3 | `concepts assign` (method param) | +| `concept_diff` | C4 | `concepts diff` | +| `semantic_search` gains optional `concept` + `concept_expand` params | C2 | `search --concept` | + +Vocabulary *writes* (add/relate/deprecate/import) stay CLI-only in v1 — +curation is a deliberate human act; agents get read + assignment-run access. +(Guide tools mirror the read set in C4, with `TOOL_INTERPRETATIONS` entries +and regenerated skill per the docsSync contract.) + +### 8.3 HTTP routes — `src/server/routes/concepts.ts` + +`conceptsRouter()` mounted at `/api/v1/concepts` behind the standard +`authMiddleware` + `repoSessionMiddleware` chain: + +| Route | Phase | +|---|---| +| `GET /api/v1/concepts` · `GET /api/v1/concepts/:key` | C1 | +| `GET /api/v1/concepts/:key/targets` · `GET /api/v1/concepts/of/:blobHash` · `GET /api/v1/concepts/coverage` | C2 | +| `POST /api/v1/search` accepts `concept` + `conceptExpand` | C2 | +| `POST /api/v1/concepts/assign` (method-scoped re-run; write-guarded like other mutating routes) | C3 | +| `GET /api/v1/concepts/diff` | C4 | + +OpenAPI (`routes/openapi.ts`) and `docs/parity.md` matrix rows updated in the +same phase as each route — the parity doc gains a **Concepts** section with +the standard CLI/REPL/LSP/Guide/MCP/HTTP columns (REPL and LSP: `—` by +design; LSP hover enrichment is a possible later nicety, §10). + +### 8.4 Config keys + +`configManager.ts` `ALL_KEYS` additions: `concepts.scheme` (default scheme +slug, default `main`), `concepts.assignThreshold` (0.35), +`concepts.minConfidence` (0.5), `concepts.expandNarrower` (true), +`concepts.autoApprove` (false). Env mirrors (`GITSEMA_CONCEPTS_*`) via +`ENV_KEY_MAP` for the first three. + +--- + +## 9. Phased implementation plan + +Sized like existing PLAN.md phases; each ends with tests, `features.md` + +README + parity.md updates, and a changeset (per CLAUDE.md). C1+C2 alone +deliver a coherent, useful feature (curated vocabulary + model-independent +tagging + faceted search); C3–C5 are each independently shippable. + +| Phase | Title | Schema | Deliverable | Effort | +|---|---|---|---|---| +| **C1** | Vocabulary core | v33 | `concept_schemes`/`concepts`/`concept_labels`/`concept_relations`; `ConceptStore` seam (sqlite + fail-loud stub); CLI `concepts list/show/add/label/relate/unrelate/deprecate/import/export`; JSON interchange; integrity rules (§4.1); MCP `concept_list`/`concept_show`; HTTP GET routes. | S–M | +| **C2** | Assignments & faceted search | v34 | `concept_assignments`; manual `tag`/`untag`/`of`; **lexical assigner** (streaming, labelEnhancer reuse); confidence/aggregation model (§5.6); `--concept` on `search`/`first-seen` with narrower-expansion; `coverage`; MCP/HTTP parity for all of it. | M | +| **C3** | Semantic assigners & LLM curation | — | `centroid` assigner (model-recorded, model-filtered at query); `llm` assigner + `propose/approve/reject` workflow (narrator plumbing, `redact.ts`, safe-by-default, cost bounds); `POST /concepts/assign`. | M | +| **C4** | Bridges, diffs & agent surface | — | Cluster↔concept bridge (§5.5) + concept names in `clusters` output; `concepts diff `; guide tools + `interpretations.ts` entries + `pnpm gen:skill`; `--concept` parity sweep over remaining query-string commands; `index doctor --extended` concept checks. | M | +| **C5** | Backend & interop completion | — | Postgres `ConceptStore` (qdrant profile routes through its Postgres companion); `storage migrate`/`info`/`doctor` coverage; scheme URIs + `exact_match`/`close_match` workflows documented; `multi_repo_search` accepts `concept` (resolved per-repo by scheme-URI + slug across the per-repo DBs); shared-scheme-file conventions. | M | + +Total: a medium–large track (~5 phases), consistent with the feature-ideas +estimate; no networking, no new daemon, no new binary dependencies. + +--- + +## 10. Remaining open questions (deliberately deferred) + +1. **Turtle/RDF import-export** — worth a best-effort `skos:Concept` subset + reader for real external taxonomies (OWASP, org-internal SKOS files)? + Needs a user with an actual RDF file in hand; JSON covers the designed + workflows. No schema impact either way. +2. **Per-language labels** (`prefLabel@lang`) — skipped in v1; adding a + `lang` column to `concept_labels` later is a trivial migration. Decide + when a non-English-vocabulary user appears. +3. **Chunk/symbol-granular automated assignment** — the schema supports it + (§4.2); whether C3's LLM assigner should classify at chunk granularity by + default (better precision, higher cost) needs a cost measurement on a real + repo first. +4. **Concept-aware LSP hover** — showing a file's concepts in the hover card + is cheap once C2 exists, but LSP surface changes belong to an LSP-track + decision, not this design. +5. **Materializing `about` edges into the structural graph** — only if a + future traversal use-case demands it (§3.3); presentational joins first. +6. **Interplay with Chunk-Level Semantic Enrichment** (see below) — if that + idea ships, its per-chunk keywords become one more lexical signal and its + keyword field could hold concept keys; nothing here depends on it. + +### Relationship to the "Chunk-Level Semantic Enrichment" idea + +This design **stands alone**: the lexical backbone (§5.2) draws on paths, FTS +content, and cluster keywords — all of which exist today. If enrichment ships +later, its LLM-extracted per-chunk keywords slot in as an additional (and +better-targeted) lexical signal, and its open "keyword normalization" design +gap gets a ready answer: normalize enrichment keywords **into concept +references** where a match exists. The dependency direction is +enrichment → (optionally feeds) → concepts, never the reverse. + +--- + +## 11. Decisions taken autonomously (pending user review) + +This refinement ran non-interactively; every product/scope call the +refine-idea flow would normally ask about was resolved from codebase evidence +and recorded here. Each is revisable before C1 is scheduled. + +1. **Curation model = hybrid with human approval gating LLM proposals** + (§6). *Rationale:* mirrors the repo's safe-by-default LLM posture + (`createDisabledProvider`, evidence-only `narrate`); manual curation must + work with zero LLM config since gitsema's core never requires an LLM; + `concepts.autoApprove` preserves the fully-automated option. +2. **Dedicated durable tables, not graph-node reuse and not an RDF store** + (§3.2). *Rationale:* `graph_nodes`/`edges` and `blob_clusters` are + truncate-and-rebuilt (verified in schema comments and + `clustering.ts`'s DELETE-then-insert transaction) — curated data cannot + live there; an RDF store violates the lightweight/SQLite constraint. +3. **Flat slugs; hierarchy only in relations; display paths derived** (§3.1). + *Rationale:* re-parenting is the most common curation edit and must not + change identity — directly serves the "scheme evolution without + invalidating history" design gap. +4. **Content-addressed, path-free assignment targets** (§5.1). *Rationale:* + CLAUDE.md's blob-first constraint; matches the knowledge-graph occurrence + discipline; makes assignments portable across repos sharing content and + correct across renames. +5. **Lexical assignment is the backbone and ships before any + embedding/LLM-based assigner** (§5.2, phase order). *Rationale:* it is the + only method that is model-independent, which is the feature's defining + property; it reuses proven machinery (`labelEnhancer.ts`, `FtsStore`). +6. **Centroid assignments record their model and are filtered by active + model at query time** (§5.3). *Rationale:* keeps the model-independence + guarantee honest — model-dependent evidence is allowed but explicitly + scoped, consistent with the locked-model-set direction (Phase 128). +7. **Confidence = per-method rows, `max` aggregation, method priors, query + floor** (§5.6). *Rationale:* follows the graph-edge confidence precedent + (surface confidence so users calibrate trust); preserving independent + evidence rows keeps re-runs idempotent per method. +8. **`ConceptStore` is relational-only with a fail-loud stub; sqlite first, + Postgres in C5** (§4.3). *Rationale:* exact `GraphStore`/ + `UnsupportedGraphStore` precedent (review9 §4 "fail loud, don't no-op"). +9. **Query surface = new `concepts` group + `--concept` facet on + `search`/`first-seen`, with MCP/HTTP shipped in the same phase as each + CLI feature; vocabulary writes CLI-only in v1** (§8). *Rationale:* the + Design Gap demanded parity from day one (parity.md §4 priority); gating + writes matches the "curation is a human act" model and keeps the MCP + surface read-mostly like other tool families. +10. **Scheme evolution via SKOS-idiomatic deprecation + `replaced_by` + forwarding; assignments never rewritten** (§7). *Rationale:* the + deprecations-registry culture of this repo applied to data: history is + preserved, queries follow forwarding chains, deletion is an explicit + escape hatch. +11. **Cross-repo interop = scheme files + scheme URIs + match relations; no + network protocol** (§6.1, C5). *Rationale:* the federation design was + withdrawn precisely for network speculation; multi-repo is + one-DB-per-repo (verified: no `repo_id` on content tables), so a + file-travelling scheme + per-repo resolution in `multi_repo_search` is + the smallest true interop story. +12. **JSON as the native interchange format; RDF/Turtle deferred** (§6.1, + open question 1). *Rationale:* lightweight-implementation constraint from + the idea entry; JSON preserves the SKOS semantics losslessly for the + adopted subset. +13. **Doc filed as `docs/design/concept-vocabulary.md`.** *Rationale:* the + skill's convention for a self-contained feature, and it sits beside the + withdrawn `docs/design/semantic-federation.md` it was salvaged from. diff --git a/docs/design/semantic-federation.md b/docs/design/semantic-federation.md index 77db5e8..63d2acf 100644 --- a/docs/design/semantic-federation.md +++ b/docs/design/semantic-federation.md @@ -1,9 +1,20 @@ # Semantic Federation: Distributing Knowledge Across Repositories -**Status:** Design document (proposed phases 154–158) -**Last updated:** 2026-07-08 +**Status:** ⛔ **WITHDRAWN (2026-07-09)** — removed from `docs/PLAN.md` before any implementation began. Retained for reference only; do not implement from this document. +**Last updated:** 2026-07-09 **Initiated by:** ChatGPT feedback on gitsema architecture +> **Why withdrawn:** The design centered on a speculative P2P/gossip network +> rather than the problem that actually motivated it (agent-scale read load +> concentrating on hosted repo endpoints), and it was written without +> reconciling against gitsema's shipped multi-tenant auth (Phases 122–126), +> storage abstraction (101–103), and locked-model-set (128) layers. It also +> collided with existing features (`semantic-blame`, `concept-lifecycle`, +> `index export/import`). The salvageable kernels — chunk-level semantic +> enrichment, a SKOS-style concept vocabulary, and prebuilt-index +> distribution for agent-scale serving — are tracked as fresh ideas in +> [`docs/feature-ideas.md`](../feature-ideas.md). + --- ## Executive Summary diff --git a/docs/feature-ideas.md b/docs/feature-ideas.md index bb91476..ec41981 100644 --- a/docs/feature-ideas.md +++ b/docs/feature-ideas.md @@ -2,7 +2,7 @@ This document tracks upcoming feature ideas that are **not yet in active development** (not in `PLAN.md`) and haven't been **fully designed** (no design file). It's a staging area for "what now?" questions and medium-term product direction. -**Last updated:** 2026-07-08 (semantic federation now fully designed → PLAN.md Phases 154–158) +**Last updated:** 2026-07-10 (SKOS-Style Concept Vocabulary refined into `docs/design/concept-vocabulary.md`; same day: Chunk-Level Semantic Enrichment refined into `docs/semantic-enrichment-plan.md`; previously: Prebuilt Index Distribution refined into `docs/prebuilt-index-distribution-plan.md`, semantic federation withdrawn from PLAN.md with salvaged kernels re-captured here) **Audience:** Developers considering next phases; product planning > **Note 1:** As of 2026-07-02, the LSP/MCP remote-delegation foundation @@ -12,11 +12,16 @@ This document tracks upcoming feature ideas that are **not yet in active develop > Those sections were removed from here; this file now tracks only what's > genuinely still just an idea. > -> **Note 2:** As of 2026-07-08, semantic federation (distributed semantic -> knowledge, peer-to-peer query routing, semantic packfiles) is now fully -> designed in `docs/design/semantic-federation.md` and scheduled as Phases -> 154–158 in `docs/PLAN.md`. The design is comprehensive; implementation -> begins after Phase 153 completes. +> **Note 2:** Semantic federation (distributed semantic knowledge, +> peer-to-peer query routing, semantic packfiles) was designed in +> `docs/design/semantic-federation.md` and briefly scheduled as PLAN.md +> Phases 154–158 (2026-07-08), then **withdrawn on 2026-07-09 before any +> implementation** — the design was speculative (P2P/gossip network) and +> unreconciled with the shipped auth/storage/model-profile layers. The +> design doc is retained, marked withdrawn. Its salvageable kernels are +> re-captured below as three independent ideas: *Prebuilt Index +> Distribution*, *Chunk-Level Semantic Enrichment*, and *SKOS-Style Concept +> Vocabulary*. --- @@ -719,6 +724,79 @@ No design committed yet. The rough shape, sketched for discussion: --- +## Prebuilt Index Distribution ("index once, serve many" for agent-scale load) + +*Salvaged from the withdrawn semantic-federation design (2026-07-09) and re-framed around its actual motivating problem.* + +**Refined into:** see [`docs/prebuilt-index-distribution-plan.md`](prebuilt-index-distribution-plan.md) +(2026-07-09) — full design: bundle manifest v2 with `embed_config`-derived +provenance, sqlite delta bundles, `index attach`/`index publish` with a +layered HTTPS resolution ladder (config URL template → gitsema server → +GitHub rolling release), server-side bundle routes under the Phase 122–126 +grant model, and a zero-server CI loop. Includes a "Decisions taken +autonomously (pending user review)" section covering every Design Gap the +original entry listed. Not yet scheduled in `PLAN.md`. + +--- + +## Chunk-Level Semantic Enrichment (summaries, keywords, entities) + +*Salvaged Layer-1 kernel of the withdrawn semantic-federation design — valuable purely locally, no networking required.* + +**Refined into:** see [`docs/semantic-enrichment-plan.md`](semantic-enrichment-plan.md) (2026-07-10). + +One-paragraph summary of the refined design: an opt-in LLM-generated metadata +layer (`summary`, `keywords`, optional `entities`) for the index's retrieval +units — whole-file blobs *and* chunks — stored in a new `enrichments` table +(schema v33) that **references** existing blob/chunk/embedding rows (no vector +duplication), written only after mandatory **inbound redaction** of LLM output +via `redact.ts` (stored summaries travel in bundles and server responses), +generated via the existing narrator model configs (no new `embed_config` +kind), and surfaced additively through `SearchResult.summary`/`keywords` +across CLI/REPL/MCP/HTTP/guide. Trigger model: `index start --semantic-enrich` +plus an `index enrich` backfill subcommand; lazy enrich-on-search was +explicitly rejected (read paths stay network-free). All four original Design +Gaps (cost controls, backfill, schema/storage-abstraction coverage, keyword +normalization) are resolved in the plan's §6 "Decisions taken autonomously +(pending user review)" table — review that section before phase-planning. +Three implementation phases (E1 core+storage+backfill, E2 index-time flag, +E3 surfacing & parity), not yet scheduled in `PLAN.md`. + +--- + +## SKOS-Style Concept Vocabulary (model-independent semantic layer) + +*Salvaged from the withdrawn semantic-federation design's keyword/SKOS thread.* + +**Refined into:** see [`docs/design/concept-vocabulary.md`](design/concept-vocabulary.md) (2026-07-10). + +One-paragraph summary of the refined design: a lightweight, curated, +**model-independent** controlled vocabulary — a real SKOS subset +(`broader`/`related` relations, pref/alt/hidden labels, concept schemes, +`exact_match`/`close_match` cross-scheme mapping, deprecation with +`replaced_by` forwarding) in four durable relational tables (schema v33) +plus a recomputable `concept_assignments` table (v34), behind a new +fail-loud `ConceptStore` on the storage seam. Assignments target +content-addressed, path-free identities (blob/chunk/symbol occurrence +keys) with per-method confidence rows: `manual` (1.0), **`lexical` as the +model-independent backbone** (labelEnhancer + FTS + cluster keywords), +`centroid` (model-recorded and model-filtered), `llm` (redacted, +safe-by-default), and a cluster↔concept bridge that gives clusters stable +names. Query surface: a new `concepts` CLI group + `--concept` faceting on +`search`/`first-seen` with narrower-transitive expansion, shipped at +MCP/HTTP parity per phase; cross-repo interop is a git-friendly JSON scheme +file + scheme URIs — no network protocol. All five original Design Gaps +(curation model, storage, assignment mechanics/confidence, query surface, +scheme evolution) are resolved in the plan's §11 "Decisions taken +autonomously (pending user review)" section — review it before +phase-planning. Five implementation phases (C1 vocabulary core, C2 +assignments + faceted search, C3 semantic/LLM assigners, C4 bridges + +diffs + agent surface, C5 Postgres + interop), not yet scheduled in +`PLAN.md`. Stands alone; Chunk-Level Semantic Enrichment keywords slot in +as an extra lexical signal if that design ships. + +--- + ## Related Issues & Documents - **Parity tracking:** See `docs/parity.md` for tool availability across interfaces @@ -736,6 +814,6 @@ No design committed yet. The rough shape, sketched for discussion: --- -**Document Status:** ✓ Current (2026-06-22) +**Document Status:** ✓ Current (2026-07-10) **Next Review:** When Semahub or the plugin API work begins **Maintainer:** jsilvanus@gmail.com diff --git a/docs/features.md b/docs/features.md index 2a60e03..403b2cc 100644 --- a/docs/features.md +++ b/docs/features.md @@ -304,6 +304,8 @@ Start with `gitsema tools serve [--port n] [--key token] [--ui]`. Authentication: optional Bearer token via `--key ` / `GITSEMA_SERVE_KEY`. Per-repo scoped tokens can be minted with `gitsema repos token add ` and are stored as **SHA-256 hashes** at rest (review7 §4.1) — the plaintext is never persisted in the database. +**Git argument-injection hardening (Phase 150 / review11 §2.1–§3.2):** every `git` call site that takes a caller-influenced ref (commit hash, branch, tag, range) routes through a shared `runGit()` helper (`src/core/git/runGit.ts`) that rejects refs failing `isSafeGitRange()` (leading `-` and non-ref characters) *before* spawning git and always inserts git's `--end-of-options` separator, so a value like `--output=/path` can never be reparsed as a git flag. This closes the network-reachable arbitrary-file-write that was reachable through `semantic_bisect`/`triage` when a "ref" began with `-`. + ### Identity & credentials core (Phase 122) User accounts (`gitsema auth create-user `, local DB bootstrap; org/role-gated @@ -342,10 +344,28 @@ create/list`, `gitsema repos grant/grants/revoke/move-to-org` — all operator-t commands that read/write the local server DB directly (`getRawDb()`), the same pattern as `gitsema auth create-user` and `gitsema repos token *`, not the remote-HTTP-client pattern used by `gitsema auth login/logout/whoami/token`. Deliberate scope limits (see -`docs/PLAN.md` Phase 123 for the full list): the ~16 pre-existing analysis/search/ -evolution/graph HTTP routes are not yet retrofitted to enforce `resolveUserRepoAccess`; -newly created repos do not default into the creator's personal org; and there is no -backfill migration granting pre-existing users a personal org retroactively. +`docs/PLAN.md` Phase 123 for the full list): newly created repos do not default into the +creator's personal org; and there is no backfill migration granting pre-existing users a +personal org retroactively. (The read-route enforcement gap flagged here in Phase 123 is +now closed by Phase 151, below.) + +### Read-route repo authorization gate (Phase 151) + +`repoAuthMiddleware` (`src/server/middleware/repoAuth.ts`) runs immediately after +`repoSessionMiddleware` on every data route that returns repo content (search, analysis, +evolution, graph, insights, protocol, watch, projections, narrate/explain, guide). In +**multi-tenant mode** (`GITSEMA_MULTI_TENANT`, defaulting to `GITSEMA_SERVE_KEY` +presence) a request naming a `repoId` must satisfy +`roleSatisfies(resolveUserRepoAccess(userId, repoId), 'read')` unless the repo is +`visibility: 'public'` — otherwise the route returns **403**. This closes review11 §2.2 +("any authenticated user, or anyone on an open server, can read any repo by naming its +`repoId`"): the grant model the auth track ships now actually gates the read surface it +was built to protect. Two credentials bypass the grant check because they already imply +access — the global `GITSEMA_SERVE_KEY` (operator/admin) and a legacy per-repo scoped +`repo_tokens` token (already scoped to its one repo). A default open single-dev server +(no key, no flag) is a no-op — no new 403s. **Repo-level only:** per-branch grant +filtering (rewriting the routes' `branch: string` into a per-user granted-branch set) is +deferred to a follow-on phase. ### SSO/OIDC identity linking (Phase 124) @@ -527,6 +547,17 @@ denied server-wide ("lock to none"). (if supplied) short-circuits before any other lookup — explicit model id, model name, active DB selection, disabled. BYOK always resolves to an HTTP/`chattydeer`-backed provider. +- **SSRF guard (Phase 152 / review11 §3.1):** because the server issues the + request to the caller-supplied `byok.http_url`, that URL is validated before + the provider is constructed — the scheme must be `http`/`https`, and hosts + resolving to loopback (`127.0.0.0/8`, `::1`), link-local (`169.254.0.0/16` + incl. the `169.254.169.254` cloud-metadata IP, `fe80::/10`), or RFC-1918 + private ranges (`10/8`, `172.16/12`, `192.168/16`) are **rejected by + default** (400 `ByokUrlValidationError`). Operators re-permit specific + internal hosts (e.g. a local model server) via `GITSEMA_BYOK_ALLOW_HOSTS`, + a comma-separated host/CIDR allowlist (default empty). This is a behavior + change for anyone previously pointing BYOK at `localhost`/a private IP — + add the host to the allowlist. ### Persistent server-side repo storage diff --git a/docs/parity.md b/docs/parity.md index 3901598..4ee66df 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -23,7 +23,7 @@ tools are available — that's §1. | Guide | Local (in-process agentic loop) | — | n/a | n/a | | `tools mcp` | stdio (spawned as a local child process) | `--websocket ` (non-standard MCP transport, kept for forward compatibility); `--http ` (Streamable HTTP — MCP's actual standard network transport, Phase 117) | `--key`/`GITSEMA_WEBSOCKET_KEY` (websocket); `--key`/`GITSEMA_MCP_HTTP_KEY` (http) | `maxPayload` 10MB + max 100 connections/sessions on both network modes (review10 §3) | | `tools lsp` | stdio (spawned as a local child process) | `--websocket ` | `--key`/`GITSEMA_WEBSOCKET_KEY` (`--websocket`) | max 100 connections; `maxPayload` 10MB | -| `tools serve` | Always a server (no local-only mode) | HTTP (REST-ish routes) | `--key`/`GITSEMA_SERVE_KEY` | `express.json({ limit })` body-size cap | +| `tools serve` | Always a server (no local-only mode) | HTTP (REST-ish routes) | `--key`/`GITSEMA_SERVE_KEY`; optional per-user session/API-key credentials (Phase 122) with a repo-level read-grant gate on data routes in multi-tenant mode (`GITSEMA_MULTI_TENANT`, Phase 151) | `express.json({ limit })` body-size cap | All network modes print a startup warning if bound to a non-loopback address with no key configured. (The unauthenticated `tools lsp --tcp` raw-TCP @@ -343,7 +343,7 @@ This section documents all flags used across CLI commands, their consistency, an | `--chunker` | — | enum | file | `index start` | Chunking strategy: `file`, `function`, or `fixed` | | `--lens` | — | enum | hybrid | `blast-radius`, `relate`, `similar`, `hotspots` | Structural/semantic lens toggle: `structural`, `semantic`, `hybrid`; MCP `graph_relate`/`graph_similar`/`blast_radius` (`lens`) and HTTP `POST /graph/{relate,similar,blast-radius}` (`lens`) mirror this since Phase 147 | | `--weight-structural` | — | float | lens-dependent | `blast-radius`, `relate`, `similar`, `hotspots` | Structural-signal weight override for the active `--lens` (`addLensOption`); **no-op on `hotspots`** specifically — its risk score is an unweighted geometric mean with no weighting hook, on both CLI and `POST /graph/hotspots` (accepted there for flag-surface parity only, Phase 139); not accepted on `POST /graph/{relate,similar,blast-radius}` (Phase 147) — those routes mirror only `lens`, matching the CLI's own no-op-if-set-elsewhere behavior for this flag on non-`hotspots` commands | -| `--depth` | — | int | varies | `deps`, `graph callers`, `graph callees`, `graph neighbors`, `graph path`, `blast-radius` | Traversal depth for graph commands; MCP `call_graph`/`graph_neighbors`/`deps`/`blast_radius` (`depth`) and HTTP `POST /graph/{callers,callees,neighbors,deps,blast-radius}` (`depth`) mirror this since Phase 108/147 | +| `--depth` | — | int | varies | `deps`, `graph callers`, `graph callees`, `graph neighbors`, `graph path`, `blast-radius` | Traversal depth for graph commands; MCP `call_graph`/`graph_neighbors`/`deps`/`blast_radius` (`depth`) and HTTP `POST /graph/{callers,callees,neighbors,deps,blast-radius}` (`depth`) mirror this since Phase 108/147. **Network-exposed `deps`/`blast_radius` `depth` is upper-bounded** at `MAX_GRAPH_DEPTH_REQUEST` (64) on HTTP + MCP (review11 §3.3); `callers`/`callees`/`neighbors` are already server-side-clamped to `MAX_GRAPH_TRAVERSAL_DEPTH` (3). The CLI (in-process) is unbounded. | | `--repos` | — | string | — | `search`, `first-seen` | Comma-separated repo IDs for multi-repo mode; MCP `semantic_search`/`first_seen` (`repos: string[]`) and HTTP `POST /search`/`POST /search/first-seen` (`repos: string[]`) mirror this since Phase 138 — results are merged into the primary-DB result list rather than returned separately, matching CLI's `--repos` behavior. `POST /analysis/multi-repo-search` remains as a thin deprecated alias (see `docs/deprecations.md`) | | `--threshold` | — | float | varies | `code-review`, `cross-repo-similarity`, `policy-check` | Similarity/distance threshold for matching | | `--base` | — | ref | varies | `regression-gate`, `code-review`, `ci-diff` | Base git ref to compare from | diff --git a/docs/prebuilt-index-distribution-plan.md b/docs/prebuilt-index-distribution-plan.md new file mode 100644 index 0000000..42dde25 --- /dev/null +++ b/docs/prebuilt-index-distribution-plan.md @@ -0,0 +1,534 @@ +# Prebuilt Index Distribution — "index once, serve many" + +**Status:** Draft (refined 2026-07-09 from the `docs/feature-ideas.md` entry of the +same name; not yet scheduled in `docs/PLAN.md`) +**Scope:** Publishable, content-addressed index artifacts keyed by +(repo, commit, embed config); delta bundles for incremental catch-up; +`index publish` / `index attach` CLI; boring-HTTPS resolution (gitsema server, +GitHub Releases, plain URL); server-side bundle serving under the existing +auth/grant model. **No P2P, no gossip, no DHT, no new network protocol, no new +daemon.** +**Origin:** Salvaged kernel of the withdrawn semantic-federation design +(`docs/design/semantic-federation.md`, ⛔ withdrawn 2026-07-09). This doc keeps +that design's one sound observation — a gitsema index is a deterministic, +shareable artifact — and discards its distribution mechanism entirely in favor +of static-artifact conventions over HTTPS. + +--- + +## 1. Motivation + +Swarms of coding agents are becoming the dominant read load on code-hosting +infrastructure. For gitsema specifically, every agent/machine/CI job that wants +semantic access to a repo today has exactly two options: + +1. **Re-embed the entire history locally** (`gitsema index start`) — expensive, + slow, and N× duplicated embedding spend for byte-identical content. For a + mid-size repo this is minutes-to-hours of GPU/API cost that some other + consumer already paid. +2. **Query one shared `gitsema tools serve` instance** — load concentrates on a + single query endpoint, which is precisely the "agents hammering one hosting + endpoint" problem that motivated the federation idea in the first place. + +gitsema's content-addressed model makes a third option obviously available: an +index for (repo, commit, embed config) is **deterministic** — two consumers +computing it independently produce semantically identical artifacts. There is +no reason for it to ever be computed twice. The design constraint "immutable +embeddings — a blob is embedded exactly once" (CLAUDE.md) currently stops at +the machine boundary; this design extends it across machines: *a blob is +embedded exactly once, anywhere.* + +Reads then scale the way static files scale — by copying artifacts through +caches and CDNs — not by adding query capacity. A consumer downloads the +prebuilt index once, then runs every subsequent query locally at zero marginal +cost to the publisher. + +Nearly all the pieces already exist: + +- `index export` / `index import` tar.gz bundles (Phase 54, + `src/cli/commands/bundleIndex.ts`) +- public repo sharing with attach-as-reader grants (Phases 126–127, + `docs/public-repo-sharing-plan.md`) +- embed-config provenance (`embed_config` table + `computeConfigHash()`, + `src/core/indexing/provenance.ts`) and profile pinning + (`repos.profile_name`, Phase 128, `docs/locked-model-set-plan.md`) +- multi-tenant auth: users/api_keys/repo_grants/audit_log (Phases 122–126) +- the server-side repo registry (`GITSEMA_DATA_DIR`, + `repos//{repo,index.db}`, `src/core/indexing/repoRegistry.ts`) + +What is missing is **distribution**: a publish/resolve/fetch convention that +ties them together. + +--- + +## 2. Current state (verified against source) + +### 2.1 The Phase 54 bundle (`src/cli/commands/bundleIndex.ts`) + +- `gitsema index export --out ` tars `.gitsema/index.db` plus any + `*.usearch` / `*.map.json` VSS files, adds a `manifest.json` with + `{schemaVersion, checksums (sha256 per file), exportedAfter, exportedAt}`, + and gzips the result. +- `gitsema index import --in ` extracts, validates per-file checksums, + **overwrites** `.gitsema/index.db` wholesale (`writeFileSync`), then opens + the DB to run migrations. +- **The `--after`/`--since` "incremental slice" is provenance-only.** Despite + the file-header comment ("filters blobs by first_seen timestamp"), the + export path archives the *entire* `index.db` unconditionally; `afterTs` is + merely recorded in the manifest and echoed on import. There is no actual + filtering and no merge-on-import — a Phase 54 "incremental" bundle is a full + bundle with a label. This is both a documentation bug today and the gap the + delta mechanism in this design fills. +- The manifest carries **no embed-config provenance, no repo identity, and no + commit tip** — an importer cannot tell which repo, which commit range, or + which embedding model a bundle is for without opening the DB. + +### 2.2 Embed-config provenance (Phase 80+/128) + +- `embed_config` rows record `{provider, model, codeModel, dimensions, + chunker, windowSize, overlap}`; `computeConfigHash()` is a SHA-256 over + exactly those seven keys, sorted (`provenance.ts:30-42`). One index DB may + legitimately contain several configs (multi-model DBs are supported; + `checkConfigCompatibility()` only rejects same-model/different-dimensions). +- **Quantization is *not* part of `configHash`.** It is recorded per embedding + row (`quantized`, `quant_min`, `quant_scale` columns on `embeddings`, + `chunk_embeddings`, `symbol_embeddings`, `module_embeddings`). Search + dequantizes at read time, so mixed-quantization reads work; it matters only + when a consumer wants to *continue indexing* into an imported DB. +- Server-side, `repos.profile_name` (v32) pins each repo to one named + embedding profile at first index; profiles are operator-defined + (`GITSEMA_EMBEDDING_PROFILES`, `src/core/embedding/profiles.ts`). + +### 2.3 Identity, auth, sharing + +- Server-side repo identity: `normalizeRepoUrl()` + + `sha256(normalizedUrl).slice(0,16)` → `repoId` (`repoRegistry.ts:195-218`). + Deterministic across deployments given the same URL. +- Phases 122–126 give the server users, hashed api_keys/sessions, + `repo_grants` (`read`/`write`/`owner`, optional branch glob), `orgs`, an + `audit_log`, and `repos.visibility` (`private`/`public`) with an + attach-as-reader auto-grant flow (`role: 'reader'`, `source: 'auto-public'`) + for public repos. + +### 2.4 Storage abstraction (Phases 101–103) + +- All index access goes through `MetadataStore`/`VectorStore`/`FtsStore` + (`src/core/storage/types.ts`). The bundle is inherently a **sqlite-file + artifact**; `storage migrate` today supports sqlite → {sqlite, postgres, + qdrant} only, which conveniently matches the natural import path for + non-sqlite deployments (import bundle to sqlite, migrate). + +### 2.5 What the withdrawn federation design got right + +Its "Layer 2" observed that semantic state can ride Git's own transport +(`refs/sema/*` refs, packfile-like bundles fetchable offline via "email, S3, +etc."). This design keeps the *artifact* framing and the *git-native option* +(as a deferred transport, §7), and drops the peer network, query routing, and +gossip entirely. + +--- + +## 3. Conceptual model & options considered + +Three independent axes, each with real alternatives: + +### Axis A — Artifact format + +| Option | Description | Verdict | +|---|---|---| +| **A1. Evolve the Phase 54 tar.gz bundle** (manifest v2) | Keep `tar.gz(sqlite db + VSS files + manifest.json)`; enrich the manifest with provenance; add a *delta* variant whose payload is a small sqlite DB containing only new rows | ⭐ chosen | +| A2. New manifest-led custom format (row-oriented binary/JSONL "semantic packfile") | Fresh serialization of blobs/embeddings/chunks à la federation's packfiles | Rejected: reinvents serialization sqlite already does; every reader/writer needs new code; loses free migrations-on-open | +| A3. Ship the raw `.gitsema/` directory (rsync/scp convention) | No format at all | Rejected: no integrity, no provenance, no delta story, WAL-file footguns | + +A1 wins because sqlite **is** the serialization: `INSERT OR IGNORE` over +content-addressed, blob-keyed rows makes delta application idempotent and +order-independent for free, and `openDatabaseAt()` already gives +schema-version validation + migrations on import. A delta bundle is just a +second sqlite file with the same schema and only the new rows — no new parser +anywhere. + +### Axis B — Resolution/discovery (how `attach` finds a bundle) + +| Option | Description | Verdict | +|---|---|---| +| **B1. Layered convention: explicit URL → config template → well-known locations** | `--from ` wins; else `bundles.url` config; else try the configured gitsema server's `/bundles//`, else the repo's forge-release convention | ⭐ chosen | +| B2. Central registry service | A new hosted "bundle registry" mapping repo URL → bundle URL | Rejected: a new service to run is exactly what this design exists to avoid; Semahub can layer one later | +| B3. Git ref namespace only (`refs/gitsema/bundles/*`) | Bundle rides the git remote itself, inheriting its auth/hosting | Deferred (§7): elegant (zero extra infra, auth for free) but pushes multi-hundred-MB binary objects into git remotes, which some forges throttle/forbid; not needed for v1 | + +Discovery within a location uses one small **bundle-index file** +(`gitsema-bundles.json`): a manifest-of-manifests listing available bundles +(full + deltas) with their tips, base commits, embed configs, sizes, and +checksums. `attach` fetches this one file, picks the best chain (see §4.4), +then downloads only what it needs. Static, cacheable, CDN-friendly. + +### Axis C — Who serves bytes + +| Option | Description | Verdict | +|---|---|---| +| **C1. Any static HTTPS host** (GitHub Releases, S3, artifact stores) | Publisher uploads bundle + bundle-index; consumers GET | ⭐ chosen as the baseline — zero-server loop | +| **C2. `gitsema tools serve` as a bundle origin** | New `GET /bundles/:repoId[/:name]` routes streaming the artifacts, gated by existing grants | ⭐ also chosen — additive; turns the server from "answers every query" into "hands out the index once per consumer" | +| C3. Peer-to-peer swap between consumers | — | Rejected: withdrawn with federation; out of scope permanently for this design | + +C1 and C2 are complements, not competitors: the format and bundle-index are +identical in both; C2 just adds an authenticated origin for private repos. + +--- + +## 4. Chosen direction + +### 4.1 Bundle manifest v2 + +`manifest.json` inside the tar.gz grows from Phase 54's four fields to: + +```jsonc +{ + "bundleFormat": 2, // 1 = Phase 54 (absent field ⇒ 1) + "kind": "full", // "full" | "delta" + "schemaVersion": 32, // sqlite schema version, as today + "checksums": { "index.db": "…" }, // sha256 per file, as today + "createdAt": 1720512000, + + // Repo identity (Git stays the source of truth: identity is the + // normalized remote URL + a commit hash; nothing here restates Git data) + "repo": { + "normalizedUrl": "github.com/acme/widget", + "repoId": "3fa1b2c4d5e6f708" // deriveRepoId(normalizedUrl), repoRegistry.ts + }, + "tipCommit": "abc123…", // history is fully indexed up to here + "baseCommit": null, // delta only: consumer must already have ≥ this + + // Embed-config provenance: verbatim embed_config rows (one DB may hold + // several models). configHash is computeConfigHash() over the same seven + // keys as today — no new version string is invented. Quantization is not + // part of configHash (see §2.2) so it is carried alongside, per config. + "embedConfigs": [ + { + "configHash": "…", + "provider": "http", "model": "text-embedding-3-small", + "codeModel": null, "dimensions": 1536, + "chunker": "file", "windowSize": null, "overlap": null, + "quantized": false + } + ], + "profileName": "default", // server-published bundles: repos.profile_name; null for local exports + + "signature": null // reserved; see §7 (deferred) +} +``` + +Import behavior by `bundleFormat`: +- Absent/`1`: legacy path, unchanged (overwrite + migrate). +- `2` + `kind: "full"`: verify checksums; refuse if the target `.gitsema/` + already has an index whose `embed_config` set conflicts under + `checkConfigCompatibility()` semantics (same model, different dimensions) + unless `--force`; then overwrite + migrate, as today. +- `2` + `kind: "delta"`: see §4.3. + +### 4.2 What a full bundle is + +Unchanged from Phase 54 physically: the whole `index.db` + VSS sidecars. A +full bundle for a server-hosted repo is naturally single-profile (server DBs +are per-repo with a pinned `profile_name`); a locally exported bundle may +carry several `embedConfigs`, and the manifest says so. The misleading +`--after`/`--since` flags on `index export` are **deprecated** in favor of +`--delta` (they never actually filtered — §2.1); per `docs/deprecations.md` +policy they keep working with a warning, and the deprecation registry gets a +row when this ships. + +### 4.3 Delta bundles + +`gitsema index export --delta --base --out `: + +1. Enumerate commits in `(base, tip]` via `git rev-list base..tip` (streaming, + like the indexer — never buffer history). +2. Collect the row-set reachable from those commits that a consumer at `base` + cannot already have: new `blobs`, their `embeddings` / `chunks` / + `chunk_embeddings` / `symbol(_embeddings)` / `structural_refs` / + `blob_fts` content / `paths`, plus the new `commits`, `blob_commits`, + `indexed_commits`, `blob_branches` rows. +3. Write them into a fresh sqlite file **with the same schema** (reuse + `openDatabaseAt()` on a temp path so migrations/DDL stay single-sourced), + tar it with a `kind: "delta"` manifest carrying `baseCommit` + `tipCommit`. + +`index import` on a delta: verify the local index's `indexed_commits` contains +`baseCommit` (else error with a pointer to the needed full bundle); `ATTACH` +the delta DB and apply `INSERT OR IGNORE` per table. Because every row pivots +on `blob_hash` (blob-first constraint), application is **idempotent and +tolerant of overlap** — re-importing a delta, or importing overlapping deltas, +is harmless. Derived state that is rebuilt rather than merged (VSS index, +`graph_nodes`/`edges`, `module_embeddings`, cluster snapshots) is *excluded* +from deltas; import prints which rebuild commands apply (`index build-vss`, +`graph build`, `index update-modules`) — same recompute-locally convention the +knowledge-graph layer already uses. + +Embed-config gate on delta import: every `configHash` in the delta manifest +must already exist in the local `embed_config` table (a delta is a +continuation, not a mixer). Mismatch → error suggesting the matching full +bundle. Quantization: reads tolerate mixed quantization (dequantize-at-read, +§2.2), so delta import only *warns* when `quantized` differs; continuing to +*index locally* on top keeps today's `--allow-mixed` behavior. + +### 4.4 Bundle index + resolution (`gitsema index attach`) + +A **bundle-index file** `gitsema-bundles.json` sits next to the bundles: + +```jsonc +{ + "formatVersion": 1, + "repo": { "normalizedUrl": "…", "repoId": "…" }, + "bundles": [ + { "kind": "full", "tipCommit": "…", "embedConfigHashes": ["…"], "profileName": "default", + "url": "gitsema-full--.tar.gz", "size": 123456789, "sha256": "…" }, + { "kind": "delta", "baseCommit": "…", "tipCommit": "…", "embedConfigHashes": ["…"], + "url": "gitsema-delta---.tar.gz", "size": 4567890, "sha256": "…" } + ] +} +``` + +(``/`` = first 12 hex chars of configHash / commit — display +convention only; the manifest inside is authoritative.) + +`gitsema index attach []` resolution order: + +1. `--from ` — a bundle-index URL, a direct bundle URL, or a local path. +2. Config `bundles.url` / `GITSEMA_BUNDLE_URL` — a URL template supporting + `{repoId}` / `{normalizedUrl}` substitution (covers S3/artifact-store + conventions). +3. The configured gitsema server (`remoteUrl` config /`GITSEMA_REMOTE`): + `GET /bundles/{repoId}/gitsema-bundles.json` (§4.5). +4. Forge convention for the repo's `origin`: on GitHub, a rolling release + tagged `gitsema-index` whose assets are the bundle-index + bundles + (asset download inherits GitHub's own repo auth — private repos work with + no gitsema-side auth at all). + +Having fetched a bundle-index, `attach` picks the cheapest valid chain: exact +tip match → newest full bundle ≤ HEAD plus its delta chain → nearest ancestor +full bundle + **local incremental `index start` for the remainder** (the +existing `--since ` path makes this free). If nothing +resolves, `attach` falls back to plain local indexing — attach is always safe +to run, never worse than the status quo. `--offline`/`--no-fallback` variants +gate each half. Profile/config selection: `attach` filters candidates to +bundles whose `embedConfigs` match the caller's resolved embed config +(provider/model/dimensions/chunker per `computeConfigHash()`), with +`--any-config` to take whatever is offered when the consumer is read-only. + +`gitsema index publish` is the inverse: export (full or `--delta` from the +last published tip recorded in the fetched bundle-index), upload to the +resolved destination, and rewrite `gitsema-bundles.json` (locally it just +writes both files to `--dest `; the GitHub-release upload lives in the CI +action, Phase E, keeping the CLI free of forge-API dependencies). + +### 4.5 Server-side bundle serving + +`gitsema tools serve` gains three additive routes (new +`src/server/routes/bundles.ts`): + +- `GET /bundles/:repoId/gitsema-bundles.json` — the bundle index. +- `GET /bundles/:repoId/:bundleName` — streams the artifact + (`ETag: sha256`, `Cache-Control: public, max-age=…` for public repos so a + CDN/reverse proxy can absorb the fan-out). +- `POST /bundles/:repoId/rebuild` — (owner/admin) materialize a fresh full + bundle from the repo's `index.db`; also run automatically after a + `remote-index` job completes when `bundles.autoPublish` is set (server + keeps last N full bundles + deltas between them; N configurable, + default 2). + +**Auth is exactly Phases 122–126 — nothing new.** Bundle routes require the +same `read` grant as query routes for private repos; public repos +(`visibility: 'public'`) allow anonymous/any-authenticated fetch and trigger +the existing attach-as-reader auto-grant (`source: 'auto-public'`) for +authenticated callers, identical to the Phase 126 registration path. The +governing invariant, stated once and enforced in the route: **a bundle +contains the repo's full text (FTS content), so bundle access must be at +least as strict as repo read access.** Bundle downloads are `audit_log`-worthy +events (same shape as grant/token events). + +### 4.6 Storage-backend scope + +Bundles are sqlite artifacts in v1, on both ends. Postgres/qdrant deployments +consume a bundle via *import to a temp sqlite DB → `storage migrate`* — which +is exactly the direction `storage migrate` already supports (sqlite sources +only). Direct non-sqlite export/import is out of scope until someone actually +needs it (§7). + +### 4.7 Fit with the design constraints + +- **Git is the source of truth** — the manifest stores only commit hashes and + the normalized remote URL; nothing Git knows is restated. +- **Blob-first** — deltas are blob-keyed row sets; `INSERT OR IGNORE` on + content-addressed keys is what makes merge trivial. +- **Immutable embeddings** — extended across machines: attach *is* the dedup + check at fleet scale. +- **Streaming** — delta export walks `rev-list` streaming; import should + stream tar entries to disk (fixing the current buffer-whole-file-in-memory + import as part of Phase A). +- **CLI-first** — everything is `gitsema index `; server routes and the + CI action are thin wrappers over the same `bundleIndex.ts` core. + +--- + +## 5. Phased implementation plan + +Sized like existing PLAN.md phases; numbering assigned later by phase-plan. +Phases A–B are pure CLI/local; C–E add distribution. Each is independently +shippable. + +**Phase A — Manifest v2 + import hardening (~250–350 LOC)** +- Extend `exportIndex`/`importIndex`: write/read `bundleFormat: 2` manifests + with repo identity, `tipCommit`, `embedConfigs` (+ per-config `quantized`), + `profileName`; keep reading format-1 bundles. +- Import: config-compatibility gate (§4.1), stream-to-disk extraction, + `PRAGMA quick_check` before declaring success. +- Deprecate `index export --after/--since` (warning + `docs/deprecations.md` + row; flags keep annotating as today). +- Tests: round-trip with manifest assertions; legacy-bundle import; conflict + refusal. Changeset (minor). + +**Phase B — Delta bundles (~400–550 LOC)** +- `index export --delta --base `: streaming row-set collection + (§4.3), same-schema delta DB via `openDatabaseAt()` on a temp path. +- `index import` delta path: base-commit precondition, `ATTACH` + + `INSERT OR IGNORE` per table, derived-state rebuild hints. +- Tests: integration — index repo at C1, export full; advance to C2, export + delta; fresh clone imports full+delta ≡ indexing C2 directly (compare + blob/embedding counts + a search result); idempotent re-import. + +**Phase C — `index attach` / `index publish` + HTTPS resolution (~450–600 LOC)** +- `gitsema-bundles.json` read/write; resolution ladder §4.4 (explicit URL → + `bundles.url` template → gitsema server → GitHub `gitsema-index` release + via plain asset URLs); chain planner (full → deltas → local catch-up); + fallback to local indexing. +- New config keys: `bundles.url`, `bundles.requireChecksum` (default true). +- Tests: resolution-order unit tests with a mocked fetch; chain-planning + cases (exact tip / delta chain / ancestor + catch-up / nothing). +- README + features.md + parity.md (new subcommands are CLI-only initially — + record that consciously in the parity matrix). + +**Phase D — Server bundle routes + auth integration (~350–500 LOC)** +- `src/server/routes/bundles.ts` (§4.5): index/artifact GET with grant + checks + attach-as-reader on public repos, ETag/Cache-Control, streamed + responses; `POST …/rebuild`; `bundles.autoPublish` + retention (keep-N). +- `audit_log` entries for bundle downloads/rebuilds. +- Tests: `serverRoutes.test.ts` additions — private 401/403, public + anonymous fetch, auto-grant row created, ETag revalidation. + +**Phase E — CI action + zero-server loop docs (~150–250 LOC + workflow)** +- A reusable GitHub Action (composite or small JS) — checkout → restore + previous bundle via `index attach` → `index start` (incremental) → + `index publish --delta` → upload assets to the rolling `gitsema-index` + release. +- Docs: "index once, serve many" guide in README/features.md; deploy.md + section for putting a CDN in front of `/bundles`. + +Total: five phases, roughly 1,600–2,250 LOC + tests/docs — comparable to the +multi-tenant-auth track's per-phase sizing. + +--- + +## 6. Decisions taken autonomously (pending user review) + +This refinement ran non-interactively; every product/scope call the skill +would normally ask about was resolved from codebase evidence + stated +constraints. Each is revisitable before phase-plan. + +1. **Evolve the Phase 54 bundle rather than a new format** (Gap: artifact + format). Rationale: sqlite is already the serialization and gives + idempotent `INSERT OR IGNORE` deltas + migrations-on-open for free + (§3-A1); a custom packfile format was the withdrawn design's path and adds + parser surface for zero benefit at these sizes. +2. **Deltas as same-schema sqlite files applied via `ATTACH` + + `INSERT OR IGNORE`**, excluding derived/rebuildable state (VSS, graph, + module centroids, clusters). Rationale: blob-first keys make this + idempotent; derived state already has recompute commands. +3. **Compatibility key = the existing `embed_config` tuple/`configHash`, with + `quantized` carried alongside per config — `computeConfigHash()` itself is + NOT changed.** Rationale: the task and Phase 128 both say derive from + `embed_config`, don't invent a version string; changing the hash would + orphan every existing `embed_config` row. Quantization can't go *into* the + hash for the same reason, and read paths dequantize anyway, so it gates + only continued indexing (warn, keep `--allow-mixed` semantics) (§2.2, + §4.3). +4. **Deprecate `index export --after/--since`** rather than implementing real + time-sliced export. Rationale: verified the flags never filtered + (§2.1) — commit-range deltas subsume the use case with correct semantics; + "warn forever" per deprecations.md policy. +5. **Command names: `gitsema index attach` and `gitsema index publish`** + (subcommands, not top-level; the feature-ideas entry left the name TBD). + Rationale: keeps the CLI-first surface inside the existing `index` group + (no `src/cli/index.ts` top-level addition), sits beside + `export`/`import` which they orchestrate, and "attach" deliberately echoes + Phase 126's attach-as-reader semantics — same product concept, artifact + edition. +6. **Resolution = layered convention (explicit URL → config template → + gitsema server → GitHub rolling release), one static + `gitsema-bundles.json` per location; no registry service** (Gap: + resolution). Rationale: a registry is a new server, which this design + exists to avoid; the ladder covers S3/artifact-store/self-hosted/forge + cases with only static files. Forge convention chosen: rolling release + tagged `gitsema-index` (release assets are the only forge artifact type + that is plain-HTTPS-fetchable, size-tolerant, and auth-inherited). +7. **v1 trust = per-file sha256 (already present) + bundle-index sha256 + + HTTPS channel/forge auth; signatures deferred** with a reserved manifest + `signature` field and a concrete deferred proposal (`ssh-keygen -Y` + detached signatures — Git's own signing tooling, zero new deps) (Gap: + integrity/trust). Rationale: v1 publishers are the repo owner or their + CI, where channel trust ≈ code trust (anyone who can tamper with the + bundle asset can tamper with the code itself); third-party mirrors are + the real signing use case and aren't in v1. Added `PRAGMA quick_check` + on import since a sqlite file from the network is parser attack surface. +8. **Private-repo auth = exactly the Phases 122–126 grant model, with the + invariant "bundle access ≥ repo read access"** because bundles embed full + FTS text; public repos reuse the attach-as-reader auto-grant path + unchanged (Gap: auth interplay). No new credential type, no signed URLs + in v1. +9. **Bundles stay sqlite-only in v1; non-sqlite deployments go through + `storage migrate`** — matching the migrate command's existing + sqlite-source-only support (§4.6). +10. **Git-ref-namespace transport (`refs/gitsema/bundles/*`) deferred, not + rejected** — kept as the one salvageable federation transport idea, but + v1 needs no git-remote write path (§7). + +--- + +## 7. Remaining open questions + +Deliberately deferred — these need either real usage data or a prototype, not +more desk design: + +1. **Signatures & key distribution.** When third-party bundle mirrors appear, + who signs, where do consumer trust anchors live (`bundles.trustedSigners` + config? allowed-signers file à la git), and does `attach` hard-fail or + warn on unsigned? The manifest field is reserved; the policy is not + designed. +2. **Git ref namespace transport.** Worth building only if a hosting pattern + emerges where releases/S3 are unavailable but git push access exists. + Costs: large binaries in remotes, forge quota policies. +3. **Server retention/GC tuning.** Keep-N full bundles is a placeholder; + real deployments may want size-based caps or delta-chain compaction + (merge k deltas into one). Decide after Phase D telemetry. +4. **Multi-profile publishing ergonomics.** Server repos are single-profile + (pinned), so one bundle chain per repo suffices there; a *local* publisher + with a deliberately multi-model DB currently publishes one chain carrying + all configs. Whether anyone wants per-config split bundles is unknown. +5. **VSS sidecar deltas.** usearch indexes are rebuild-only in this design + (excluded from deltas). If rebuild time at large blob counts proves + painful for attachers, revisit shipping the sidecar in full bundles only + at first (it already is) vs. incremental HNSW updates. +6. **Direct non-sqlite bundle import** (postgres/qdrant without the temp- + sqlite hop) — only if the two-step path proves operationally annoying. + +--- + +## 8. Relationship to other documents + +- Supersedes the "Prebuilt Index Distribution" entry in + `docs/feature-ideas.md` (now a pointer here). +- Background only: `docs/design/semantic-federation.md` (⛔ withdrawn) — §2.5 + lists what was kept; nothing else from it applies. +- Builds on: `docs/public-repo-sharing-plan.md` (visibility/attach-as-reader), + `docs/locked-model-set-plan.md` (profiles/`repos.profile_name`), + `docs/multi-tenant-auth-plan.md` (grants/audit), + `docs/storage-backends-plan.md` (sqlite-artifact scope, §4.6). +- On shipping Phase A: add the `index export --after/--since` row to + `docs/deprecations.md`; update `docs/parity.md` for the new subcommands + (CLI-only at first, by decision); changeset per phase. diff --git a/docs/semantic-enrichment-plan.md b/docs/semantic-enrichment-plan.md new file mode 100644 index 0000000..bfc8777 --- /dev/null +++ b/docs/semantic-enrichment-plan.md @@ -0,0 +1,498 @@ +# Chunk-Level Semantic Enrichment (summaries, keywords, entities) — Design Plan + +**Status:** draft — refined from the `docs/feature-ideas.md` entry of the same +name (itself the salvaged Layer-1 kernel of the withdrawn +`docs/design/semantic-federation.md`; that doc's networking layers remain out +of scope and withdrawn). +**Target phases:** not yet scheduled. Next free PLAN.md phase numbers at time +of writing are 154+; note that 154–158 were briefly occupied by the withdrawn +federation track before removal, so the phase-plan step may prefer to start at +159 to keep history unambiguous. +**Scope:** an opt-in, LLM-generated metadata layer — `summary`, `keywords`, +optional `entities` — attached to the index's existing retrieval units +(whole-file blobs and chunks), stored as rows that **reference** the existing +blob/chunk/embedding rows (vectors are never duplicated; embeddings stay the +single source of truth), redacted **before storage**, and surfaced through the +existing search/first-seen/MCP/HTTP/guide result paths. **Out of scope:** any +networking or federation (withdrawn), the SKOS concept vocabulary (separate +feature-ideas entry; §8 records the interface reserved for it), symbol-level +enrichment (open question, §7), and lazy enrich-on-search (rejected, §3.3). + +This refinement ran non-interactively, so every product/scope decision the +refine-idea flow would normally put to the user was taken autonomously from +codebase evidence and recorded in **§6 “Decisions taken autonomously (pending +user review)”**. Review §6 before turning this doc into PLAN.md phases. + +--- + +## 1. Motivation + +Search results, MCP tool output, and guide grounding today surface raw scores, +paths, line ranges, and (at best) a truncated head of raw content: + +- `renderResults()` (`src/core/search/ranking.ts:53-74`) prints + `score path [blob:hash] date :lines` — no description of *what the code is*. +- The only "snippet" in the system is `explainFormatter.ts`'s + `content.slice(0, 200)` (`src/core/search/analysis/explainFormatter.ts:46`), + a blind prefix of raw text. +- MCP tools and HTTP routes serialize the same `SearchResult` objects + (`src/core/models/types.ts:21`), so agents receive hashes and scores, then + must re-fetch and re-read full blob content to learn that a hit is, say, + "the JWT validation middleware" — burning tokens on content the indexer + already read and embedded once. + +The indexer understands each unit of content exactly once (content-addressed, +CLAUDE.md design constraint #3). This design captures a small, durable, +human/LLM-readable distillation of that understanding at the same moment — +or in a later backfill pass — so every downstream consumer (CLI users, MCP +agents, the guide loop) gets a one-line answer to "what is this?" without +re-reading the content. + +Hard requirements carried in from the feature-ideas entry: + +1. Enrichment rows **reference** existing chunk/embedding rows. No vector is + ever copied or recomputed; no blob content is duplicated. +2. LLM output passes through the existing redaction layer + (`src/core/narrator/redact.ts`) **before storage** — stored summaries can + later leave the machine (the Phase 54 bundle exports the whole `index.db` + file verbatim, `src/cli/commands/bundleIndex.ts`; server routes return + search results to remote clients). +3. The design covers the storage abstraction (`src/core/storage/`, all three + backends), not just sqlite. +4. Enrichment uses the existing narrator/guide model infrastructure + (Phase 91+, `gitsema models`), not a new provider stack. + +--- + +## 2. Current state (verified against source) + +| Claim | Evidence | +|---|---| +| The default chunker is `file` — most indexed repos have **no `chunks` rows at all**, only whole-file `embeddings` keyed `(blob_hash, model)` | `chunkerStrategy = 'file'` default and `useChunking = chunkerStrategy !== 'file'`, `src/core/indexing/indexer.ts:274,324`; CLAUDE.md `--chunker` default | +| Chunk rows are keyed by a **local autoincrement id**; only `(blob_hash, start_line, end_line)` is portable/content-stable | `chunks` table, `src/core/db/schema.ts:8-13`; `chunk_embeddings` PK `(chunk_id, model)`, `schema.ts:18-28` | +| All persisted reads/writes go through the async storage seam — `MetadataStore` / `VectorStore` / `FtsStore` / `GraphStore`; relational-only data already has a precedent (`structural_refs` lives on `MetadataStore.storeStructuralRefs`, and `GraphStore` throws on Qdrant profiles) | `src/core/storage/types.ts:91-120,216-247`; qdrant profile has **no metadata store of its own** — `src/core/storage/qdrant/` contains only `connection.ts`/`profile.ts`/`vectorStore.ts`, metadata+FTS come from the companion Postgres store | +| Postgres has its own idempotent migrations file that mirrors the sqlite tables | `CREATE TABLE IF NOT EXISTS chunks/chunk_embeddings`, `src/core/storage/postgres/migrations.ts:69-78` | +| The narrator model stack is complete and reusable: named configs in `embed_config` (`kind='narrator'/'guide'`), active-selection in `settings`, `resolveNarratorProvider()`, BYOK, CLI-subprocess backends | `src/core/narrator/resolveNarrator.ts` (config CRUD + `createNarratorProviderFor` + `resolveNarratorProvider`); `NarratorProvider.narrate({systemPrompt,userPrompt,maxTokens}) → {prose,tokensUsed,redactedFields,llmEnabled}`, `src/core/narrator/types.ts:13-42` | +| **Outbound** redaction is already enforced inside the providers (prompts are redacted before leaving the process), with fired patterns returned and audited | `redactedUser`/`redactedSystem` + `withAudit('narrate', …, allFired, fn)`, `src/core/narrator/chattydeerProvider.ts:95-140`; `redact()`/`redactAll()`, `src/core/narrator/redact.ts:103-130`; `withAudit`, `src/core/narrator/audit.ts:51` (operation type is currently `'narrate' \| 'explain'`) | +| **No inbound redaction exists** — LLM *responses* are returned to the caller as prose and never stored, so nothing today redacts model output. Storing summaries makes inbound redaction a new, mandatory step | absence of any `redact` call on `result.explanation` in `chattydeerProvider.ts:116-124`; grep for `redact` shows only prompt-side call sites | +| Search results are assembled in one place — paths + first-seen are batch-joined onto the top-K entries; this is the natural join point for enrichment fields | result-assembly block, `src/core/search/analysis/vectorSearch.ts:540-600` (`pathsByBlob`, `getFirstSeenMap`, `SearchResult` construction with `chunkId`/`startLine`/`endLine`) | +| MCP and HTTP serialize `SearchResult` objects as JSON — new optional fields propagate to both interfaces with zero per-tool work | `serializeSearchResults`, `src/mcp/registerTool.ts:6`; `res.json({ blobResults, commitResults })`, `src/server/routes/search.ts:344-347` | +| The CLI text renderer is the only surface that needs an explicit change to *show* new fields | `renderResults`, `src/core/search/ranking.ts:53-74` | +| Post-pass maintenance subcommands over an existing index are an established pattern | `index update-modules`, `index rebuild-fts`, `index build-vss` (CLAUDE.md command table) | +| Per-blob error tolerance is the indexing convention — provider failures are caught per unit and counted in stats, never fatal | chunk-embed catch + `stats` counters, `src/core/indexing/indexer.ts:859-880`; CLAUDE.md "Error handling" | +| `index export` copies the whole `index.db` into the bundle — any new table (and any secret stored in it) automatically leaves the machine with the bundle | `src/cli/commands/bundleIndex.ts:1-60` | +| Tool-surface conventions: usage lives in `GUIDE_TOOLS` descriptions + MCP `registerTool` descriptions; result-reading guidance lives in `TOOL_INTERPRETATIONS`; `pnpm gen:skill` regenerates the skill and a `docsSync` test enforces `GUIDE_TOOLS ⊆ TOOL_INTERPRETATIONS` | header of `src/core/narrator/interpretations.ts:1-39`; CLAUDE.md "Tool interpretations" | +| Current sqlite schema version is **32**; this design adds **v33** | CLAUDE.md schema table; `CURRENT_SCHEMA_VERSION` in `src/core/db/sqlite.ts` | + +**Conclusion:** every ingredient except two already exists — the missing +pieces are (a) an *inbound* redaction step (LLM output → storage) and (b) a +place to put the metadata. Everything else is composition: narrator providers +for the calls, the storage seam for persistence, the single result-assembly +site for surfacing, and additive JSON fields for MCP/HTTP parity. + +--- + +## 3. Conceptual model & options considered + +Enrichment is a function `enrich(unit) → {summary, keywords, entities?}` +applied at most once per `(unit, enricher-model)` pair — the same +"embedded exactly once" invariant that governs vectors (CLAUDE.md design +constraint #3), applied to a second, cheaper artifact. Four axes had real +alternatives. + +### 3.1 Axis A — What is a "unit"? (granularity) + +| Option | Pros | Cons | +|---|---|---| +| **A1. Chunks only** (the idea's title) | Matches the federation design's Layer-1 sketch; finest-grained summaries | The default chunker is `file` → **no chunk rows exist** for most repos (§2 row 1); the feature would be a silent no-op for the default configuration | +| **A2. Whole-file blobs only** | Covers the default path; simplest keying (`blob_hash`) | Chunked repos (function/fixed) lose the fine-grained "JWT validation handler, lines 42–87" value that motivated the idea | +| **A3. Retrieval units: file-level always, chunk-level when chunk rows exist** ✅ | Covers both realities; mirrors exactly what search can return (`SearchResult.kind` is `file`/`chunk`); one table serves both | Slightly wider schema (nullable line range) | +| A4. Also symbols/modules/commits | Maximal coverage | Cost multiplies; symbols already carry a human-readable identity (`qualifiedName(signature)`, Phase 105) which *is* a summary of sorts; modules/commits have other mechanisms (`module_embeddings` centroids, commit messages). Deferred (§7) | + +**Chosen: A3.** A unit is either a whole-file blob (`blob_hash`, lines NULL) +or a chunk (`blob_hash`, `start_line`, `end_line`). The chunk key +deliberately uses the **portable content coordinates** rather than the local +autoincrement `chunk_id` (§2 row 2), so enrichment rows survive +`index export`/`import` and are joinable in every backend. + +### 3.2 Axis B — Where does it live? (schema) + +| Option | Pros | Cons | +|---|---|---| +| B1. Nullable columns on `chunk_embeddings` (the federation sketch's "extend chunk_embeddings" idea) | No new table | Cannot represent file-level units (A3); mutates rows that constraint #3 treats as immutable; couples enrichment lifetime to one embedding model's rows; Qdrant stores vectors *outside* Postgres — there is no `chunk_embeddings` row to put a column on in that profile | +| B2. Content-addressed `semantic_objects` table duplicating `embedding BLOB` per object (the withdrawn federation schema, `semantic-federation.md:88-104`) | Self-contained envelope | **Violates the no-vector-duplication hard requirement**; existed to serve gossip/packfiles that are withdrawn | +| **B3. New `enrichments` table that references existing rows** ✅ | File+chunk units in one place; vectors untouched; multiple enricher models can coexist (keyed per model, like every embedding table); relational-only → trivially portable to Postgres; rides along in bundles automatically | One new table + v33 migration in two backends | + +**Chosen: B3.** + +``` +enrichments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, -- (BIGSERIAL in Postgres) + blob_hash TEXT NOT NULL REFERENCES blobs(blob_hash), + start_line INTEGER, -- NULL, NULL = whole-file unit + end_line INTEGER, -- both set = chunk unit (1-indexed, inclusive) + model TEXT NOT NULL, -- enricher model-config NAME (embed_config.model) + summary TEXT NOT NULL, -- redacted; target <= ~200 chars + keywords TEXT NOT NULL, -- redacted; JSON array of normalized strings, <= 10 + entities TEXT, -- redacted; JSON array of {name, kind}, <= 10; NULL if none + redacted_fields TEXT NOT NULL, -- JSON array of fired redaction pattern names ([] if clean) + tokens_used INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL -- unix seconds +) +UNIQUE INDEX idx_enrichments_unit ON (blob_hash, COALESCE(start_line, -1), COALESCE(end_line, -1), model) +INDEX idx_enrichments_blob ON (blob_hash) +``` + +(Drizzle cannot express the `COALESCE` unique index directly; the sqlite +migration creates it with raw SQL, the same escape hatch existing migrations +use. Postgres gets the identical expression index in +`storage/postgres/migrations.ts`.) + +- `model` stores the **config name** (`embed_config.model` for + `kind='narrator'/'guide'` rows), matching how `resolveNarrator.ts` names + configs. Two different enricher models can each enrich the same unit; reads + prefer the currently-resolved enricher and fall back to the most recent row + (§4.4). +- `redacted_fields` persists which redaction patterns fired on the *stored* + output — the audit-visibility analogue of `NarrateResponse.redactedFields`. +- **No FTS row is written for summaries in v1** — see §7 (open question: + hybrid search over summaries). + +**Storage-seam coverage (hard requirement 3):** enrichment is relational +metadata, so it extends `MetadataStore` (`src/core/storage/types.ts`) — the +exact pattern `structural_refs` used (§2 row 3): + +```ts +/** A stored (or to-be-stored) enrichment row for one retrieval unit. */ +export interface EnrichmentRecord { + blobHash: string + startLine?: number // both unset = whole-file unit + endLine?: number + model: string // enricher config name + summary: string // already redacted by the caller + keywords: string[] // already redacted + normalized by the caller + entities?: Array<{ name: string; kind: string }> + redactedFields: string[] + tokensUsed: number +} + +interface MetadataStore { + // ... existing methods ... + /** Upserts one enrichment row (unique per unit+model). */ + putEnrichment(rec: EnrichmentRecord): Promise + /** Batch lookup for result assembly: all enrichments for these blobs, optionally one model. */ + getEnrichments(blobHashes: string[], model?: string): Promise + /** Units (file- and chunk-level) lacking an enrichment for `model`; limited for backfill batching. */ + listUnenrichedUnits(model: string, opts?: { limit?: number; ext?: string[] }): Promise> + /** Row count, optionally per model — feeds `gitsema status` / `index doctor`. */ + countEnrichments(model?: string): Promise +} +``` + +Backend coverage: **sqlite** implements against the v33 table; **postgres** +implements against its mirrored table; **qdrant** profiles are covered *by +construction* — their `MetadataStore` **is** the companion Postgres store (§2 +row 3), so no qdrant-specific code exists, and unlike `GraphStore` nothing +needs to throw. `storage migrate` copies the table like any other metadata +table (sqlite→postgres/qdrant paths gain it in the same phase as the +migration). + +### 3.3 Axis C — When does enrichment run? (trigger & cost model) + +| Option | Pros | Cons | +|---|---|---| +| C1. Always-on at index time | Zero extra commands | Violates safe-by-default: indexing would silently call an LLM; cost surprise on large histories | +| **C2. Opt-in at index time (`--semantic-enrich`) + explicit backfill (`index enrich`)** ✅ | Explicit consent to every LLM call (matches `narrate`/`explain`'s evidence-only-by-default posture); backfill covers already-indexed repos; dedup makes both resumable | Two entry points to document | +| C3. Lazy enrich-on-first-search-hit | Pay only for what's read | **Rejected:** read paths (`search`, MCP tools, HTTP GET routes) are network-free-and-fast today; a search that sometimes blocks on N LLM calls (and *writes* to the DB) breaks that contract and the safe-by-default rule. Revisit only if eager cost proves prohibitive (§7) | +| C4. "Top-N by search demand" scoring | Focuses spend | Requires a query-log/popularity mechanism that doesn't exist; premature. The `--enrich-max` cap + backfill `--limit` give the operator the same lever manually | + +**Chosen: C2**, with these cost controls (resolving Design Gap 1): + +- **Dedup is the primary control.** A `(unit, model)` pair is enriched at + most once, ever — checked against the unique index before any LLM call, + exactly like `deduper.ts` for embeddings. Content-addressing means + unchanged files across commits/branches never re-enrich. +- **Per-run cap:** `--enrich-max ` (default: unlimited) stops issuing new + LLM calls after n units; combined with dedup this gives free + **resumability** — rerunning continues where the cap stopped, no + checkpoint table needed. +- **Scope inheritance:** index-time enrichment applies to exactly the units + the run indexes (so `--ext`, `--exclude`, `--include-glob`, `--max-size` + all constrain it for free); backfill takes its own `--ext`/`--limit`. +- **Concurrency:** a separate `p-limit` pool, `--enrich-concurrency` + (default **2**) — LLM calls are slower and pricier than embedding calls; + reusing the embedding `--concurrency` (default 4) would double-dip the + same budget. +- **Batching:** one unit per LLM call. A multi-unit-per-prompt variant + (k units in, k JSON objects out) was considered and rejected for v1: + chat-completions output attribution is fragile (one malformed element + poisons the whole batch) and per-unit token accounting/auditing gets + muddy. Recorded in §7 as a cost optimization to revisit. +- **Truncation:** unit content sent to the LLM is capped (first ~6 KB — + aligned with `--max-size`'s spirit; summaries of a file's head are + acceptable, blown context windows are not). +- **Dry-run:** `index enrich --dry-run` prints unit counts (and units-per- + model breakdown) without calling anything. + +### 3.4 Axis D — Which model, and how is output handled? + +**Model resolution (resolving the "narrator plumbing" prerequisite, no new +subsystem):** reuse `resolveNarratorProvider()` untouched. Resolution order +for the enricher: explicit `--enrich-model ` (a named `embed_config` +narrator/guide config) → the **active narrator config** → hard error with +the `gitsema models add … --narrator … --activate` hint. No silent skip: the +user explicitly opted in with `--semantic-enrich`, so "no model configured" +is an error, not a no-op. **No new `kind` in `embed_config`** — an +"enricher" is just a narrator-shaped model used for a different prompt; +adding a fourth kind would ripple through `models` CLI/MCP/HTTP for zero +expressiveness gain. (BYOK: server-side enrichment is out of scope in v1 — +§6 D11 — so the BYOK path doesn't arise.) + +**Output contract:** the system prompt requests **strict JSON** +`{"summary": string, "keywords": string[], "entities": [{"name","kind"}]}` +with hard limits (summary ≤ 200 chars target; ≤ 10 keywords; ≤ 10 entities; +entity `kind` ∈ `framework | protocol | service | domain | other`). The +response is parsed defensively (strip code fences, find first `{`…last `}`); +on parse failure retry **once** with a "JSON only" reminder; a second +failure counts the unit in `stats.enrichFailed` and moves on — the +per-blob non-fatal error convention (§2 row 11). + +**Redaction (hard requirement 2) — both directions:** + +1. *Outbound* — already handled inside every `NarratorProvider` (§2 row 6); + nothing to add. +2. *Inbound (new)* — after parsing, `redact()` is applied to the summary, + every keyword, and every entity name **before** the `putEnrichment()` + call. Fired pattern names from both directions are merged into + `redacted_fields`. This is belt-and-braces on purpose: even though the + input was redacted, an LLM can reconstruct or hallucinate secret-shaped + strings, and this table ships in bundles and server responses verbatim. +3. *Audit* — each call is wrapped in `withAudit('enrich', …)` + (`audit.ts`'s operation union gains `'enrich'`), so the existing + `[llm_audit]` log line covers enrichment traffic. + +**Keyword normalization (resolving Design Gap 4):** freeform terms, +normalized at write time — lowercased, trimmed, internal whitespace +collapsed to single spaces, deduplicated, each ≤ 40 chars, ≤ 10 kept. **No +concept-vocabulary references now.** The SKOS idea (feature-ideas.md) gets a +clean seam anyway: a future `concept_assignments` table would reference +enrichment rows by their `id` — nothing in *this* schema pre-commits to +that, which is exactly the "reserve by not entangling" posture the withdrawn +federation postmortem argued for. + +--- + +## 4. Chosen direction — end-to-end shape + +### 4.1 New module: `src/core/enrichment/` + +``` +src/core/enrichment/ + enricher.ts — enrichUnit(provider, unitContent, meta) → parsed+redacted EnrichmentRecord + (prompt build, JSON parse w/ one retry, inbound redact, withAudit('enrich')) + prompts.ts — system prompt + JSON contract (kept out of enricher.ts for testability) + backfill.ts — runEnrichBackfill(profile, provider, opts) — drives listUnenrichedUnits → + git cat-file content fetch (showBlob / line-slice for chunks) → enrichUnit → + putEnrichment, under p-limit(--enrich-concurrency) +``` + +It imports `resolveNarratorProvider` from `narrator/` but lives outside it — +`narrator/` stays about narrate/explain/guide prose; enrichment is an +indexing-side pipeline. Content for backfill comes from Git (`showBlob`, +constraint #1: Git is the source of truth), *not* from `blob_fts` (which is +optional — `fts: null` profiles must still backfill). + +### 4.2 Index-time path (`gitsema index start --semantic-enrich`) + +In `indexer.ts`, immediately after a unit's embedding write succeeds +(`writeFileBlob` for file units at :741/:829; the chunk `upsert` at :873), +the unit is queued to the enrichment pool. Enrichment failures never fail +the blob (it is already indexed); they increment `stats.enrichFailed`. New +stats: `enriched`, `enrichSkipped` (dedup hits), `enrichFailed`, +`enrichTokens`. Flags: `--semantic-enrich`, `--enrich-model `, +`--enrich-max `, `--enrich-concurrency `. +`--semantic-enrich` + `--remote` errors out in v1 (§6 D11). + +### 4.3 Backfill path (`gitsema index enrich`) + +New maintenance subcommand (precedent: `index update-modules`, +`index rebuild-fts`): resolves the enricher model, then enriches every +already-indexed unit missing a row for that model. Flags: `--model `, +`--limit `, `--ext `, `--concurrency `, `--dry-run`, `-y`. +Resumable by construction (dedup); safe to re-run; exit code contract: +plain 0/1 (not a gate). This resolves Design Gap 2. + +### 4.4 Surfacing (read path) + +- `SearchResult` (`src/core/models/types.ts`) gains optional fields: + `summary?: string`, `keywords?: string[]` (entities intentionally not + surfaced in results v1 — niche until a consumer exists; they remain + queryable in the table). +- Attachment happens at the single assembly site + (`vectorSearch.ts:540-600`): one batched + `getEnrichments(blobHashes, resolvedModel)` alongside the existing + paths/first-seen joins; file-kind results match rows with NULL lines, + chunk-kind results match on `(blobHash, startLine, endLine)`. Model + preference: the currently-resolved enricher config name if one is + configured, else the most recent row per unit. Missing enrichment ⇒ + fields absent (never an error, never an LLM call — reads stay + network-free). +- **CLI/REPL:** `renderResults()` prints an indented `↳ summary + [kw1, kw2, …]` line under each enriched result. No flag needed to enable; + a `--no-enrich` display flag is *not* added (one indented line is cheap; + flags are not). +- **MCP + HTTP:** zero per-tool work — the new optional fields flow through + `serializeSearchResults` and the routes' `res.json(...)` automatically + (§2 rows 8–9). Additive JSON fields; no breaking change (parity.md §4's + break-if-needed license is not needed here). +- **Guide:** search-family `GUIDE_TOOLS` results inherit the fields the same + way (they serialize `SearchResult`s), which is the actual token-saving + win: the agent reads the summary instead of calling a content fetch. +- **Parity surface for the new verb:** `index enrich` gets an MCP tool + `enrich` (args: `model?`, `limit?`, `ext?`, `dry_run?`) and HTTP + `POST /enrich`, mirroring how the `index` MCP tool exposes re-indexing; + guide gets the same tool in the `admin` category, index-gated and + LLM-gated (returns `{error}` when no index / no model, per convention). +- **Conventions compliance:** `TOOL_INTERPRETATIONS` gains an `enrich` + entry, and the entries for `semantic_search` / `search_history` / + `first_seen` / `code_search` get one added sentence describing the + optional `summary`/`keywords` fields and their caveat ("LLM-generated, + redacted, may be stale relative to a re-chunked config"); then + `pnpm gen:skill` regenerates the skill and the `docsSync` test pins it. +- **Status/health:** `gitsema status` reports enrichment coverage + (`countEnrichments` vs unit counts, per model); `index doctor` gains a + check for orphan enrichments (rows whose blob/unit no longer exists — + e.g. after `index gc`) and `index gc` deletes enrichments for collected + blobs. + +### 4.5 What is deliberately NOT built + +- No new vector, no re-embedding, no second source of semantic truth — + ranking is untouched; enrichment is presentation/grounding metadata only. +- No FTS row for summaries (yet — §7). +- No enrichment of symbols/modules/commits (§7). +- No lazy enrichment in read paths (§3.3, rejected). +- No new `embed_config` kind, no new provider class, no new daemon, no + networking of any sort. + +--- + +## 5. Phased implementation plan + +Sized like existing PLAN.md phases; each phase independently shippable, +each ends with docs (features.md, README command tables, parity.md where +tool surfaces change, deprecations.md n/a) + a changeset (`minor`). + +### Phase E1 — Enrichment core + storage + backfill (the CLI-first slice) + +- Schema **v33**: `enrichments` table + indexes in `src/core/db/schema.ts` + + migration in `sqlite.ts`; mirrored `CREATE TABLE IF NOT EXISTS` in + `src/core/storage/postgres/migrations.ts`; `storage migrate` copies it. +- `MetadataStore` gains `putEnrichment` / `getEnrichments` / + `listUnenrichedUnits` / `countEnrichments` (sqlite + postgres + implementations; qdrant covered via companion Postgres). +- `src/core/enrichment/{enricher,prompts,backfill}.ts` — prompt/JSON + contract, one-retry parse, **inbound redaction**, `withAudit('enrich')` + (extend the operation union in `audit.ts`). +- `gitsema index enrich` subcommand (`--model`, `--limit`, `--ext`, + `--concurrency`, `--dry-run`, `-y`). +- Tests: unit (JSON parse edge cases; normalization; inbound redaction with + a summary containing a planted `sk-…` key; unit-key matching incl. the + COALESCE unique index) + integration (temp repo, mock narrator provider, + backfill twice ⇒ second run is a full dedup no-op; `rawDb.close()` before + `rmSync` per the Windows CI rule). +- **Effort: the largest of the three — schema + seam + engine (~medium).** + +### Phase E2 — Index-time enrichment (`--semantic-enrich`) + +- `--semantic-enrich`, `--enrich-model`, `--enrich-max`, + `--enrich-concurrency` on `index start`; enrichment pool wired after the + embedding writes in `indexer.ts`; new stats counters in the run summary; + `--remote` incompatibility error. +- Tests: integration run with `--chunker file` and `--chunker fixed` + (file-level vs chunk-level units), cap behavior (`--enrich-max 3` then + rerun resumes), failure counting with a provider that errors on the 2nd + call. +- **Effort: small (plumbing into an existing pipeline).** + +### Phase E3 — Surfacing & interface parity + +- `SearchResult.summary`/`keywords` + assembly-site join + (`vectorSearch.ts`); `renderResults` indented line; verify MCP/HTTP + passthrough (supertest + MCP tool tests asserting field presence). +- `enrich` MCP tool + `POST /enrich` route + guide tool; + `TOOL_INTERPRETATIONS` entries (new `enrich`, amended search-family); + `pnpm gen:skill`; `docs/parity.md` matrix row for `enrich` and flag-parity + notes; `status` coverage line; `index doctor` orphan check + `index gc` + cleanup. +- **Effort: small–medium, mostly breadth (parity checklist).** + +--- + +## 6. Decisions taken autonomously (pending user review) + +This refinement ran non-interactively; the following would normally have +been clarifying questions. Each is reversible before phase-planning. + +| # | Decision | Rationale | +|---|---|---| +| **D1** | Units are **whole-file blobs AND chunks** (§3.1 A3), despite the idea's "chunk-level" title | The default chunker is `file` — chunk-only enrichment would no-op for default-configured repos (`indexer.ts:274,324`). Enriching exactly the set of things search can return keeps the feature aligned with its consumer | +| **D2** | **New `enrichments` table**, not columns on `chunk_embeddings` (§3.2 B3) | Columns can't represent file-level units; embedding rows are treated as immutable (constraint #3); Qdrant profiles have no relational `chunk_embeddings` row to extend; per-model coexistence needs its own key | +| **D3** | Chunk units keyed by **(blob_hash, start_line, end_line)**, not `chunk_id` | `chunk_id` is a local autoincrement — not portable across `export`/`import` or backends; content coordinates are stable for immutable blob content | +| **D4** | Storage seam = **new methods on `MetadataStore`** (no fourth store interface) | Enrichment is plain relational metadata; `structural_refs` set this exact precedent (`types.ts:119`); a dedicated `EnrichmentStore` would add interface surface for four methods | +| **D5** | **Eager opt-in + backfill only; lazy enrich-on-search rejected** (§3.3) | Read paths are network-free and fast today; silently calling an LLM (and writing) inside `search` breaks the repo's safe-by-default posture (`narrate`/`explain` evidence-only precedent) | +| **D6** | **Reuse narrator configs; no new `embed_config` kind, no new flag on `models`** | `resolveNarratorProvider` already does config CRUD/activation/BYOK; an enricher differs from a narrator only in prompt. Resolution: `--enrich-model ` → active narrator config → hard error (opt-in must not silently no-op) | +| **D7** | **Inbound redaction is mandatory and recorded** — `redact()` on summary/keywords/entities before `putEnrichment`, fired patterns persisted in `redacted_fields` | Hard requirement; providers only redact outbound today (`chattydeerProvider.ts:95-140`); bundles copy the whole DB (`bundleIndex.ts`), so stored text must already be clean | +| **D8** | **Strict-JSON contract, one retry, then non-fatal per-unit failure** | Matches the indexer's per-blob error convention (`indexer.ts:859-880`); a flaky LLM must not fail an indexing run | +| **D9** | **Keywords freeform + normalized** (lowercase/trim/dedup/≤10/≤40 chars); no SKOS references reserved in-schema (Design Gap 4) | SKOS is a separate undesigned idea; a future assignment table can reference `enrichments.id` without this schema pre-committing to anything | +| **D10** | Cost controls = **dedup + `--enrich-max` + separate `--enrich-concurrency` (2) + per-unit calls + ~6 KB content cap**; no popularity-based selection; resumability via dedup, no checkpoint table (Design Gap 1) | Dedup mirrors the embeddings invariant and makes reruns free; demand-based selection needs query-log infra that doesn't exist; multi-unit batching rejected for attribution fragility (revisit in §7) | +| **D11** | **`--semantic-enrich` is incompatible with `--remote` in v1** (hard error); server-side enrichment deferred | Remote indexing delegates embedding to a server whose model set/BYOK rules are governed by locked-model-set-plan.md; designing server-side enrichment now would re-entangle this with that track — the exact mistake the withdrawn federation design made | +| **D12** | Surfacing is **additive** (optional `summary`/`keywords` on `SearchResult`, always attached when present, no toggle flag); entities stored but not surfaced in results v1 | Additive JSON breaks no interface (parity.md §4 escape hatch unneeded); a display toggle is flag noise; entities have no consumer yet | +| **D13** | Doc filename `docs/semantic-enrichment-plan.md` | Matches the dominant recent precedent family (`storage-backends-plan.md`, `locked-model-set-plan.md`, `multi-tenant-auth-plan.md`, `public-repo-sharing-plan.md`) for infra-shaped, phase-ready designs | + +--- + +## 7. Remaining open questions (genuinely deferred) + +1. **FTS over summaries.** Writing summaries into the FTS store (a separate + namespace or a joined document) could boost `--hybrid` recall ("what does + this do"-shaped queries match summaries better than code tokens). Needs + an `eval`-harness measurement before committing — and a decision about + `fts: null` profiles. Extension phase candidate. +2. **Batched multi-unit prompts.** Could cut per-call overhead severely on + large backfills; rejected for v1 (attribution fragility, §3.3). Revisit + with real cost data from E1/E2 usage. +3. **Symbol-level enrichment.** Symbols already carry + `qualifiedName(signature)`; whether an LLM summary adds enough over that + to justify the spend is unproven. If added later it is one more unit + shape in the same table (symbol coordinates), not a redesign. +4. **Server-side enrichment for `tools serve` / remote indexing** (D11). + Interacts with the locked-model-set allow-list and BYOK never-persist + rules; design alongside that track when demand exists. +5. **On-demand enrichment** (rejected for v1, §3.3) — only revisit if eager + cost proves prohibitive in practice, and then only behind an explicit + `--enrich-on-read`-style opt-in with a write-path budget. +6. **SKOS concept vocabulary linkage** — separate feature-ideas entry; + `enrichments.id` is the anchor a future assignment table would use. + +--- + +## 8. Relationship to the withdrawn federation design + +This doc deliberately keeps the *only* part of +`docs/design/semantic-federation.md` Layer 1 that was valuable without a +network: the metadata envelope. Differences from that sketch, on purpose: + +- **No `semantic_objects` content-addressed envelope, no embedded vector + copy, no signer field** — those existed to serve gossip/packfile exchange + (withdrawn). Enrichment rows reference; they do not envelop. +- **No `profile_version`/`model_dimensions` duplication** — embedding + provenance already lives in `embed_config` and the per-row `model` + columns; repeating it here would create a second source of truth. +- **`language`/`structural_refs` fields dropped** — `embeddings.file_type` + and the `structural_refs` table (Phase 106) already store them. +- What survives: chunk-level granularity (extended to file-level units for + the default chunker), `summary`/`keywords`/`entities`, and the + "summaries travel where vectors are too heavy" insight — which here means + bundles and API responses rather than gossip, and is exactly why + redaction-before-storage is non-negotiable. diff --git a/src/core/git/branchDiff.ts b/src/core/git/branchDiff.ts index b6cdc43..747e948 100644 --- a/src/core/git/branchDiff.ts +++ b/src/core/git/branchDiff.ts @@ -1,26 +1,27 @@ -import { execFileSync } from 'node:child_process' import { getActiveSession } from '../db/sqlite.js' import { isSafeGitRange } from './refSafety.js' +import { runGit, UnsafeGitRefError } from './runGit.js' /** * Returns the merge-base commit hash for two branches/refs. * - * Runs `git merge-base `. + * Runs `git merge-base ` via `runGit()`, which rejects + * unsafe refs (leading `-`) before spawning git and always separates the + * positional refs with `--end-of-options` (Phase 150 / review11 §3.2). * Throws if the branches have no common ancestor or git fails. */ export function getMergeBase(branchA: string, branchB: string, repoPath = '.'): string { - if (!isSafeGitRange(branchA) || !isSafeGitRange(branchB)) { - throw new Error(`Unsafe git ref supplied to merge-base: ${JSON.stringify({ branchA, branchB })}`) - } - let out: string try { - out = execFileSync('git', ['merge-base', branchA, branchB], { + out = runGit('merge-base', [], [branchA, branchB], { cwd: repoPath, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }).trim() } catch (err) { + if (err instanceof UnsafeGitRefError) { + throw new Error(`Unsafe git ref supplied to merge-base: ${JSON.stringify({ branchA, branchB })}`) + } throw new Error( `Cannot find merge base for "${branchA}" and "${branchB}": ${err instanceof Error ? err.message : String(err)}`, ) @@ -55,13 +56,14 @@ export function getBranchExclusiveBlobs( let gitOutput: string try { - gitOutput = execFileSync( - 'git', - ['log', `${mergeBaseHash}..${branch}`, '--format=%H'], + gitOutput = runGit( + 'log', + ['--format=%H'], + [`${mergeBaseHash}..${branch}`], { cwd: repoPath, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, ) } catch { - return [] // branch not found or no git; return empty gracefully + return [] // unsafe ref, branch not found, or no git; return empty gracefully } const commitHashes = gitOutput diff --git a/src/core/git/runGit.ts b/src/core/git/runGit.ts new file mode 100644 index 0000000..c42408d --- /dev/null +++ b/src/core/git/runGit.ts @@ -0,0 +1,76 @@ +/** + * Shared, injection-safe `git` invocation helper (Phase 150 / review11 §2.1 + * + §3.2). + * + * Background: review9/10 fixed the *shell-string* form of git command + * injection by switching call sites from `execSync(\`git … ${x}\`)` (shell) + * to `execFileSync('git', [...])` (no shell). That defeats shell + * metacharacters (`;`, `|`, `$()`) but **not** git's own option parser — an + * argument beginning with `-` is still parsed as a *flag*, not a positional + * value. `git log --output=` (and similarly dangerous flags on other + * subcommands) then becomes an arbitrary-file-write primitive when a + * network-facing caller controls a "ref" string end-to-end + * (`semantic_bisect`/`triage`, review11 §2.1, PoC-confirmed). + * + * `runGit()` closes this at the sink, uniformly, for every call site that + * takes a caller-influenced ref: + * 1. Every `ref` is validated with `isSafeGitRange()` before git ever runs + * — rejects leading `-` and any character outside normal git revision + * syntax. + * 2. The argv always inserts git's `--end-of-options` marker immediately + * before the refs, so even a ref that somehow passed validation could + * never be parsed as a flag. + * + * Why `--end-of-options` and not a bare `--`: git's plain `--` separates + * *revisions* (before it) from *pathspecs* (after it) for commands like + * `log`/`show` — putting a ref after a bare `--` makes git treat it as a + * pathspec instead of a revision, silently breaking resolution of valid refs + * (verified empirically: `git log -1 --format=%ct -- HEAD~1` prints nothing, + * `git log -1 --format=%ct --end-of-options HEAD~1` resolves correctly). + * `--end-of-options` (git ≥ 2.24) is git's own mechanism for exactly this: + * "stop parsing option flags here" without reclassifying what follows. + */ + +import { execFileSync } from 'node:child_process' +import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process' +import { isSafeGitRange } from './refSafety.js' + +/** + * `runGit()` always returns a string, so callers may omit `encoding` + * (it defaults to 'utf8'); everything else in the exec options is passthrough. + */ +export type RunGitOptions = Omit & { + encoding?: ExecFileSyncOptionsWithStringEncoding['encoding'] +} + +export class UnsafeGitRefError extends Error { + constructor(ref: string) { + super(`Unsafe git ref: ${JSON.stringify(ref)}`) + this.name = 'UnsafeGitRefError' + } +} + +/** + * Runs `git [...flags] --end-of-options [...refs]` and returns + * stdout. Throws `UnsafeGitRefError` (without spawning git) if any `ref` + * fails `isSafeGitRange()`. + * + * `flags` are developer-controlled option strings (e.g. `-1`, + * `--format=%ct`) — never pass caller-supplied input as a flag. `refs` are + * caller-influenced positional values (commit hashes, branch names, ranges) + * — always pass caller-supplied revision input here, never in `flags`. + */ +export function runGit( + subcommand: string, + flags: string[], + refs: string[], + options: RunGitOptions, +): string { + for (const ref of refs) { + if (!isSafeGitRange(ref)) { + throw new UnsafeGitRefError(ref) + } + } + const argv = [subcommand, ...flags, '--end-of-options', ...refs] + return execFileSync('git', argv, { ...options, encoding: options.encoding ?? 'utf8' }) +} diff --git a/src/core/search/clustering/clustering.ts b/src/core/search/clustering/clustering.ts index 14f0c06..f32fd51 100644 --- a/src/core/search/clustering/clustering.ts +++ b/src/core/search/clustering/clustering.ts @@ -1,4 +1,3 @@ -import { execFileSync } from 'node:child_process' import Database from 'better-sqlite3' import { getActiveSession } from '../../db/sqlite.js' import { embeddings, paths } from '../../db/schema.js' @@ -6,7 +5,7 @@ import { logger } from '../../../utils/logger.js' import { bufferToEmbedding } from '../../../utils/embedding.js' import { cosineSimilarity } from '../analysis/vectorSearch.js' import { enhanceClusters, type EnhancedLabelOptions, type ClusterEnhancerInput } from './labelEnhancer.js' -import { isSafeGitRange } from '../../git/refSafety.js' +import { runGit, UnsafeGitRefError } from '../../git/runGit.js' // --------------------------------------------------------------------------- // Public types @@ -570,20 +569,20 @@ export function resolveRefToTimestamp(ref: string, repoPath = '.'): number { const d = new Date(ref) if (!isNaN(d.getTime())) return Math.floor(d.getTime() / 1000) - if (!isSafeGitRange(ref)) { - throw new Error(`Unsafe git ref: ${JSON.stringify(ref)}`) - } - - // Try as a git ref using git log + // Try as a git ref using git log. runGit() rejects unsafe refs (leading + // `-`) before spawning git, and always separates the ref with + // `--end-of-options` so it can never be parsed as a flag (Phase 150 / + // review11 §2.1 — the confirmed arbitrary-file-overwrite sink). try { - const out = execFileSync('git', ['log', '-1', '--format=%ct', ref], { + const out = runGit('log', ['-1', '--format=%ct'], [ref], { cwd: repoPath, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }).trim() const ts = parseInt(out, 10) if (!isNaN(ts) && ts > 0) return ts - } catch { + } catch (err) { + if (err instanceof UnsafeGitRefError) throw err // fall through } diff --git a/src/core/search/temporal/timeSearch.ts b/src/core/search/temporal/timeSearch.ts index cfe3d12..33f1783 100644 --- a/src/core/search/temporal/timeSearch.ts +++ b/src/core/search/temporal/timeSearch.ts @@ -1,9 +1,8 @@ import { getActiveSession } from '../../db/sqlite.js' -import { execFileSync } from 'node:child_process' import { dirname, resolve } from 'node:path' import { blobCommits, commits } from '../../db/schema.js' import { inArray, eq } from 'drizzle-orm' -import { isSafeGitRange } from '../../git/refSafety.js' +import { runGit, UnsafeGitRefError } from '../../git/runGit.js' export interface FirstSeenInfo { commitHash: string @@ -163,18 +162,20 @@ export function parseDateArg(value: string): number { // fall through to process.cwd() } - if (!isSafeGitRange(value)) { - throw new Error(`Unsafe git ref: ${JSON.stringify(value)}`) - } - + // runGit() rejects unsafe refs (leading `-`) before spawning git, and always + // separates the ref with `--end-of-options` so it can never be parsed as a + // flag (Phase 150 / review11 §3.2). try { - const commitHash = execFileSync('git', ['rev-parse', '--verify', value], { encoding: 'utf8', cwd: gitCwd }).trim() + const commitHash = runGit('rev-parse', ['--verify'], [value], { encoding: 'utf8', cwd: gitCwd }).trim() if (commitHash) { - const tsOut = execFileSync('git', ['show', '-s', '--format=%ct', commitHash], { encoding: 'utf8', cwd: gitCwd }).trim() + const tsOut = runGit('show', ['-s', '--format=%ct'], [commitHash], { encoding: 'utf8', cwd: gitCwd }).trim() const ts = parseInt(tsOut, 10) if (!isNaN(ts)) return ts } - } catch { + } catch (err) { + if (err instanceof UnsafeGitRefError) { + throw new Error(`Unsafe git ref: ${JSON.stringify(value)}`) + } // ignore and throw below } diff --git a/src/core/storage/types.ts b/src/core/storage/types.ts index d388a41..a3e25a7 100644 --- a/src/core/storage/types.ts +++ b/src/core/storage/types.ts @@ -170,6 +170,16 @@ export interface GraphEdgeRecord { */ export const MAX_GRAPH_TRAVERSAL_DEPTH = 3 +/** + * Upper bound for a network-supplied `depth` request parameter on the graph + * closure/blast-radius tools (Phase 152 / review11 §3.3). These tools accept a + * caller-chosen depth ("default: unlimited"); this cap is a generous but finite + * ceiling so a network client can't request a pathologically deep traversal. + * Deliberately far above `MAX_GRAPH_TRAVERSAL_DEPTH` — real dependency closures + * never approach it, so it never rejects a legitimate value. + */ +export const MAX_GRAPH_DEPTH_REQUEST = 64 + /** A node reached during a `GraphStore` traversal (Phase 108, knowledge-graph §6). */ export interface GraphHit { nodeKey: string diff --git a/src/mcp/tools/graph.ts b/src/mcp/tools/graph.ts index 6e66f53..63fd432 100644 --- a/src/mcp/tools/graph.ts +++ b/src/mcp/tools/graph.ts @@ -13,6 +13,7 @@ import { coChange } from '../../core/graph/coChange.js' import { blastRadius } from '../../core/graph/blastRadius.js' import { renderBlastRadius } from '../../cli/lib/graphRender.js' import { parseLens } from '../../cli/lib/lens.js' +import { MAX_GRAPH_DEPTH_REQUEST } from '../../core/storage/types.js' import type { EdgeType, GraphHit } from '../../core/storage/types.js' import type { SemanticHit } from '../../core/graph/semanticNeighbors.js' @@ -302,7 +303,7 @@ export function registerGraphTools(server: McpServer) { { identifier: z.string().describe('A symbol qualified name, file path, or literal node key (file:..., symbol:..., external:...)'), reverse: z.boolean().optional().describe('Walk dependents (inbound edges) instead of dependencies (outbound edges)'), - depth: z.number().int().positive().optional().describe('Traversal depth (default: unlimited)'), + depth: z.number().int().positive().max(MAX_GRAPH_DEPTH_REQUEST).optional().describe(`Traversal depth (default: unlimited, max ${MAX_GRAPH_DEPTH_REQUEST})`), edge_types: z.array(EDGE_TYPE_ENUM).optional().describe('Edge types to traverse (default: imports, calls, extends, implements)'), }, async ({ identifier, reverse, depth, edge_types }) => { @@ -363,7 +364,7 @@ export function registerGraphTools(server: McpServer) { { symbol: z.string().describe('A symbol qualified name, file path, or literal node key (file:..., symbol:..., external:...)'), lens: z.enum(['semantic', 'structural', 'hybrid']).optional().default('hybrid').describe('Which lens(es) to include (default: hybrid)'), - depth: z.number().int().positive().optional().describe('Structural traversal depth (default: unlimited within MAX_GRAPH_TRAVERSAL_DEPTH)'), + depth: z.number().int().positive().max(MAX_GRAPH_DEPTH_REQUEST).optional().describe('Structural traversal depth (default: unlimited within MAX_GRAPH_TRAVERSAL_DEPTH)'), top_k: z.number().int().positive().max(500).optional().describe('Number of semantic neighbors to return (default 10)'), }, async ({ symbol, lens, depth, top_k }) => { diff --git a/src/server/app.ts b/src/server/app.ts index 4c098e8..db02a9e 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -62,6 +62,7 @@ import type { EmbeddingProviderPair } from '../core/embedding/profiles.js' import type { ChunkStrategy } from '../core/chunking/chunker.js' import { authMiddleware } from './middleware/auth.js' import { repoSessionMiddleware } from './middleware/repoSession.js' +import { repoAuthMiddleware } from './middleware/repoAuth.js' import { requestTimingMiddleware, metricsRegistry, refreshIndexGauges, syncProcessCounters } from './middleware/metrics.js' import { buildRateLimiter } from './middleware/rateLimiter.js' import { statusRouter } from './routes/status.js' @@ -192,35 +193,40 @@ export function createApp(options: AppOptions): Express { // repo scope of a per-repo auth token) to a persisted repo's index DB and // makes it the active DB session for the request. With no repoId, requests // fall through to the default cwd `.gitsema/index.db` session unchanged. - app.use(`${base}/search`, repoSessionMiddleware, searchRouter({ textProvider, codeProvider })) + // + // repoAuthMiddleware (Phase 151 / review11 §2.2) runs immediately after, so + // the addressed repo's DB is active (authoritative source of its + // visibility). In multi-tenant mode it 403s a caller who holds no `read` + // grant on a named private repo; in single-dev/open mode it is a no-op. + app.use(`${base}/search`, repoSessionMiddleware, repoAuthMiddleware, searchRouter({ textProvider, codeProvider })) - app.use(`${base}/evolution`, repoSessionMiddleware, evolutionRouter({ textProvider })) + app.use(`${base}/evolution`, repoSessionMiddleware, repoAuthMiddleware, evolutionRouter({ textProvider })) app.use( `${base}/remote`, remoteRouter({ textProvider, codeProvider, chunkerStrategy, concurrency, profiles }), ) - app.use(`${base}/analysis`, repoSessionMiddleware, analysisRouter({ textProvider })) + app.use(`${base}/analysis`, repoSessionMiddleware, repoAuthMiddleware, analysisRouter({ textProvider })) // Phase 148: bisect/refactor-candidates/lifecycle/cherry-pick-suggest/ // file-diff/pr-report/regression-gate/code-review/heatmap/map — CLI // commands that previously had no HTTP route or MCP tool at all. - app.use(`${base}/insights`, repoSessionMiddleware, insightsRouter({ textProvider })) + app.use(`${base}/insights`, repoSessionMiddleware, repoAuthMiddleware, insightsRouter({ textProvider })) // Phase 110/111: structural knowledge-graph routes (hotspots, …) - app.use(`${base}/graph`, repoSessionMiddleware, graphRouter()) + app.use(`${base}/graph`, repoSessionMiddleware, repoAuthMiddleware, graphRouter()) // Phase 113: generic LSP/MCP remote-delegation dispatch — see src/server/routes/protocol.ts - app.use(`${base}/protocol`, repoSessionMiddleware, protocolRouter()) + app.use(`${base}/protocol`, repoSessionMiddleware, repoAuthMiddleware, protocolRouter()) - app.use(`${base}/watch`, repoSessionMiddleware, watchRouter({ textProvider })) + app.use(`${base}/watch`, repoSessionMiddleware, repoAuthMiddleware, watchRouter({ textProvider })) - app.use(`${base}/projections`, repoSessionMiddleware, projectionsRouter()) + app.use(`${base}/projections`, repoSessionMiddleware, repoAuthMiddleware, projectionsRouter()) // Narrator/explainer endpoints (POST /narrate, POST /explain) and guide chat - app.use(base, repoSessionMiddleware, narratorRouter) - app.use(`${base}/guide`, repoSessionMiddleware, guideRouter) + app.use(base, repoSessionMiddleware, repoAuthMiddleware, narratorRouter) + app.use(`${base}/guide`, repoSessionMiddleware, repoAuthMiddleware, guideRouter) // Phase 64: Capabilities manifest — machine-readable list of server capabilities app.get(`${base}/capabilities`, (_req, res) => { diff --git a/src/server/middleware/auth.ts b/src/server/middleware/auth.ts index af50062..fb77dce 100644 --- a/src/server/middleware/auth.ts +++ b/src/server/middleware/auth.ts @@ -10,6 +10,21 @@ declare global { interface Request { repoId?: string userId?: number + /** + * True when `req.repoId` was set by a legacy per-repo scoped token + * (`repo_tokens`) rather than by an explicit `repoId` in the request. + * A scoped token already implies access to its one repo, so + * `repoAuthMiddleware` (Phase 151) skips the grant/visibility check + * for these requests. + */ + repoTokenScoped?: boolean + /** + * True when the request authenticated with the global `GITSEMA_SERVE_KEY` + * — the operator/admin credential. `repoAuthMiddleware` (Phase 151) + * treats it as full access, so a single-key server's behavior is + * unchanged by the multi-tenant read gate. + */ + globalKeyAuth?: boolean } } } @@ -82,6 +97,7 @@ export function authMiddleware(req: Request, res: Response, next: NextFunction): expectedBuf.length === actualBuf.length && timingSafeEqual(expectedBuf, actualBuf) if (isGlobal) { + req.globalKeyAuth = true next() return } @@ -97,6 +113,7 @@ export function authMiddleware(req: Request, res: Response, next: NextFunction): .get(tokenHash) as { repo_id: string } | undefined if (row) { req.repoId = row.repo_id + req.repoTokenScoped = true next() return } diff --git a/src/server/middleware/repoAuth.ts b/src/server/middleware/repoAuth.ts new file mode 100644 index 0000000..ead7ffa --- /dev/null +++ b/src/server/middleware/repoAuth.ts @@ -0,0 +1,113 @@ +/** + * Read-route repo authorization gate (Phase 151 / review11 §2.2). + * + * The Multi-Tenant Auth Track (Phases 122–125) built `repo_grants` + + * `resolveUserRepoAccess`/`roleSatisfies`, but they were wired only into the + * index/register (`remote.ts`) and grant-management (`orgs.ts`) routes — none + * of the ~16 read/search/analysis/evolution/graph/insights routes enforced + * them. `repoSessionMiddleware` served any named `repoId` to any caller with + * no grant check ("read any private repo by ID"). + * + * This middleware runs **after** `repoSessionMiddleware` (so the active DB + * session is the addressed repo's own DB — the authoritative source of that + * repo's `visibility`, matching the Phase 126 read in `remote.ts`) and, when + * the server is in **multi-tenant mode**, requires the caller to hold a `read` + * grant on the addressed repo unless it is `public`. + * + * Enforcement is opt-in (`isMultiTenantMode()`), so a default open single-dev + * server (no key, no flag) is unchanged. Two credentials bypass the grant + * check because they already imply access: the global `GITSEMA_SERVE_KEY` + * (operator/admin) and a legacy per-repo scoped `repo_tokens` token (already + * scoped to its one repo). + * + * Repo-level only: per-branch grant filtering (rewriting the routes' + * `branch: string` param into a per-user granted-branch set) is deferred to a + * follow-on phase — see PLAN.md Phase 151 "Deferred". + */ + +import type { Request, Response, NextFunction } from 'express' +import { getActiveSession, getRawDb } from '../../core/db/sqlite.js' +import { getRepo } from '../../core/indexing/repoRegistry.js' +import { resolveUserRepoAccess, roleSatisfies } from '../../core/auth/grants.js' +import { resolveRequestedRepoId } from './repoSession.js' +import { logger } from '../../utils/logger.js' + +/** + * True when the server should enforce per-user repo authorization on read + * routes. Opt-in: + * - `GITSEMA_MULTI_TENANT` set explicitly wins (`1`/`true`/`yes`/`on` → + * enabled; anything else → disabled), giving operators an escape hatch. + * - Otherwise falls back to `GITSEMA_SERVE_KEY` presence: a keyed server is + * a shared deployment, so the gate engages. + * - A default open server (no key, no flag) stays unchanged. + */ +export function isMultiTenantMode(): boolean { + const flag = process.env.GITSEMA_MULTI_TENANT + if (flag !== undefined && flag !== '') { + const v = flag.toLowerCase() + return v === '1' || v === 'true' || v === 'yes' || v === 'on' + } + return Boolean(process.env.GITSEMA_SERVE_KEY) +} + +export function repoAuthMiddleware(req: Request, res: Response, next: NextFunction): void { + // Single-dev / open deployment: no enforcement, behavior unchanged. + if (!isMultiTenantMode()) { + next() + return + } + + const { repoId } = resolveRequestedRepoId(req) + // No repo addressed → default cwd session, single-repo semantics — nothing + // to gate. + if (!repoId) { + next() + return + } + + // Operator/admin (global key) has full access. + if (req.globalKeyAuth) { + next() + return + } + + // A legacy per-repo scoped token already implies access to its one repo + // (and `repoSessionMiddleware` has already 403'd any mismatch). + if (req.repoTokenScoped) { + next() + return + } + + // Public repos are readable by anyone. Visibility is read from the repo's + // own (now-active) DB mirror row — the authoritative source used by the + // Phase 126 register path (`remote.ts`). A missing mirror row defaults to + // private (safe default → grant required). + let isPublic = false + try { + const repo = getRepo(getActiveSession(), repoId) + isPublic = repo?.visibility === 'public' + } catch (err) { + logger.debug(`repoAuthMiddleware: visibility lookup failed for ${repoId}: ${err instanceof Error ? err.message : String(err)}`) + } + if (isPublic) { + next() + return + } + + // Private repo: require a `read` grant for the resolved user. Grants live in + // the control-plane DB (`getRawDb()`), unaffected by the active-session + // switch above. + if (req.userId !== undefined) { + try { + const role = resolveUserRepoAccess(getRawDb(), req.userId, repoId) + if (roleSatisfies(role, 'read')) { + next() + return + } + } catch (err) { + logger.debug(`repoAuthMiddleware: grant lookup failed for user ${req.userId} on ${repoId}: ${err instanceof Error ? err.message : String(err)}`) + } + } + + res.status(403).json({ error: 'Forbidden: no read access to this repo' }) +} diff --git a/src/server/middleware/repoSession.ts b/src/server/middleware/repoSession.ts index 483e0e4..fe45f04 100644 --- a/src/server/middleware/repoSession.ts +++ b/src/server/middleware/repoSession.ts @@ -15,19 +15,32 @@ import { getRegistrySession, getRepo } from '../../core/indexing/repoRegistry.js * - If no `repoId` is resolved (single-dev mode), the request proceeds against * the default cwd-relative `.gitsema/index.db` session, unchanged. */ -export function repoSessionMiddleware(req: Request, res: Response, next: NextFunction): void { +/** + * Resolves the repoId a request is asking for, using the same precedence as + * `repoSessionMiddleware`: an explicit `repoId` in the request body (POST) or + * query string (GET), else the per-repo scope of a legacy scoped token + * (`req.repoId`, set by `authMiddleware`). Returns `repoId: undefined` in + * single-dev mode (no repoId named, no scoped token). Shared with + * `repoAuthMiddleware` (Phase 151) so both middlewares agree on which repo is + * being addressed. + */ +export function resolveRequestedRepoId(req: Request): { requestedRepoId?: string; repoId?: string } { const bodyRepoId = req.body && typeof req.body === 'object' && typeof (req.body as Record)['repoId'] === 'string' ? (req.body as Record)['repoId'] as string : undefined const queryRepoId = typeof req.query['repoId'] === 'string' ? req.query['repoId'] as string : undefined const requestedRepoId = bodyRepoId ?? queryRepoId + return { requestedRepoId, repoId: requestedRepoId ?? req.repoId } +} + +export function repoSessionMiddleware(req: Request, res: Response, next: NextFunction): void { + const { requestedRepoId, repoId } = resolveRequestedRepoId(req) if (req.repoId && requestedRepoId && requestedRepoId !== req.repoId) { res.status(403).json({ error: 'Token is not authorized for this repo' }) return } - const repoId = requestedRepoId ?? req.repoId if (!repoId) { next() return diff --git a/src/server/routes/graph.ts b/src/server/routes/graph.ts index cd9497d..73eb207 100644 --- a/src/server/routes/graph.ts +++ b/src/server/routes/graph.ts @@ -33,6 +33,7 @@ import { deps, DEPS_EDGE_TYPES } from '../../core/graph/deps.js' import { coChange } from '../../core/graph/coChange.js' import { blastRadius } from '../../core/graph/blastRadius.js' import { parseLens } from '../../cli/lib/lens.js' +import { MAX_GRAPH_DEPTH_REQUEST } from '../../core/storage/types.js' import type { EdgeType } from '../../core/storage/types.js' const HotspotsBodySchema = z.object({ @@ -81,7 +82,9 @@ const CyclesBodySchema = z.object({ const DepsBodySchema = z.object({ identifier: z.string().min(1), reverse: z.boolean().optional(), - depth: z.number().int().positive().optional(), + // Upper-bounded per review11 §3.3 — deps' core BFS uses `depth ?? Infinity` + // (no server-side clamp), so a network-supplied depth needs an explicit cap. + depth: z.number().int().positive().max(MAX_GRAPH_DEPTH_REQUEST).optional(), edgeTypes: z.array(EdgeTypeSchema).optional(), }) @@ -93,7 +96,9 @@ const CoChangeBodySchema = z.object({ const BlastRadiusBodySchema = z.object({ symbol: z.string().min(1), lens: z.enum(['semantic', 'structural', 'hybrid']).optional().default('hybrid'), - depth: z.number().int().positive().optional(), + // Structural traversal already clamps to MAX_GRAPH_TRAVERSAL_DEPTH; this + // generous schema cap documents an explicit upper bound (review11 §3.3). + depth: z.number().int().positive().max(MAX_GRAPH_DEPTH_REQUEST).optional(), topK: z.number().int().positive().max(500).optional(), }) diff --git a/tests/byokCredentials.test.ts b/tests/byokCredentials.test.ts index a739ec0..7c4c038 100644 --- a/tests/byokCredentials.test.ts +++ b/tests/byokCredentials.test.ts @@ -68,11 +68,40 @@ describe('byokConfig()', () => { await expect(byokConfig({ httpUrl: 'http://127.0.0.1:9999' })).rejects.toThrow('blocked') }) + it('rejects the cloud metadata IP (169.254.169.254) by default', async () => { + await expect(byokConfig({ httpUrl: 'http://169.254.169.254/latest/meta-data/' })).rejects.toThrow('blocked') + }) + + it('rejects RFC-1918 private ranges by default', async () => { + await expect(byokConfig({ httpUrl: 'http://10.1.2.3:8080/v1' })).rejects.toThrow('blocked') + await expect(byokConfig({ httpUrl: 'http://192.168.0.5/v1' })).rejects.toThrow('blocked') + await expect(byokConfig({ httpUrl: 'http://172.16.9.9/v1' })).rejects.toThrow('blocked') + }) + + it('rejects a "localhost" hostname by default', async () => { + await expect(byokConfig({ httpUrl: 'http://localhost:11434/v1' })).rejects.toThrow('blocked') + }) + + it('rejects a non-http(s) scheme', async () => { + await expect(byokConfig({ httpUrl: 'file:///etc/passwd' })).rejects.toThrow(/http or https/) + }) + + it('allows a public https URL', async () => { + const config = await byokConfig({ httpUrl: 'https://api.example.com/v1' }) + expect(config.params.httpUrl).toBe('https://api.example.com/v1') + }) + it('allows allowlisted hosts and CIDRs via GITSEMA_BYOK_ALLOW_HOSTS', async () => { process.env.GITSEMA_BYOK_ALLOW_HOSTS = '127.0.0.1,10.0.0.0/8' const config = await byokConfig({ httpUrl: 'http://127.0.0.1:9999' }) expect(config.params.httpUrl).toBe('http://127.0.0.1:9999') }) + + it('re-permits an RFC-1918 host once its CIDR is allowlisted', async () => { + process.env.GITSEMA_BYOK_ALLOW_HOSTS = '10.0.0.0/8' + const config = await byokConfig({ httpUrl: 'http://10.1.2.3:8080/v1' }) + expect(config.params.httpUrl).toBe('http://10.1.2.3:8080/v1') + }) }) describe('resolveNarratorProvider({ byok }) — never persists, never DB-checked', () => { diff --git a/tests/integration/gitArgInjection.test.ts b/tests/integration/gitArgInjection.test.ts new file mode 100644 index 0000000..e7aba09 --- /dev/null +++ b/tests/integration/gitArgInjection.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { mkdtempSync, writeFileSync, rmSync, existsSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { execSync } from 'node:child_process' +import { runGit, UnsafeGitRefError } from '../../src/core/git/runGit.js' +import { resolveRefToTimestamp } from '../../src/core/search/clustering/clustering.js' + +/** + * Phase 150 / review11 §2.1 regression suite. The git argument-injection + * class: `execFileSync('git', [...])` defeats shell metacharacters but NOT + * git's own option parser — a "ref" beginning with `-` is parsed as a flag. + * `git log --output=` is then an arbitrary-file-write primitive + * reachable from `semantic_bisect`/`triage`. These tests assert the sinks + * reject leading-`-` refs (and never write the PoC file). + */ + +let repoDir: string + +beforeAll(() => { + repoDir = mkdtempSync(join(tmpdir(), 'gitsema-rungit-')) + execSync('git init', { cwd: repoDir, stdio: 'pipe' }) + execSync('git config user.email "test@test.com"', { cwd: repoDir, stdio: 'pipe' }) + execSync('git config user.name "Test"', { cwd: repoDir, stdio: 'pipe' }) + execSync('git config commit.gpgsign false', { cwd: repoDir, stdio: 'pipe' }) + writeFileSync(join(repoDir, 'f.txt'), 'hello', 'utf8') + execSync('git add f.txt', { cwd: repoDir, stdio: 'pipe' }) + execSync('git commit -m init', { cwd: repoDir, stdio: 'pipe' }) +}) + +afterAll(() => { + try { + rmSync(repoDir, { recursive: true, force: true }) + } catch { + // best-effort cleanup + } +}) + +describe('runGit', () => { + it('rejects a leading-dash ref before spawning git', () => { + expect(() => runGit('log', ['-1', '--format=%ct'], ['--output=pwned.txt'], { cwd: repoDir })) + .toThrow(UnsafeGitRefError) + }) + + it('does NOT write the attacker-chosen file (arbitrary-file-write PoC)', () => { + const pwned = join(repoDir, 'pwned.txt') + expect(() => runGit('log', ['-1', '--format=%ct'], [`--output=${pwned}`], { cwd: repoDir })) + .toThrow(UnsafeGitRefError) + expect(existsSync(pwned)).toBe(false) + }) + + it('resolves a legitimate ref through the --end-of-options separator', () => { + const out = runGit('log', ['-1', '--format=%ct'], ['HEAD'], { cwd: repoDir }).trim() + expect(parseInt(out, 10)).toBeGreaterThan(0) + }) + + it('rejects other shell-metacharacter refs (defense in depth)', () => { + expect(() => runGit('log', ['-1', '--format=%ct'], ['HEAD; rm -rf /'], { cwd: repoDir })) + .toThrow(UnsafeGitRefError) + expect(() => runGit('log', ['-1', '--format=%ct'], ['$(touch x)'], { cwd: repoDir })) + .toThrow(UnsafeGitRefError) + }) +}) + +describe('resolveRefToTimestamp (semantic_bisect / triage sink)', () => { + it('throws on a --output= injection ref rather than shelling out', () => { + const pwned = join(repoDir, 'pwned-bisect.txt') + expect(() => resolveRefToTimestamp(`--output=${pwned}`, repoDir)).toThrow(/Unsafe git ref/) + expect(existsSync(pwned)).toBe(false) + }) + + it('throws on a bare leading-dash ref', () => { + expect(() => resolveRefToTimestamp('--all', repoDir)).toThrow(/Unsafe git ref/) + }) + + it('still resolves a valid commit ref', () => { + const ts = resolveRefToTimestamp('HEAD', repoDir) + expect(ts).toBeGreaterThan(0) + }) + + it('still resolves an ISO date string (no git call)', () => { + const ts = resolveRefToTimestamp('2024-01-15', repoDir) + expect(ts).toBe(Math.floor(new Date('2024-01-15').getTime() / 1000)) + }) +}) diff --git a/tests/repoAuthMiddleware.test.ts b/tests/repoAuthMiddleware.test.ts new file mode 100644 index 0000000..1fc94d8 --- /dev/null +++ b/tests/repoAuthMiddleware.test.ts @@ -0,0 +1,209 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import type { Request, Response, NextFunction } from 'express' + +const mockGetRepo = vi.fn() +const mockResolveUserRepoAccess = vi.fn() + +vi.mock('../src/core/indexing/repoRegistry.js', () => ({ + getRepo: (...args: unknown[]) => mockGetRepo(...args), +})) + +vi.mock('../src/core/db/sqlite.js', () => ({ + getActiveSession: () => ({ rawDb: 'active-db' }), + getRawDb: () => 'control-plane-db', +})) + +vi.mock('../src/core/auth/grants.js', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + resolveUserRepoAccess: (...args: unknown[]) => mockResolveUserRepoAccess(...args), + // roleSatisfies is pure — use the real one. + } +}) + +import { repoAuthMiddleware, isMultiTenantMode } from '../src/server/middleware/repoAuth.js' + +function makeReq(opts: { + body?: unknown + query?: unknown + repoId?: string + userId?: number + globalKeyAuth?: boolean + repoTokenScoped?: boolean +}): Request { + return { + body: opts.body ?? {}, + query: opts.query ?? {}, + repoId: opts.repoId, + userId: opts.userId, + globalKeyAuth: opts.globalKeyAuth, + repoTokenScoped: opts.repoTokenScoped, + } as unknown as Request +} + +function makeRes(): Response { + return { + status: vi.fn().mockReturnThis(), + json: vi.fn().mockReturnThis(), + } as unknown as Response +} + +const ENV_KEYS = ['GITSEMA_MULTI_TENANT', 'GITSEMA_SERVE_KEY'] as const +const savedEnv: Record = {} + +beforeEach(() => { + mockGetRepo.mockReset() + mockResolveUserRepoAccess.mockReset() + for (const k of ENV_KEYS) savedEnv[k] = process.env[k] + for (const k of ENV_KEYS) delete process.env[k] +}) + +afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k] + else process.env[k] = savedEnv[k] + } +}) + +describe('isMultiTenantMode', () => { + it('is off by default (no key, no flag)', () => { + expect(isMultiTenantMode()).toBe(false) + }) + it('is on when GITSEMA_SERVE_KEY is set', () => { + process.env.GITSEMA_SERVE_KEY = 'secret' + expect(isMultiTenantMode()).toBe(true) + }) + it('honors an explicit GITSEMA_MULTI_TENANT=1', () => { + expect(isMultiTenantMode()).toBe(false) + process.env.GITSEMA_MULTI_TENANT = '1' + expect(isMultiTenantMode()).toBe(true) + }) + it('lets GITSEMA_MULTI_TENANT=0 disable enforcement even with a serve key', () => { + process.env.GITSEMA_SERVE_KEY = 'secret' + process.env.GITSEMA_MULTI_TENANT = '0' + expect(isMultiTenantMode()).toBe(false) + }) +}) + +describe('repoAuthMiddleware', () => { + it('passes through unchanged in single-dev mode (no key/flag)', () => { + const req = makeReq({ body: { repoId: 'private-repo' } }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(next).toHaveBeenCalledTimes(1) + expect(res.status).not.toHaveBeenCalled() + expect(mockGetRepo).not.toHaveBeenCalled() + }) + + describe('multi-tenant mode', () => { + beforeEach(() => { + process.env.GITSEMA_MULTI_TENANT = '1' + }) + + it('passes through when no repoId is addressed', () => { + const req = makeReq({}) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(next).toHaveBeenCalledTimes(1) + expect(res.status).not.toHaveBeenCalled() + }) + + it('403s an un-granted user reading a private repo', () => { + mockGetRepo.mockReturnValue({ id: 'private-repo', visibility: 'private' }) + mockResolveUserRepoAccess.mockReturnValue(undefined) + const req = makeReq({ body: { repoId: 'private-repo' }, userId: 42 }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(res.status).toHaveBeenCalledWith(403) + expect(next).not.toHaveBeenCalled() + }) + + it('403s an unauthenticated caller (no userId) reading a private repo', () => { + mockGetRepo.mockReturnValue({ id: 'private-repo', visibility: 'private' }) + const req = makeReq({ body: { repoId: 'private-repo' } }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(res.status).toHaveBeenCalledWith(403) + expect(next).not.toHaveBeenCalled() + expect(mockResolveUserRepoAccess).not.toHaveBeenCalled() + }) + + it('allows a user holding a read grant on a private repo', () => { + mockGetRepo.mockReturnValue({ id: 'private-repo', visibility: 'private' }) + mockResolveUserRepoAccess.mockReturnValue('read') + const req = makeReq({ body: { repoId: 'private-repo' }, userId: 42 }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(mockResolveUserRepoAccess).toHaveBeenCalledWith('control-plane-db', 42, 'private-repo') + expect(next).toHaveBeenCalledTimes(1) + expect(res.status).not.toHaveBeenCalled() + }) + + it('allows a user holding a higher (owner) grant', () => { + mockGetRepo.mockReturnValue({ id: 'private-repo', visibility: 'private' }) + mockResolveUserRepoAccess.mockReturnValue('owner') + const req = makeReq({ body: { repoId: 'private-repo' }, userId: 7 }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(next).toHaveBeenCalledTimes(1) + }) + + it('allows any caller to read a public repo (no grant needed)', () => { + mockGetRepo.mockReturnValue({ id: 'public-repo', visibility: 'public' }) + const req = makeReq({ body: { repoId: 'public-repo' } }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(next).toHaveBeenCalledTimes(1) + expect(res.status).not.toHaveBeenCalled() + expect(mockResolveUserRepoAccess).not.toHaveBeenCalled() + }) + + it('bypasses the grant check for global-key (admin) auth', () => { + const req = makeReq({ body: { repoId: 'private-repo' }, globalKeyAuth: true }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(next).toHaveBeenCalledTimes(1) + expect(mockGetRepo).not.toHaveBeenCalled() + }) + + it('bypasses the grant check for a legacy per-repo scoped token', () => { + const req = makeReq({ repoId: 'scoped-repo', repoTokenScoped: true }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(next).toHaveBeenCalledTimes(1) + expect(mockGetRepo).not.toHaveBeenCalled() + }) + + it('treats a missing repo mirror row as private (grant required)', () => { + mockGetRepo.mockReturnValue(null) + mockResolveUserRepoAccess.mockReturnValue(undefined) + const req = makeReq({ body: { repoId: 'ghost-repo' }, userId: 1 }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(res.status).toHaveBeenCalledWith(403) + expect(next).not.toHaveBeenCalled() + }) + + it('resolves the repoId from the query string too', () => { + mockGetRepo.mockReturnValue({ id: 'q-repo', visibility: 'private' }) + mockResolveUserRepoAccess.mockReturnValue('read') + const req = makeReq({ query: { repoId: 'q-repo' }, userId: 9 }) + const res = makeRes() + const next = vi.fn() as NextFunction + repoAuthMiddleware(req, res, next) + expect(mockResolveUserRepoAccess).toHaveBeenCalledWith('control-plane-db', 9, 'q-repo') + expect(next).toHaveBeenCalledTimes(1) + }) + }) +}) diff --git a/tests/serverRoutes.test.ts b/tests/serverRoutes.test.ts index 0219292..88201ce 100644 --- a/tests/serverRoutes.test.ts +++ b/tests/serverRoutes.test.ts @@ -1653,6 +1653,16 @@ describe('POST /api/v1/graph/deps', () => { const res = await request(app).post('/api/v1/graph/deps').send({}) expect(res.status).toBe(400) }) + + it('rejects an over-large depth (review11 §3.3 upper bound)', async () => { + const res = await request(app).post('/api/v1/graph/deps').send({ identifier: 'foo', depth: 100000 }) + expect(res.status).toBe(400) + }) + + it('accepts a depth within the bound', async () => { + const res = await request(app).post('/api/v1/graph/deps').send({ identifier: 'foo', depth: 3 }) + expect(res.status).toBe(200) + }) }) describe('POST /api/v1/graph/co-change', () => { @@ -1686,6 +1696,11 @@ describe('POST /api/v1/graph/blast-radius', () => { const res = await request(app).post('/api/v1/graph/blast-radius').send({}) expect(res.status).toBe(400) }) + + it('rejects an over-large depth (review11 §3.3 upper bound)', async () => { + const res = await request(app).post('/api/v1/graph/blast-radius').send({ symbol: 'foo', depth: 100000 }) + expect(res.status).toBe(400) + }) }) // ===========================================================================