diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f5a12a..f6d39f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,7 +88,7 @@ jobs: sleep 30 publish llmem-index sleep 30 - publish llmem-cli + publish llmem sleep 30 publish llmem-server @@ -150,7 +150,7 @@ jobs: if [ "${{ matrix.cross }}" = "true" ]; then BUILD_CMD="cross" fi - $BUILD_CMD build --release --target ${{ matrix.target }} -p llmem-cli + $BUILD_CMD build --release --target ${{ matrix.target }} -p llmem $BUILD_CMD build --release --target ${{ matrix.target }} -p llmem-server shell: bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45c04b3..e436f37 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,39 @@ Conventional commits via `sr commit`: Scope by crate: `feat(core):`, `feat(cli):`, `feat(server):`, `docs(spec):`. +## Testing + +### Unit and integration tests + +Run the full test suite with: + +```bash +cargo test --workspace +# or +just test +``` + +Tests are colocated with their source in `#[cfg(test)]` modules, except for the CLI which uses a dedicated file at `crates/llmem-cli/tests/integration.rs`. + +| Crate | Tests | What is covered | +|-------|-------|-----------------| +| `llmem-core` | 21 | Config TOML roundtrip and dot-notation get/set; embedding store binary format and hash-based change detection; inbox capacity eviction and JSON persistence; MEMORY.md index parsing, search, and save; memory file frontmatter parsing and markdown roundtrip; `FileBackend` store/get/remove/list | +| `llmem-index` | 16 | HNSW insert, remove, save/load, and recall ≥ 90% on 200 vectors; IVF-Flat insert, remove, save/load, and recall ≥ 85% on 200 vectors; cosine similarity, dot product, L2 distance, and normalization; tree-sitter Rust chunking and language-extension mapping | +| `llmem-quant` | 39 | Lloyd-Max codebook structure and scalar quantize/dequantize at 1–4 bits; bit-packing roundtrip for 1–4 bits including non-byte-aligned counts; `TurboQuantMse` roundtrip MSE, norm preservation, zero vector, and empirical MSE against theoretical bound; `TurboQuantProd` unbiased inner-product property and fast estimate; QJL determinism and unbiased inner-product property; rotation orthogonality, forward/inverse roundtrip, and norm preservation; compressed embedding store save/load for MSE and Prod variants | +| `llmem-server` | 6 | HTTP handler tests using `tower::ServiceExt::oneshot` (no network): `/health` status and version fields; `/search` empty state, text matching, `top_k` truncation, and level filtering; `/reload` response | +| `llmem-cli` | 15 | End-to-end CLI tests that spawn the binary as a subprocess: `init`, `memorize` (with type, name, and stdin JSON), `note` (inbox capacity enforcement), `remember` (match and no-match), `reflect`, `consolidate` (dry-run and real), `forget` (success and nonexistent), `ctx switch`/`show`, `config init`/`get`/`set` | + +### End-to-end validation + +`scripts/validate.sh` runs every CLI command and the RAG server against a clean, isolated `$HOME` directory and reports pass/fail with timing for each operation. It requires release binaries built beforehand: + +```bash +cargo build --release +bash scripts/validate.sh +``` + +The script covers all command groups in order — `init`, `config`, `memorize`, `note`, `remember`, `reflect`, `learn`, `consolidate`, `forget`, `ctx`, and the server endpoints (`/health`, `/search`, `/reload`) — and exits with the number of failures as its exit code. + ## Pull Requests 1. Fork the repository diff --git a/Cargo.lock b/Cargo.lock index 54be7c5..124ee09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -180,6 +186,12 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.57" @@ -210,6 +222,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.0" @@ -256,6 +295,17 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "cookie" version = "0.18.1" @@ -300,6 +350,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -325,6 +411,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "deranged" version = "0.5.8" @@ -375,6 +467,18 @@ dependencies = [ "litrs", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "equivalent" version = "1.0.2" @@ -510,6 +614,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -531,6 +646,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "http" version = "1.4.0" @@ -772,12 +893,45 @@ dependencies = [ "serde_core", ] +[[package]] +name = "insta" +version = "1.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99322078b2c076829a1db959d49da554fabc4342257fc0ba5a070a1eb3a01cd8" +dependencies = [ + "console", + "once_cell", + "serde", + "similar", + "tempfile", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -840,24 +994,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] -name = "llmem-cli" -version = "0.1.0" +name = "llmem" +version = "0.2.0" dependencies = [ "anyhow", "chrono", "clap", + "insta", "llmem-core", "llmem-index", "serde", "serde_json", + "tempfile", "toml", ] [[package]] name = "llmem-core" -version = "0.1.0" +version = "0.2.0" dependencies = [ "dirs", + "insta", "serde", "serde_json", "serde_yaml", @@ -869,9 +1026,11 @@ dependencies = [ [[package]] name = "llmem-index" -version = "0.1.0" +version = "0.2.0" dependencies = [ + "criterion", "ignore", + "insta", "llmem-core", "serde", "tempfile", @@ -886,8 +1045,10 @@ dependencies = [ [[package]] name = "llmem-quant" -version = "0.1.0" +version = "0.2.0" dependencies = [ + "criterion", + "insta", "rand", "rand_distr", "serde", @@ -897,15 +1058,20 @@ dependencies = [ [[package]] name = "llmem-server" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "axum", + "http", + "http-body-util", + "insta", "llmem-core", "llmem-index", "serde", "serde_json", + "tempfile", "tokio", + "tower", ] [[package]] @@ -990,6 +1156,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "option-ext" version = "0.2.0" @@ -1037,6 +1209,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -1140,6 +1340,26 @@ dependencies = [ "rand", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1395,6 +1615,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "slab" version = "0.4.12" @@ -1543,6 +1769,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.50.0" @@ -1935,6 +2171,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index 813e368..6b1478d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,3 +49,5 @@ tree-sitter-go = "0.23" # Testing tempfile = "3" +criterion = { version = "0.5", features = ["html_reports"] } +insta = { version = "1", features = ["json", "yaml"] } diff --git a/README.md b/README.md index f5b3872..919c0e6 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,76 @@ curl "http://localhost:3179/reload" # hot-reload after context switch See the full [Specification](SPECIFICATION.md) for details on file format, dynamic loading, precedence rules, and integration guides. +## Benchmarks + + +### Distance Functions + +| Function | 32-d | 128-d | 384-d | +|---|---|---|---| +| `cosine_similarity` | 12 ns | 59 ns | 207 ns | +| `dot_product` | 4 ns | 28 ns | 120 ns | +| `l2_distance_squared` | 5 ns | 30 ns | 125 ns | +| `normalize` | 18 ns | 82 ns | 239 ns | + +### HNSW Index (500 vectors, dim=32) + +| Operation | Time | +|---|---| +| Build (500 inserts) | 32.7 ms | +| Search top-1 | 15.2 µs | +| Search top-10 | 15.2 µs | +| Search top-50 | 15.2 µs | +| Save to disk | 91 µs | +| Load from disk | 85 µs | + +### IVF-Flat Index (500 vectors, dim=32) + +| Operation | Time | +|---|---| +| Train (k-means, 16 clusters) | 2.2 ms | +| Search top-1 | 11.9 µs | +| Search top-10 | 12.0 µs | +| Search top-50 | 12.1 µs | +| Save to disk | 66 µs | +| Load from disk | 57 µs | + +### TurboQuant MSE (dim=128) + +| Bit-width | Quantize | Dequantize | +|---|---|---| +| 1-bit | 3.9 µs | 991 ns | +| 2-bit | 3.9 µs | 988 ns | +| 3-bit | 3.9 µs | 997 ns | +| 4-bit | 4.1 µs | 998 ns | + +### TurboQuant Prod (dim=128) + +| Bit-width | Quantize | Dequantize | IP Estimate | +|---|---|---|---| +| 2-bit | 116 µs | 141 µs | 111 µs | +| 3-bit | 115 µs | 111 µs | 111 µs | +| 4-bit | 115 µs | 111 µs | 112 µs | + +### Bit Packing + +| Operation | 128x2b | 384x2b | 384x4b | +|---|---|---|---| +| Pack | 161 ns | 539 ns | 264 ns | +| Unpack | 90 ns | 270 ns | 241 ns | + +> Measured on Apple Silicon (M-series) with `cargo bench`. Run `cargo bench` to reproduce. + + +## Testing + +```bash +just test # cargo test --workspace +bash scripts/validate.sh # full E2E validation (requires release build) +``` + +See [CONTRIBUTING.md](CONTRIBUTING.md#testing) for what each test suite covers and per-crate test counts. + ## Agent Skill This repo's conventions are available as portable agent skills in [`skills/`](skills/), following the [Agent Skills Specification](https://agentskills.io/specification). diff --git a/crates/llmem-cli/Cargo.toml b/crates/llmem-cli/Cargo.toml index 6016d61..5619c1a 100644 --- a/crates/llmem-cli/Cargo.toml +++ b/crates/llmem-cli/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "llmem-cli" +name = "llmem" version = "0.2.0" edition.workspace = true license.workspace = true @@ -24,3 +24,8 @@ serde = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } chrono = { version = "0.4", features = ["serde"] } + +[dev-dependencies] +tempfile = { workspace = true } +serde_json = { workspace = true } +insta = { workspace = true } diff --git a/crates/llmem-cli/src/main.rs b/crates/llmem-cli/src/main.rs index fcf4640..00a12a4 100644 --- a/crates/llmem-cli/src/main.rs +++ b/crates/llmem-cli/src/main.rs @@ -694,6 +694,7 @@ fn run(cli: Cli) -> Result<()> { let days_since_access = fm .last_accessed .as_ref() + .or(fm.created_at.as_ref()) .and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok()) .map(|dt| (now - dt.to_utc()).num_days().max(0) as u64) .unwrap_or(u64::MAX); diff --git a/crates/llmem-cli/tests/integration.rs b/crates/llmem-cli/tests/integration.rs new file mode 100644 index 0000000..b4fabad --- /dev/null +++ b/crates/llmem-cli/tests/integration.rs @@ -0,0 +1,662 @@ +use insta::assert_json_snapshot; +use serde_json::Value; +use std::path::PathBuf; +use std::process::Command; + +/// Get the path to the llmem binary built by cargo. +fn llmem_bin() -> PathBuf { + let mut path = PathBuf::from(env!("CARGO_BIN_EXE_llmem")); + // Fallback: if the macro doesn't resolve, try target/debug + if !path.exists() { + path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/debug/llmem"); + } + path +} + +/// Run llmem with the given args in an isolated HOME directory. +/// Returns (stdout as JSON, exit code). +fn run(home: &std::path::Path, args: &[&str]) -> (Value, i32) { + let output = Command::new(llmem_bin()) + .args(args) + .env("HOME", home) + .output() + .expect("failed to execute llmem"); + + let code = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout); + let json: Value = + serde_json::from_str(&stdout).unwrap_or_else(|_| panic!("invalid JSON output: {stdout}")); + (json, code) +} + +#[test] +fn init_creates_memory_directory() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("myproject"); + std::fs::create_dir_all(&project).unwrap(); + + let (json, code) = run(&home, &["init", "--root", project.to_str().unwrap()]); + assert_eq!(code, 0); + assert_eq!(json["ok"], true); + assert_eq!(json["data"]["level"], "project"); + + // Verify MEMORY.md was created + let mem_dir = PathBuf::from(json["data"]["path"].as_str().unwrap()); + assert!(mem_dir.join("MEMORY.md").exists()); +} + +#[test] +fn memorize_creates_memory_file() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + // Init first + run(&home, &["init", "--root", project.to_str().unwrap()]); + + // Memorize + let (json, code) = run( + &home, + &[ + "memorize", + "always use tests", + "--root", + project.to_str().unwrap(), + ], + ); + assert_eq!(code, 0); + assert_eq!(json["ok"], true); + assert_eq!(json["data"]["action"], "created"); + assert!( + json["data"]["file"] + .as_str() + .unwrap() + .starts_with("feedback_") + ); +} + +#[test] +fn memorize_with_type_and_name() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + + let (json, code) = run( + &home, + &[ + "memorize", + "user prefers vim", + "-t", + "user", + "--name", + "vim-preference", + "--root", + project.to_str().unwrap(), + ], + ); + assert_eq!(code, 0); + assert_eq!(json["data"]["file"], "user_vim-preference.md"); +} + +#[test] +fn note_adds_to_inbox() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + + let (json, code) = run( + &home, + &["note", "check logging", "--root", project.to_str().unwrap()], + ); + assert_eq!(code, 0); + assert_eq!(json["data"]["inbox_size"], 1); + assert_eq!(json["data"]["capacity"], 7); +} + +#[test] +fn remember_finds_memorized_content() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + run( + &home, + &[ + "memorize", + "prefer rust for cli tools", + "--root", + project.to_str().unwrap(), + ], + ); + + let (json, code) = run( + &home, + &[ + "remember", + "rust", + "--level", + "project", + "--root", + project.to_str().unwrap(), + ], + ); + assert_eq!(code, 0); + let memories = json["data"]["memories"].as_array().unwrap(); + assert_eq!(memories.len(), 1); + assert!(memories[0]["body"].as_str().unwrap().contains("rust")); +} + +#[test] +fn remember_returns_empty_for_no_match() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + + let (json, code) = run( + &home, + &[ + "remember", + "nonexistent", + "--level", + "project", + "--root", + project.to_str().unwrap(), + ], + ); + assert_eq!(code, 0); + assert_eq!(json["data"]["memories"].as_array().unwrap().len(), 0); +} + +#[test] +fn reflect_shows_memories_and_inbox() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + run( + &home, + &[ + "memorize", + "prefer rust", + "--root", + project.to_str().unwrap(), + ], + ); + run( + &home, + &["note", "todo item", "--root", project.to_str().unwrap()], + ); + + let (json, code) = run(&home, &["reflect", "--root", project.to_str().unwrap()]); + assert_eq!(code, 0); + assert_eq!(json["data"]["memories"].as_array().unwrap().len(), 1); + assert_eq!(json["data"]["inbox"]["size"], 1); +} + +#[test] +fn consolidate_promotes_inbox_items() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + run( + &home, + &[ + "note", + "important finding", + "--root", + project.to_str().unwrap(), + ], + ); + + // Dry run first + let (json, code) = run( + &home, + &[ + "consolidate", + "--dry-run", + "--root", + project.to_str().unwrap(), + ], + ); + assert_eq!(code, 0); + assert_eq!(json["data"]["promoted"], 1); + assert_eq!(json["data"]["dry_run"], true); + + // Real consolidation + let (json, code) = run(&home, &["consolidate", "--root", project.to_str().unwrap()]); + assert_eq!(code, 0); + assert_eq!(json["data"]["promoted"], 1); + assert_eq!(json["data"]["dry_run"], false); + + // Verify inbox is now empty + let (json, _) = run(&home, &["reflect", "--root", project.to_str().unwrap()]); + assert_eq!(json["data"]["inbox"]["size"], 0); +} + +#[test] +fn forget_removes_memory() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + let (memorize_json, _) = run( + &home, + &[ + "memorize", + "temp memory", + "--root", + project.to_str().unwrap(), + ], + ); + let filename = memorize_json["data"]["file"].as_str().unwrap(); + + // Forget it + let (json, code) = run( + &home, + &["forget", filename, "--root", project.to_str().unwrap()], + ); + assert_eq!(code, 0); + assert_eq!(json["data"]["action"], "forgotten"); + + // Verify it's gone + let (json, _) = run(&home, &["reflect", "--root", project.to_str().unwrap()]); + assert_eq!(json["data"]["memories"].as_array().unwrap().len(), 0); +} + +#[test] +fn forget_nonexistent_fails() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + + let (json, code) = run( + &home, + &[ + "forget", + "nonexistent.md", + "--root", + project.to_str().unwrap(), + ], + ); + assert_eq!(code, 1); + assert_eq!(json["ok"], false); +} + +#[test] +fn ctx_switch_and_show() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + // Show before switch — should be null + let (json, code) = run(&home, &["ctx", "show"]); + assert_eq!(code, 0); + assert!(json["data"]["context"].is_null()); + + // Switch + let (json, code) = run(&home, &["ctx", "switch", project.to_str().unwrap()]); + assert_eq!(code, 0); + assert!(json["data"]["context"].as_str().unwrap().contains("proj")); + + // Show after switch + let (json, code) = run(&home, &["ctx", "show"]); + assert_eq!(code, 0); + assert!(json["data"]["context"].as_str().unwrap().contains("proj")); +} + +#[test] +fn config_init_and_get() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + + // Init config + let (json, code) = run(&home, &["config", "init"]); + assert_eq!(code, 0); + assert_eq!(json["data"]["action"], "created"); + + // Get a known key + let (json, code) = run(&home, &["config", "get", "embedding.model"]); + assert_eq!(code, 0); + assert_eq!(json["data"]["value"], "nomic-embed-text"); +} + +#[test] +fn config_set_updates_value() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + + run(&home, &["config", "init"]); + + // Set + let (json, code) = run(&home, &["config", "set", "embedding.model", "test-model"]); + assert_eq!(code, 0); + assert_eq!(json["data"]["value"], "test-model"); + + // Verify + let (json, _) = run(&home, &["config", "get", "embedding.model"]); + assert_eq!(json["data"]["value"], "test-model"); +} + +#[test] +fn memorize_stdin_json() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + + let input = serde_json::json!({ + "type": "user", + "name": "stdin-test", + "description": "test from stdin", + "body": "detailed body content", + "level": "project" + }); + + let output = Command::new(llmem_bin()) + .args([ + "memorize", + "ignored", + "--stdin", + "--root", + project.to_str().unwrap(), + ]) + .env("HOME", &home) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .and_then(|mut child| { + use std::io::Write; + child + .stdin + .take() + .unwrap() + .write_all(input.to_string().as_bytes()) + .unwrap(); + child.wait_with_output() + }) + .expect("failed to run with stdin"); + + assert!(output.status.success()); + let json: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(json["data"]["file"], "user_stdin-test.md"); +} + +#[test] +fn multiple_notes_respect_capacity() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + + // Add more notes than the default capacity (7) + for i in 0..10 { + run( + &home, + &[ + "note", + &format!("note number {i}"), + "--root", + project.to_str().unwrap(), + ], + ); + } + + let (json, _) = run(&home, &["reflect", "--root", project.to_str().unwrap()]); + // Inbox should be capped at capacity (7) + assert!(json["data"]["inbox"]["size"].as_u64().unwrap() <= 7); +} + +/// Redact dynamic fields from CLI JSON output for deterministic snapshots. +fn redact_cli_json(mut json: Value) -> Value { + // Redact timestamps and paths that vary between runs + if let Some(data) = json.get_mut("data") { + // Redact path fields + for key in ["path", "context"] { + if data.get(key).is_some_and(|v| v.is_string()) { + data[key] = Value::String("[path]".to_string()); + } + } + // Redact embedded field (depends on Ollama availability) + if data.get("embedded").is_some() { + data["embedded"] = Value::String("[env-dependent]".to_string()); + } + // Redact config show output (contains paths) + if data.get("config").is_some_and(|v| v.is_string()) { + data["config"] = Value::String("[toml]".to_string()); + } + // Redact memories array timestamps + if let Some(memories) = data.get_mut("memories") { + if let Some(arr) = memories.as_array_mut() { + for mem in arr { + for ts_key in ["last_accessed", "created_at", "indexed_at"] { + if mem.get(ts_key).is_some_and(|v| v.is_string()) { + mem[ts_key] = Value::String("[timestamp]".to_string()); + } + } + } + } + } + // Redact inbox item timestamps + if let Some(inbox) = data.get_mut("inbox") { + if let Some(items) = inbox.get_mut("items") { + if let Some(arr) = items.as_array_mut() { + for item in arr { + if item.get("created_at").is_some_and(|v| v.is_string()) { + item["created_at"] = Value::String("[timestamp]".to_string()); + } + } + } + } + } + // Redact consolidation timestamps + if let Some(actions) = data.get_mut("actions") { + if let Some(arr) = actions.as_array_mut() { + for action in arr { + for ts_key in ["created_at", "last_accessed"] { + if action.get(ts_key).is_some_and(|v| v.is_string()) { + action[ts_key] = Value::String("[timestamp]".to_string()); + } + } + } + } + } + } + json +} + +#[test] +fn snapshot_init_output() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + let (json, _) = run(&home, &["init", "--root", project.to_str().unwrap()]); + assert_json_snapshot!(redact_cli_json(json)); +} + +#[test] +fn snapshot_memorize_output() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + let (json, _) = run( + &home, + &[ + "memorize", + "always write tests", + "-t", + "feedback", + "--name", + "write-tests", + "--root", + project.to_str().unwrap(), + ], + ); + let json = redact_cli_json(json); + assert_json_snapshot!(json); +} + +#[test] +fn snapshot_note_output() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + let (json, _) = run( + &home, + &[ + "note", + "investigate logging", + "--root", + project.to_str().unwrap(), + ], + ); + assert_json_snapshot!(json); +} + +#[test] +fn snapshot_reflect_output() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + run( + &home, + &[ + "memorize", + "prefer rust", + "-t", + "feedback", + "--name", + "prefer-rust", + "--root", + project.to_str().unwrap(), + ], + ); + run( + &home, + &["note", "check logging", "--root", project.to_str().unwrap()], + ); + + let (json, _) = run(&home, &["reflect", "--root", project.to_str().unwrap()]); + assert_json_snapshot!(redact_cli_json(json)); +} + +#[test] +fn snapshot_consolidate_dry_run_output() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + run( + &home, + &[ + "note", + "important observation", + "--root", + project.to_str().unwrap(), + ], + ); + + let (json, _) = run( + &home, + &[ + "consolidate", + "--dry-run", + "--root", + project.to_str().unwrap(), + ], + ); + assert_json_snapshot!(redact_cli_json(json)); +} + +#[test] +fn snapshot_config_get_output() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + + run(&home, &["config", "init"]); + let (json, _) = run(&home, &["config", "get", "embedding.model"]); + assert_json_snapshot!(json); +} + +#[test] +fn snapshot_forget_error_output() { + let tmp = tempfile::tempdir().unwrap(); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + + run(&home, &["init", "--root", project.to_str().unwrap()]); + let (json, _) = run( + &home, + &[ + "forget", + "nonexistent.md", + "--root", + project.to_str().unwrap(), + ], + ); + assert_json_snapshot!(json); +} diff --git a/crates/llmem-cli/tests/snapshots/integration__snapshot_config_get_output.snap b/crates/llmem-cli/tests/snapshots/integration__snapshot_config_get_output.snap new file mode 100644 index 0000000..ba0a572 --- /dev/null +++ b/crates/llmem-cli/tests/snapshots/integration__snapshot_config_get_output.snap @@ -0,0 +1,11 @@ +--- +source: crates/llmem-cli/tests/integration.rs +expression: json +--- +{ + "data": { + "key": "embedding.model", + "value": "nomic-embed-text" + }, + "ok": true +} diff --git a/crates/llmem-cli/tests/snapshots/integration__snapshot_consolidate_dry_run_output.snap b/crates/llmem-cli/tests/snapshots/integration__snapshot_consolidate_dry_run_output.snap new file mode 100644 index 0000000..fcd3cd5 --- /dev/null +++ b/crates/llmem-cli/tests/snapshots/integration__snapshot_consolidate_dry_run_output.snap @@ -0,0 +1,19 @@ +--- +source: crates/llmem-cli/tests/integration.rs +expression: redact_cli_json(json) +--- +{ + "data": { + "actions": [ + { + "action": "promote", + "file": "feedback_important-observation.md", + "source": "note" + } + ], + "decayed": 0, + "dry_run": true, + "promoted": 1 + }, + "ok": true +} diff --git a/crates/llmem-cli/tests/snapshots/integration__snapshot_forget_error_output.snap b/crates/llmem-cli/tests/snapshots/integration__snapshot_forget_error_output.snap new file mode 100644 index 0000000..55421a0 --- /dev/null +++ b/crates/llmem-cli/tests/snapshots/integration__snapshot_forget_error_output.snap @@ -0,0 +1,8 @@ +--- +source: crates/llmem-cli/tests/integration.rs +expression: json +--- +{ + "error": "not found: nonexistent.md", + "ok": false +} diff --git a/crates/llmem-cli/tests/snapshots/integration__snapshot_init_output.snap b/crates/llmem-cli/tests/snapshots/integration__snapshot_init_output.snap new file mode 100644 index 0000000..8d7b343 --- /dev/null +++ b/crates/llmem-cli/tests/snapshots/integration__snapshot_init_output.snap @@ -0,0 +1,11 @@ +--- +source: crates/llmem-cli/tests/integration.rs +expression: redact_cli_json(json) +--- +{ + "data": { + "level": "project", + "path": "[path]" + }, + "ok": true +} diff --git a/crates/llmem-cli/tests/snapshots/integration__snapshot_memorize_output.snap b/crates/llmem-cli/tests/snapshots/integration__snapshot_memorize_output.snap new file mode 100644 index 0000000..0461566 --- /dev/null +++ b/crates/llmem-cli/tests/snapshots/integration__snapshot_memorize_output.snap @@ -0,0 +1,12 @@ +--- +source: crates/llmem-cli/tests/integration.rs +expression: json +--- +{ + "data": { + "action": "created", + "embedded": "[env-dependent]", + "file": "feedback_write-tests.md" + }, + "ok": true +} diff --git a/crates/llmem-cli/tests/snapshots/integration__snapshot_note_output.snap b/crates/llmem-cli/tests/snapshots/integration__snapshot_note_output.snap new file mode 100644 index 0000000..e512d97 --- /dev/null +++ b/crates/llmem-cli/tests/snapshots/integration__snapshot_note_output.snap @@ -0,0 +1,11 @@ +--- +source: crates/llmem-cli/tests/integration.rs +expression: json +--- +{ + "data": { + "capacity": 7, + "inbox_size": 1 + }, + "ok": true +} diff --git a/crates/llmem-cli/tests/snapshots/integration__snapshot_reflect_output.snap b/crates/llmem-cli/tests/snapshots/integration__snapshot_reflect_output.snap new file mode 100644 index 0000000..81bfbc9 --- /dev/null +++ b/crates/llmem-cli/tests/snapshots/integration__snapshot_reflect_output.snap @@ -0,0 +1,30 @@ +--- +source: crates/llmem-cli/tests/integration.rs +expression: redact_cli_json(json) +--- +{ + "data": { + "inbox": { + "capacity": 7, + "items": [ + { + "attention_score": 0.5, + "created_at": "[timestamp]", + "id": "check-logging", + "source": "note" + } + ], + "size": 1 + }, + "memories": [ + { + "file": "feedback_prefer-rust.md", + "source": "memorize", + "strength": 1.0, + "summary": "prefer rust", + "title": "prefer rust" + } + ] + }, + "ok": true +} diff --git a/crates/llmem-core/Cargo.toml b/crates/llmem-core/Cargo.toml index d6d0fbd..1978e72 100644 --- a/crates/llmem-core/Cargo.toml +++ b/crates/llmem-core/Cargo.toml @@ -22,3 +22,4 @@ ureq = { version = "3", features = ["json"] } [dev-dependencies] tempfile = { workspace = true } +insta = { workspace = true } diff --git a/crates/llmem-core/src/config.rs b/crates/llmem-core/src/config.rs index 4768a09..ad58d0e 100644 --- a/crates/llmem-core/src/config.rs +++ b/crates/llmem-core/src/config.rs @@ -376,4 +376,13 @@ mod tests { let expanded = expand_tilde("~/.llmem"); assert!(!expanded.to_string_lossy().starts_with("~")); } + + #[test] + fn snapshot_default_config_toml() { + let mut config = Config::default(); + // Use a fixed root so the snapshot is deterministic + config.storage.root = "~/.llmem".to_string(); + let toml_str = toml::to_string_pretty(&config).unwrap(); + insta::assert_snapshot!(toml_str); + } } diff --git a/crates/llmem-core/src/inbox.rs b/crates/llmem-core/src/inbox.rs index 112df5c..b2f2dd6 100644 --- a/crates/llmem-core/src/inbox.rs +++ b/crates/llmem-core/src/inbox.rs @@ -168,4 +168,42 @@ mod tests { assert!(inbox.is_empty()); assert_eq!(inbox.capacity, 7); } + + #[test] + fn snapshot_inbox_json() { + let mut inbox = Inbox::new(7); + inbox.push(InboxItem { + id: "prefer-rust".to_string(), + content: "Always use Rust for CLI tools".to_string(), + source: "note".to_string(), + attention_score: 0.9, + created_at: "2026-03-29T10:00:00Z".to_string(), + file_source: None, + }); + inbox.push(InboxItem { + id: "api-handler".to_string(), + content: "fn handle_request() in src/server.rs".to_string(), + source: "learn".to_string(), + attention_score: 0.6, + created_at: "2026-03-29T11:00:00Z".to_string(), + file_source: Some(FileSource { + file: "src/server.rs".to_string(), + start_line: Some(42), + end_line: Some(80), + kind: "function".to_string(), + }), + }); + inbox.last_updated = Some("2026-03-29T11:00:00Z".to_string()); + insta::assert_json_snapshot!(inbox); + } + + #[test] + fn snapshot_inbox_after_eviction() { + let mut inbox = Inbox::new(2); + inbox.push(make_item("a", 0.3)); + inbox.push(make_item("b", 0.9)); + inbox.push(make_item("c", 0.6)); + // "a" should be evicted (lowest score) + insta::assert_json_snapshot!(inbox); + } } diff --git a/crates/llmem-core/src/index.rs b/crates/llmem-core/src/index.rs index 4b0e8c8..13e72f0 100644 --- a/crates/llmem-core/src/index.rs +++ b/crates/llmem-core/src/index.rs @@ -226,4 +226,45 @@ mod tests { let reloaded = MemoryIndex::load(&dir).unwrap(); assert_eq!(reloaded.entries.len(), 1); } + + #[test] + fn snapshot_index_entry_line() { + let entry = IndexEntry { + title: "Rust Expertise".to_string(), + file: "user_rust-expertise.md".to_string(), + summary: "deep Rust knowledge, 10+ years experience".to_string(), + }; + insta::assert_snapshot!(entry.to_line()); + } + + #[test] + fn snapshot_memory_index_rendered() { + let index = MemoryIndex { + entries: vec![ + IndexEntry { + title: "Prefer Rust".to_string(), + file: "feedback_prefer-rust.md".to_string(), + summary: "use Rust for CLI tools".to_string(), + }, + IndexEntry { + title: "API Docs".to_string(), + file: "reference_api-docs.md".to_string(), + summary: "REST API at api.example.com/docs".to_string(), + }, + IndexEntry { + title: "Auth Rewrite".to_string(), + file: "project_auth-rewrite.md".to_string(), + summary: "legal compliance requires new auth middleware".to_string(), + }, + ], + dir: PathBuf::from("/tmp/test"), + }; + let rendered: String = index + .entries + .iter() + .map(|e| e.to_line()) + .collect::>() + .join("\n"); + insta::assert_snapshot!(rendered); + } } diff --git a/crates/llmem-core/src/memory.rs b/crates/llmem-core/src/memory.rs index 353f6b9..c9b83d5 100644 --- a/crates/llmem-core/src/memory.rs +++ b/crates/llmem-core/src/memory.rs @@ -194,4 +194,51 @@ Use open standards. let result = MemoryFile::parse("no frontmatter here", "bad.md"); assert!(result.is_err()); } + + #[test] + fn snapshot_memory_file_markdown() { + let mem = MemoryFile { + frontmatter: Frontmatter { + name: "prefer-rust".to_string(), + description: "Use Rust for CLI tools".to_string(), + memory_type: MemoryType::Feedback, + created_at: Some("2026-03-01T00:00:00Z".to_string()), + last_accessed: Some("2026-03-29T12:00:00Z".to_string()), + access_count: 5, + strength: 1.5, + consolidated_from: None, + source: Some("memorize".to_string()), + }, + body: "Always prefer Rust for CLI tools.\n\n**Why:** Performance and safety.\n\n**How to apply:** Default to Rust for new CLIs.".to_string(), + }; + insta::assert_snapshot!(mem.to_markdown()); + } + + #[test] + fn snapshot_memory_file_filename() { + let mem = MemoryFile { + frontmatter: Frontmatter { + name: "api-endpoint".to_string(), + description: "REST API docs".to_string(), + memory_type: MemoryType::Reference, + ..Default::default() + }, + body: String::new(), + }; + insta::assert_snapshot!(mem.filename(), @"reference_api-endpoint.md"); + } + + #[test] + fn snapshot_frontmatter_yaml() { + let fm = Frontmatter { + name: "merge-freeze".to_string(), + description: "Merge freeze for mobile release".to_string(), + memory_type: MemoryType::Project, + created_at: Some("2026-03-25T00:00:00Z".to_string()), + strength: 1.0, + source: Some("note".to_string()), + ..Default::default() + }; + insta::assert_yaml_snapshot!(fm); + } } diff --git a/crates/llmem-core/src/snapshots/llmem_core__config__tests__snapshot_default_config_toml.snap b/crates/llmem-core/src/snapshots/llmem_core__config__tests__snapshot_default_config_toml.snap new file mode 100644 index 0000000..b03048e --- /dev/null +++ b/crates/llmem-core/src/snapshots/llmem_core__config__tests__snapshot_default_config_toml.snap @@ -0,0 +1,47 @@ +--- +source: crates/llmem-core/src/config.rs +expression: toml_str +--- +[storage] +root = "~/.llmem" + +[embedding] +provider = "ollama" +host = "http://localhost:11434" +model = "nomic-embed-text" + +[recall] +budget = 2000 +priority = [ + "feedback", + "project", + "user", + "reference", +] + +[index] +max_lines = 200 + +[code] +languages = [ + "rust", + "python", + "javascript", + "go", +] +max_chunk_lines = 100 + +[quantization] +enabled = false +bits = 2 +algorithm = "mse" +temporal_weight = 0.20000000298023224 + +[consolidation] +decay_days = 90 +merge_threshold = 0.8500000238418579 +protected_access_count = 5 +max_memories = 200 + +[inbox] +capacity = 7 diff --git a/crates/llmem-core/src/snapshots/llmem_core__inbox__tests__snapshot_inbox_after_eviction.snap b/crates/llmem-core/src/snapshots/llmem_core__inbox__tests__snapshot_inbox_after_eviction.snap new file mode 100644 index 0000000..bab1f06 --- /dev/null +++ b/crates/llmem-core/src/snapshots/llmem_core__inbox__tests__snapshot_inbox_after_eviction.snap @@ -0,0 +1,23 @@ +--- +source: crates/llmem-core/src/inbox.rs +expression: inbox +--- +{ + "capacity": 2, + "items": [ + { + "id": "b", + "content": "content for b", + "source": "note", + "attention_score": 0.9, + "created_at": "2026-03-27T00:00:00Z" + }, + { + "id": "c", + "content": "content for c", + "source": "note", + "attention_score": 0.6, + "created_at": "2026-03-27T00:00:00Z" + } + ] +} diff --git a/crates/llmem-core/src/snapshots/llmem_core__inbox__tests__snapshot_inbox_json.snap b/crates/llmem-core/src/snapshots/llmem_core__inbox__tests__snapshot_inbox_json.snap new file mode 100644 index 0000000..8292ca8 --- /dev/null +++ b/crates/llmem-core/src/snapshots/llmem_core__inbox__tests__snapshot_inbox_json.snap @@ -0,0 +1,30 @@ +--- +source: crates/llmem-core/src/inbox.rs +expression: inbox +--- +{ + "capacity": 7, + "items": [ + { + "id": "prefer-rust", + "content": "Always use Rust for CLI tools", + "source": "note", + "attention_score": 0.9, + "created_at": "2026-03-29T10:00:00Z" + }, + { + "id": "api-handler", + "content": "fn handle_request() in src/server.rs", + "source": "learn", + "attention_score": 0.6, + "created_at": "2026-03-29T11:00:00Z", + "file_source": { + "file": "src/server.rs", + "start_line": 42, + "end_line": 80, + "kind": "function" + } + } + ], + "last_updated": "2026-03-29T11:00:00Z" +} diff --git a/crates/llmem-core/src/snapshots/llmem_core__index__tests__snapshot_index_entry_line.snap b/crates/llmem-core/src/snapshots/llmem_core__index__tests__snapshot_index_entry_line.snap new file mode 100644 index 0000000..6ac019f --- /dev/null +++ b/crates/llmem-core/src/snapshots/llmem_core__index__tests__snapshot_index_entry_line.snap @@ -0,0 +1,5 @@ +--- +source: crates/llmem-core/src/index.rs +expression: entry.to_line() +--- +- [Rust Expertise](user_rust-expertise.md) — deep Rust knowledge, 10+ years experience diff --git a/crates/llmem-core/src/snapshots/llmem_core__index__tests__snapshot_memory_index_rendered.snap b/crates/llmem-core/src/snapshots/llmem_core__index__tests__snapshot_memory_index_rendered.snap new file mode 100644 index 0000000..45f6231 --- /dev/null +++ b/crates/llmem-core/src/snapshots/llmem_core__index__tests__snapshot_memory_index_rendered.snap @@ -0,0 +1,7 @@ +--- +source: crates/llmem-core/src/index.rs +expression: rendered +--- +- [Prefer Rust](feedback_prefer-rust.md) — use Rust for CLI tools +- [API Docs](reference_api-docs.md) — REST API at api.example.com/docs +- [Auth Rewrite](project_auth-rewrite.md) — legal compliance requires new auth middleware diff --git a/crates/llmem-core/src/snapshots/llmem_core__memory__tests__snapshot_frontmatter_yaml.snap b/crates/llmem-core/src/snapshots/llmem_core__memory__tests__snapshot_frontmatter_yaml.snap new file mode 100644 index 0000000..25dcaea --- /dev/null +++ b/crates/llmem-core/src/snapshots/llmem_core__memory__tests__snapshot_frontmatter_yaml.snap @@ -0,0 +1,10 @@ +--- +source: crates/llmem-core/src/memory.rs +expression: fm +--- +name: merge-freeze +description: Merge freeze for mobile release +type: project +created_at: "2026-03-25T00:00:00Z" +strength: 1 +source: note diff --git a/crates/llmem-core/src/snapshots/llmem_core__memory__tests__snapshot_memory_file_markdown.snap b/crates/llmem-core/src/snapshots/llmem_core__memory__tests__snapshot_memory_file_markdown.snap new file mode 100644 index 0000000..7651335 --- /dev/null +++ b/crates/llmem-core/src/snapshots/llmem_core__memory__tests__snapshot_memory_file_markdown.snap @@ -0,0 +1,20 @@ +--- +source: crates/llmem-core/src/memory.rs +expression: mem.to_markdown() +--- +--- +name: prefer-rust +description: Use Rust for CLI tools +type: feedback +created_at: 2026-03-01T00:00:00Z +last_accessed: 2026-03-29T12:00:00Z +access_count: 5 +strength: 1.5 +source: memorize +--- + +Always prefer Rust for CLI tools. + +**Why:** Performance and safety. + +**How to apply:** Default to Rust for new CLIs. diff --git a/crates/llmem-index/Cargo.toml b/crates/llmem-index/Cargo.toml index fde6663..dbdd069 100644 --- a/crates/llmem-index/Cargo.toml +++ b/crates/llmem-index/Cargo.toml @@ -34,3 +34,9 @@ lang-go = ["dep:tree-sitter-go"] [dev-dependencies] tempfile = { workspace = true } +criterion = { workspace = true } +insta = { workspace = true } + +[[bench]] +name = "ann_benchmarks" +harness = false diff --git a/crates/llmem-index/benches/ann_benchmarks.rs b/crates/llmem-index/benches/ann_benchmarks.rs new file mode 100644 index 0000000..beec5a0 --- /dev/null +++ b/crates/llmem-index/benches/ann_benchmarks.rs @@ -0,0 +1,152 @@ +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use llmem_index::AnnIndex; +use llmem_index::distance::{cosine_similarity, dot_product, l2_distance_squared, normalize}; +use llmem_index::hnsw::HnswIndex; +use llmem_index::ivf::{IvfConfig, IvfFlatIndex}; + +fn make_vector(dim: usize, seed: f32) -> Vec { + (0..dim).map(|i| ((i as f32 + seed) * 0.1).sin()).collect() +} + +fn bench_distance_functions(c: &mut Criterion) { + let mut group = c.benchmark_group("distance"); + + for dim in [32, 128, 384] { + let a = make_vector(dim, 1.0); + let b = make_vector(dim, 2.0); + + group.bench_with_input(BenchmarkId::new("cosine", dim), &dim, |bench, _| { + bench.iter(|| cosine_similarity(&a, &b)); + }); + + group.bench_with_input(BenchmarkId::new("dot_product", dim), &dim, |bench, _| { + bench.iter(|| dot_product(&a, &b)); + }); + + group.bench_with_input(BenchmarkId::new("l2_squared", dim), &dim, |bench, _| { + bench.iter(|| l2_distance_squared(&a, &b)); + }); + + group.bench_with_input(BenchmarkId::new("normalize", dim), &dim, |bench, _| { + bench.iter(|| { + let mut v = a.clone(); + normalize(&mut v); + v + }); + }); + } + + group.finish(); +} + +fn bench_hnsw(c: &mut Criterion) { + let mut group = c.benchmark_group("hnsw"); + let dim = 32; + let n = 500; + + let vectors: Vec> = (0..n).map(|i| make_vector(dim, i as f32)).collect(); + + // Benchmark insert (build from scratch) + group.bench_function("insert_500", |bench| { + bench.iter(|| { + let mut index = HnswIndex::with_defaults(dim); + for (i, v) in vectors.iter().enumerate() { + index.insert(&format!("item_{i}"), v).unwrap(); + } + index + }); + }); + + // Build index once for search benchmarks + let mut index = HnswIndex::with_defaults(dim); + for (i, v) in vectors.iter().enumerate() { + index.insert(&format!("item_{i}"), v).unwrap(); + } + + let query = make_vector(dim, 42.0); + + for k in [1, 10, 50] { + group.bench_with_input(BenchmarkId::new("search_top_k", k), &k, |bench, &k| { + bench.iter(|| index.search(&query, k).unwrap()); + }); + } + + // Benchmark save/load + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("bench.hnsw"); + index.save(&path).unwrap(); + + group.bench_function("save_500", |bench| { + bench.iter(|| index.save(&path).unwrap()); + }); + + group.bench_function("load_500", |bench| { + bench.iter(|| HnswIndex::load_from(&path).unwrap()); + }); + + group.finish(); +} + +fn bench_ivf(c: &mut Criterion) { + let mut group = c.benchmark_group("ivf"); + let dim = 32; + let n = 500; + + let vectors: Vec> = (0..n).map(|i| make_vector(dim, i as f32)).collect(); + + // Benchmark training + group.bench_function("train_500", |bench| { + bench.iter(|| { + let mut index = IvfFlatIndex::new( + dim, + IvfConfig { + n_lists: 16, + n_probe: 10, + kmeans_iters: 20, + }, + ); + index.train(&vectors); + index + }); + }); + + // Build index for search benchmarks + let mut index = IvfFlatIndex::new( + dim, + IvfConfig { + n_lists: 16, + n_probe: 10, + kmeans_iters: 20, + }, + ); + index.train(&vectors); + for (i, v) in vectors.iter().enumerate() { + index.insert(&format!("item_{i}"), v).unwrap(); + } + + let query = make_vector(dim, 42.0); + + for k in [1, 10, 50] { + group.bench_with_input(BenchmarkId::new("search_top_k", k), &k, |bench, &k| { + bench.iter(|| index.search(&query, k).unwrap()); + }); + } + + // Benchmark save/load + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("bench.ivf"); + index.save(&path).unwrap(); + + group.bench_function("save_500", |bench| { + bench.iter(|| index.save(&path).unwrap()); + }); + + group.bench_function("load_500", |bench| { + bench.iter(|| IvfFlatIndex::load_from(&path).unwrap()); + }); + + group.finish(); +} + +criterion_group!(benches, bench_distance_functions, bench_hnsw, bench_ivf); +criterion_main!(benches); diff --git a/crates/llmem-quant/Cargo.toml b/crates/llmem-quant/Cargo.toml index 13f43c5..936e994 100644 --- a/crates/llmem-quant/Cargo.toml +++ b/crates/llmem-quant/Cargo.toml @@ -19,3 +19,9 @@ rand_distr = "0.5" [dev-dependencies] tempfile = { workspace = true } +criterion = { workspace = true } +insta = { workspace = true } + +[[bench]] +name = "quant_benchmarks" +harness = false diff --git a/crates/llmem-quant/benches/quant_benchmarks.rs b/crates/llmem-quant/benches/quant_benchmarks.rs new file mode 100644 index 0000000..56a9e3a --- /dev/null +++ b/crates/llmem-quant/benches/quant_benchmarks.rs @@ -0,0 +1,144 @@ +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use llmem_quant::mse::TurboQuantMse; +use llmem_quant::pack; +use llmem_quant::prod::TurboQuantProd; + +fn random_unit_vector(dim: usize, seed: u64) -> Vec { + // Simple deterministic pseudo-random vector using xorshift + let mut state = seed; + let mut v: Vec = (0..dim) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state as f32 / u64::MAX as f32) * 2.0 - 1.0 + }) + .collect(); + let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 0.0 { + for x in &mut v { + *x /= norm; + } + } + v +} + +fn bench_mse_quantize(c: &mut Criterion) { + let mut group = c.benchmark_group("turbo_quant_mse/quantize"); + + for dim in [64, 128, 384] { + let quant = TurboQuantMse::new(dim, 2, 42).unwrap(); + let x = random_unit_vector(dim, 7); + + group.bench_with_input(BenchmarkId::new("dim", dim), &dim, |bench, _| { + bench.iter(|| quant.quantize(&x).unwrap()); + }); + } + + group.finish(); +} + +fn bench_mse_dequantize(c: &mut Criterion) { + let mut group = c.benchmark_group("turbo_quant_mse/dequantize"); + + for dim in [64, 128, 384] { + let quant = TurboQuantMse::new(dim, 2, 42).unwrap(); + let x = random_unit_vector(dim, 7); + let q = quant.quantize(&x).unwrap(); + + group.bench_with_input(BenchmarkId::new("dim", dim), &dim, |bench, _| { + bench.iter(|| quant.dequantize(&q).unwrap()); + }); + } + + group.finish(); +} + +fn bench_mse_bits(c: &mut Criterion) { + let mut group = c.benchmark_group("turbo_quant_mse/bits"); + let dim = 128; + let x = random_unit_vector(dim, 7); + + for bits in 1..=4u8 { + let quant = TurboQuantMse::new(dim, bits, 42).unwrap(); + + group.bench_with_input(BenchmarkId::new("quantize", bits), &bits, |bench, _| { + bench.iter(|| quant.quantize(&x).unwrap()); + }); + + let q = quant.quantize(&x).unwrap(); + group.bench_with_input(BenchmarkId::new("dequantize", bits), &bits, |bench, _| { + bench.iter(|| quant.dequantize(&q).unwrap()); + }); + } + + group.finish(); +} + +fn bench_prod(c: &mut Criterion) { + let mut group = c.benchmark_group("turbo_quant_prod"); + let dim = 128; + let x = random_unit_vector(dim, 7); + let query = random_unit_vector(dim, 99); + + for bits in 2..=4u8 { + let quant = TurboQuantProd::new(dim, bits, 42, 99).unwrap(); + + group.bench_with_input(BenchmarkId::new("quantize", bits), &bits, |bench, _| { + bench.iter(|| quant.quantize(&x).unwrap()); + }); + + let q = quant.quantize(&x).unwrap(); + + group.bench_with_input(BenchmarkId::new("dequantize", bits), &bits, |bench, _| { + bench.iter(|| quant.dequantize(&q).unwrap()); + }); + + group.bench_with_input( + BenchmarkId::new("inner_product_estimate", bits), + &bits, + |bench, _| { + bench.iter(|| quant.inner_product_estimate(&query, &q).unwrap()); + }, + ); + } + + group.finish(); +} + +fn bench_pack(c: &mut Criterion) { + let mut group = c.benchmark_group("pack"); + + for (dim, bits) in [(128, 2u8), (384, 2), (384, 4)] { + let indices: Vec = (0..dim).map(|i| (i % (1 << bits)) as u8).collect(); + + group.bench_with_input( + BenchmarkId::new("pack", format!("{dim}x{bits}b")), + &dim, + |bench, _| { + bench.iter(|| pack::pack_indices(&indices, bits).unwrap()); + }, + ); + + let packed = pack::pack_indices(&indices, bits).unwrap(); + group.bench_with_input( + BenchmarkId::new("unpack", format!("{dim}x{bits}b")), + &dim, + |bench, _| { + bench.iter(|| pack::unpack_indices(&packed, bits, dim).unwrap()); + }, + ); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_mse_quantize, + bench_mse_dequantize, + bench_mse_bits, + bench_prod, + bench_pack +); +criterion_main!(benches); diff --git a/crates/llmem-server/Cargo.toml b/crates/llmem-server/Cargo.toml index bdebd93..20494f8 100644 --- a/crates/llmem-server/Cargo.toml +++ b/crates/llmem-server/Cargo.toml @@ -23,3 +23,10 @@ tokio = { workspace = true } anyhow = { workspace = true } serde_json = { workspace = true } serde = { workspace = true } + +[dev-dependencies] +tower = "0.5" +tempfile = { workspace = true } +http-body-util = "0.1" +http = "1" +insta = { workspace = true } diff --git a/crates/llmem-server/src/main.rs b/crates/llmem-server/src/main.rs index 5c65d75..d0af761 100644 --- a/crates/llmem-server/src/main.rs +++ b/crates/llmem-server/src/main.rs @@ -13,7 +13,10 @@ use std::sync::{Arc, RwLock}; struct AppState { project_index: RwLock>, global_index: RwLock>, + /// The original project root path (for display in /health). project_root: RwLock>, + /// Resolved project memory directory (config-derived from project_root). + project_mem_dir: RwLock>, } #[derive(Deserialize)] @@ -107,8 +110,8 @@ async fn search( let include_project = params.level != "global"; let include_global = params.level != "project"; - if include_project && let Some(root) = state.project_root.read().ok().and_then(|r| r.clone()) { - let dir = project_dir(&root); + if include_project && let Some(dir) = state.project_mem_dir.read().ok().and_then(|d| d.clone()) + { results.extend(search_level(&dir, "project")); } @@ -128,11 +131,16 @@ async fn reload(State(state): State>) -> Json { && let Ok(ctx) = std::fs::read_to_string(&ctx_file) { let root = PathBuf::from(ctx.trim()); + let mem_dir = project_dir(&root); + if let Ok(mut pr) = state.project_root.write() { - *pr = Some(root.clone()); + *pr = Some(root); + } + if let Ok(mut pd) = state.project_mem_dir.write() { + *pd = Some(mem_dir.clone()); } - let hnsw_path = project_dir(&root).join(".index.hnsw"); + let hnsw_path = mem_dir.join(".index.hnsw"); if hnsw_path.exists() && let Ok(idx) = HnswIndex::load_from(&hnsw_path) && let Ok(mut pi) = state.project_index.write() @@ -145,10 +153,20 @@ async fn reload(State(state): State>) -> Json { Json(serde_json::json!({"status": "reloaded"})) } +/// Build the router with the given state. Extracted for testability. +fn build_router(state: Arc) -> Router { + Router::new() + .route("/health", get(health)) + .route("/search", get(search)) + .route("/reload", get(reload)) + .with_state(state) +} + #[tokio::main] async fn main() -> Result<()> { // Load initial context let mut project_root = None; + let mut project_mem_dir = None; let mut project_hnsw = None; let mut global_hnsw = None; @@ -159,11 +177,13 @@ async fn main() -> Result<()> { && let Ok(ctx) = std::fs::read_to_string(&ctx_file) { let root = PathBuf::from(ctx.trim()); - let hnsw_path = project_dir(&root).join(".index.hnsw"); + let mem_dir = project_dir(&root); + let hnsw_path = mem_dir.join(".index.hnsw"); if hnsw_path.exists() { project_hnsw = HnswIndex::load_from(&hnsw_path).ok(); } project_root = Some(root); + project_mem_dir = Some(mem_dir); } } @@ -179,13 +199,10 @@ async fn main() -> Result<()> { project_index: RwLock::new(project_hnsw), global_index: RwLock::new(global_hnsw), project_root: RwLock::new(project_root), + project_mem_dir: RwLock::new(project_mem_dir), }); - let app = Router::new() - .route("/health", get(health)) - .route("/search", get(search)) - .route("/reload", get(reload)) - .with_state(state); + let app = build_router(state); let addr = "127.0.0.1:3179"; println!("llmem-server listening on {addr}"); @@ -195,3 +212,181 @@ async fn main() -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::extract::Request; + use http_body_util::BodyExt; + use tower::ServiceExt; + + fn test_state() -> Arc { + Arc::new(AppState { + project_index: RwLock::new(None), + global_index: RwLock::new(None), + project_root: RwLock::new(None), + project_mem_dir: RwLock::new(None), + }) + } + + fn test_state_with_root(root: PathBuf) -> Arc { + Arc::new(AppState { + project_index: RwLock::new(None), + global_index: RwLock::new(None), + project_root: RwLock::new(Some(root)), + project_mem_dir: RwLock::new(None), + }) + } + + fn test_state_with_mem_dir(root: PathBuf, mem_dir: PathBuf) -> Arc { + Arc::new(AppState { + project_index: RwLock::new(None), + global_index: RwLock::new(None), + project_root: RwLock::new(Some(root)), + project_mem_dir: RwLock::new(Some(mem_dir)), + }) + } + + async fn response_json(app: Router, uri: &str) -> serde_json::Value { + let req = Request::builder().uri(uri).body(Body::empty()).unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + serde_json::from_slice(&body).unwrap() + } + + #[tokio::test] + async fn health_returns_ok() { + let app = build_router(test_state()); + let json = response_json(app, "/health").await; + + assert_eq!(json["status"], "ok"); + assert_eq!(json["version"], env!("CARGO_PKG_VERSION")); + assert_eq!(json["project_memories"], 0); + assert_eq!(json["global_memories"], 0); + assert!(json["active_context"].is_null()); + } + + #[tokio::test] + async fn health_shows_active_context() { + let state = test_state_with_root(PathBuf::from("/tmp/test-project")); + let app = build_router(state); + let json = response_json(app, "/health").await; + + assert_eq!(json["active_context"], "/tmp/test-project"); + } + + #[tokio::test] + async fn search_empty_returns_empty_array() { + let app = build_router(test_state()); + let json = response_json(app, "/search?q=anything").await; + + assert!(json.is_array()); + assert_eq!(json.as_array().unwrap().len(), 0); + } + + #[tokio::test] + async fn search_finds_matching_memories() { + let tmp = tempfile::tempdir().unwrap(); + let project_root = tmp.path().join("myproject"); + let mem_dir = tmp.path().join("mem"); + std::fs::create_dir_all(&mem_dir).unwrap(); + + std::fs::write( + mem_dir.join("MEMORY.md"), + "- [prefer rust](feedback_prefer-rust.md) — use rust for CLI tools\n\ + - [write tests](feedback_write-tests.md) — always write tests\n", + ) + .unwrap(); + + let state = test_state_with_mem_dir(project_root, mem_dir); + let app = build_router(state); + + let json = response_json(app, "/search?q=rust").await; + let results = json.as_array().unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0]["title"], "prefer rust"); + assert_eq!(results[0]["level"], "project"); + assert_eq!(results[0]["score"], 1.0); + } + + #[tokio::test] + async fn search_respects_top_k() { + let tmp = tempfile::tempdir().unwrap(); + let project_root = tmp.path().join("proj"); + let mem_dir = tmp.path().join("mem"); + std::fs::create_dir_all(&mem_dir).unwrap(); + + std::fs::write( + mem_dir.join("MEMORY.md"), + "- [test one](feedback_test-one.md) — first test memory\n\ + - [test two](feedback_test-two.md) — second test memory\n", + ) + .unwrap(); + + let state = test_state_with_mem_dir(project_root, mem_dir); + let app = build_router(state); + + let json = response_json(app, "/search?q=test&top_k=1").await; + assert_eq!(json.as_array().unwrap().len(), 1); + } + + #[tokio::test] + async fn reload_returns_reloaded() { + let app = build_router(test_state()); + let json = response_json(app, "/reload").await; + assert_eq!(json["status"], "reloaded"); + } + + #[tokio::test] + async fn snapshot_health_no_context() { + let app = build_router(test_state()); + let mut json = response_json(app, "/health").await; + // Version changes between releases, redact it + json["version"] = serde_json::Value::String("[version]".to_string()); + insta::assert_json_snapshot!(json); + } + + #[tokio::test] + async fn snapshot_health_with_context() { + let state = test_state_with_root(PathBuf::from("/tmp/my-project")); + let app = build_router(state); + let mut json = response_json(app, "/health").await; + json["version"] = serde_json::Value::String("[version]".to_string()); + insta::assert_json_snapshot!(json); + } + + #[tokio::test] + async fn snapshot_search_with_results() { + let tmp = tempfile::tempdir().unwrap(); + let mem_dir = tmp.path().join("mem"); + std::fs::create_dir_all(&mem_dir).unwrap(); + std::fs::write( + mem_dir.join("MEMORY.md"), + "- [prefer rust](feedback_prefer-rust.md) — use rust for CLI tools\n\ + - [write tests](feedback_write-tests.md) — always write tests\n\ + - [api docs](reference_api-docs.md) — REST API documentation\n", + ) + .unwrap(); + + let state = test_state_with_mem_dir(tmp.path().to_path_buf(), mem_dir); + let app = build_router(state); + let json = response_json(app, "/search?q=test").await; + insta::assert_json_snapshot!(json); + } + + #[tokio::test] + async fn snapshot_search_empty() { + let app = build_router(test_state()); + let json = response_json(app, "/search?q=nothing").await; + insta::assert_json_snapshot!(json); + } + + #[tokio::test] + async fn snapshot_reload() { + let app = build_router(test_state()); + let json = response_json(app, "/reload").await; + insta::assert_json_snapshot!(json); + } +} diff --git a/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_health_no_context.snap b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_health_no_context.snap new file mode 100644 index 0000000..40df16c --- /dev/null +++ b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_health_no_context.snap @@ -0,0 +1,11 @@ +--- +source: crates/llmem-server/src/main.rs +expression: json +--- +{ + "active_context": null, + "global_memories": 0, + "project_memories": 0, + "status": "ok", + "version": "[version]" +} diff --git a/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_health_with_context.snap b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_health_with_context.snap new file mode 100644 index 0000000..2c1b6a4 --- /dev/null +++ b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_health_with_context.snap @@ -0,0 +1,11 @@ +--- +source: crates/llmem-server/src/main.rs +expression: json +--- +{ + "active_context": "/tmp/my-project", + "global_memories": 0, + "project_memories": 0, + "status": "ok", + "version": "[version]" +} diff --git a/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_reload.snap b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_reload.snap new file mode 100644 index 0000000..b415410 --- /dev/null +++ b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_reload.snap @@ -0,0 +1,7 @@ +--- +source: crates/llmem-server/src/main.rs +expression: json +--- +{ + "status": "reloaded" +} diff --git a/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_search_empty.snap b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_search_empty.snap new file mode 100644 index 0000000..ddc6792 --- /dev/null +++ b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_search_empty.snap @@ -0,0 +1,5 @@ +--- +source: crates/llmem-server/src/main.rs +expression: json +--- +[] diff --git a/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_search_with_results.snap b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_search_with_results.snap new file mode 100644 index 0000000..18edd3b --- /dev/null +++ b/crates/llmem-server/src/snapshots/llmem_server__tests__snapshot_search_with_results.snap @@ -0,0 +1,13 @@ +--- +source: crates/llmem-server/src/main.rs +expression: json +--- +[ + { + "file": "feedback_write-tests.md", + "level": "project", + "score": 1.0, + "summary": "always write tests", + "title": "write tests" + } +] diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..9654d3e --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,56 @@ +### Distance Functions + +| Function | 32-d | 128-d | 384-d | +|---|---|---|---| +| `cosine_similarity` | 12 ns | 59 ns | 207 ns | +| `dot_product` | 4 ns | 28 ns | 120 ns | +| `l2_distance_squared` | 5 ns | 30 ns | 125 ns | +| `normalize` | 18 ns | 82 ns | 239 ns | + +### HNSW Index (500 vectors, dim=32) + +| Operation | Time | +|---|---| +| Build (500 inserts) | 32.7 ms | +| Search top-1 | 15.2 µs | +| Search top-10 | 15.2 µs | +| Search top-50 | 15.2 µs | +| Save to disk | 91 µs | +| Load from disk | 85 µs | + +### IVF-Flat Index (500 vectors, dim=32) + +| Operation | Time | +|---|---| +| Train (k-means, 16 clusters) | 2.2 ms | +| Search top-1 | 11.9 µs | +| Search top-10 | 12.0 µs | +| Search top-50 | 12.1 µs | +| Save to disk | 66 µs | +| Load from disk | 57 µs | + +### TurboQuant MSE (dim=128) + +| Bit-width | Quantize | Dequantize | +|---|---|---| +| 1-bit | 3.9 µs | 991 ns | +| 2-bit | 3.9 µs | 988 ns | +| 3-bit | 3.9 µs | 997 ns | +| 4-bit | 4.1 µs | 998 ns | + +### TurboQuant Prod (dim=128) + +| Bit-width | Quantize | Dequantize | IP Estimate | +|---|---|---|---| +| 2-bit | 116 µs | 141 µs | 111 µs | +| 3-bit | 115 µs | 111 µs | 111 µs | +| 4-bit | 115 µs | 111 µs | 112 µs | + +### Bit Packing + +| Operation | 128x2b | 384x2b | 384x4b | +|---|---|---|---| +| Pack | 161 ns | 539 ns | 264 ns | +| Unpack | 90 ns | 270 ns | 241 ns | + +> Measured on Apple Silicon (M-series) with `cargo bench`. Run `cargo bench` to reproduce. diff --git a/scripts/validate.sh b/scripts/validate.sh new file mode 100755 index 0000000..1047635 --- /dev/null +++ b/scripts/validate.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +set -uo pipefail + +# ── llmem full system validation ───────────────────────────────────────────── +# Runs every command against a clean, isolated environment and reports +# pass/fail with timing for each operation. + +LLMEM="${LLMEM_BIN:-$(dirname "$0")/../target/release/llmem}" +SERVER="${LLMEM_SERVER_BIN:-$(dirname "$0")/../target/release/llmem-server}" + +# ── Theme ──────────────────────────────────────────────────────────────────── +BOLD="\033[1m" +DIM="\033[2m" +RESET="\033[0m" +GREEN="\033[32m" +RED="\033[31m" +CYAN="\033[36m" +WHITE="\033[97m" +BAR_FG="\033[38;5;75m" + +PASS_COUNT=0 +FAIL_COUNT=0 +TOTAL_MS=0 +declare -a RESULTS=() + +# ── Helpers ────────────────────────────────────────────────────────────────── + +now_ms() { python3 -c 'import time; print(int(time.time()*1000))'; } + +# Shared output variable — avoids subshell variable loss +OUT="" + +# run NAME CMD... — expect success +run() { + local name="$1"; shift + local t0; t0=$(now_ms) + local rc=0 + OUT=$("$@" 2>/dev/null) || rc=$? + local t1; t1=$(now_ms) + local ms=$((t1 - t0)) + TOTAL_MS=$((TOTAL_MS + ms)) + if [ "$rc" -eq 0 ]; then + PASS_COUNT=$((PASS_COUNT + 1)); RESULTS+=("PASS|${ms}|${name}") + else + FAIL_COUNT=$((FAIL_COUNT + 1)); RESULTS+=("FAIL|${ms}|${name}") + fi +} + +# run_fail NAME CMD... — expect nonzero exit +run_fail() { + local name="$1"; shift + local t0; t0=$(now_ms) + local rc=0 + OUT=$("$@" 2>/dev/null) || rc=$? + local t1; t1=$(now_ms) + local ms=$((t1 - t0)) + TOTAL_MS=$((TOTAL_MS + ms)) + if [ "$rc" -ne 0 ]; then + PASS_COUNT=$((PASS_COUNT + 1)); RESULTS+=("PASS|${ms}|${name}") + else + FAIL_COUNT=$((FAIL_COUNT + 1)); RESULTS+=("FAIL|${ms}|${name}") + fi +} + +# eq PY_EXPR EXPECTED LABEL +eq() { + local expr="$1" expected="$2" label="$3" + local actual + actual=$(echo "$OUT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps($expr))" 2>/dev/null) || actual="__PARSE_ERROR__" + if [ "$actual" = "$expected" ]; then + PASS_COUNT=$((PASS_COUNT + 1)); RESULTS+=("PASS|0|${label}") + else + FAIL_COUNT=$((FAIL_COUNT + 1)); RESULTS+=("FAIL|0|${label} (expected $expected, got $actual)") + fi +} + +# ok PY_BOOL_EXPR LABEL +ok() { + local expr="$1" label="$2" + local val + val=$(echo "$OUT" | python3 -c "import sys,json; d=json.load(sys.stdin); print('yes' if ($expr) else 'no')" 2>/dev/null) || val="no" + if [ "$val" = "yes" ]; then + PASS_COUNT=$((PASS_COUNT + 1)); RESULTS+=("PASS|0|${label}") + else + FAIL_COUNT=$((FAIL_COUNT + 1)); RESULTS+=("FAIL|0|${label}") + fi +} + +cleanup() { + [ -n "${SERVER_PID:-}" ] && kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + rm -rf "$TMPDIR" +} +trap cleanup EXIT + +# ── Setup ──────────────────────────────────────────────────────────────────── + +TMPDIR=$(mktemp -d) +export HOME="$TMPDIR/home" +mkdir -p "$HOME" +PROJECT="$TMPDIR/myproject" +mkdir -p "$PROJECT" +REPO_ROOT="$(cd "$(dirname "$0")/.."; pwd)" +SERVER_PID="" + +echo "" +echo -e "${BOLD}${CYAN} llmem${RESET}${BOLD} system validation${RESET}" +echo -e "${DIM} ────────────────────────────────────────${RESET}" +echo "" + +# ── 1. Init ────────────────────────────────────────────────────────────────── +echo -e "${WHITE} init${RESET}" + +run "init project" "$LLMEM" init --root "$PROJECT" +eq "d['data']['level']" '"project"' "init: level is project" +eq "d['ok']" 'true' "init: ok is true" + +run "init global" "$LLMEM" init --global +eq "d['data']['level']" '"global"' "init global: level is global" + +# ── 2. Config ──────────────────────────────────────────────────────────────── +echo -e "${WHITE} config${RESET}" + +run "config init" "$LLMEM" config init +eq "d['data']['action']" '"created"' "config init: action is created" + +run "config get" "$LLMEM" config get embedding.model +eq "d['data']['value']" '"nomic-embed-text"' "config get: default model" + +run "config set" "$LLMEM" config set embedding.model "test-embed" +eq "d['data']['value']" '"test-embed"' "config set: value updated" + +run "config get verify" "$LLMEM" config get embedding.model +eq "d['data']['value']" '"test-embed"' "config set: persisted" + +# Restore for embedding to work +"$LLMEM" config set embedding.model "nomic-embed-text" >/dev/null 2>/dev/null + +run "config show" "$LLMEM" config show +ok "len(d['data']['config']) > 100" "config show: non-empty" + +run "config path" "$LLMEM" config path +ok "'config.toml' in d['data']['path']" "config path: ends in config.toml" + +# ── 3. Memorize ────────────────────────────────────────────────────────────── +echo -e "${WHITE} memorize${RESET}" + +run "memorize feedback" "$LLMEM" memorize "always write tests" --root "$PROJECT" +eq "d['data']['action']" '"created"' "memorize: action is created" +ok "d['data']['file'].startswith('feedback_')" "memorize: file prefix" + +run "memorize user" "$LLMEM" memorize "prefers rust" -t user --name "lang-pref" --root "$PROJECT" +eq "d['data']['file']" '"user_lang-pref.md"' "memorize user: correct filename" + +run "memorize project" "$LLMEM" memorize "merge freeze march 5" -t project --name "freeze" --root "$PROJECT" +eq "d['data']['file']" '"project_freeze.md"' "memorize project: correct filename" + +run "memorize reference" "$LLMEM" memorize "see Linear INGEST board" -t reference --name "linear-board" --root "$PROJECT" +eq "d['data']['file']" '"reference_linear-board.md"' "memorize reference: correct filename" + +# Upsert same name +run "memorize upsert" "$LLMEM" memorize "prefers rust and go" -t user --name "lang-pref" --root "$PROJECT" +eq "d['data']['action']" '"updated"' "memorize upsert: action is updated" + +# Stdin +run "memorize stdin" sh -c "echo '{\"type\":\"feedback\",\"name\":\"stdin-mem\",\"description\":\"from stdin\",\"body\":\"body content\",\"level\":\"project\"}' | '$LLMEM' memorize ignored --stdin --root '$PROJECT'" +eq "d['data']['file']" '"feedback_stdin-mem.md"' "memorize stdin: correct file" + +# ── 4. Note ────────────────────────────────────────────────────────────────── +echo -e "${WHITE} note${RESET}" + +run "note 1" "$LLMEM" note "check logging" --root "$PROJECT" +eq "d['data']['inbox_size']" '1' "note: inbox_size is 1" + +run "note 2" "$LLMEM" note "review auth middleware" --root "$PROJECT" +eq "d['data']['inbox_size']" '2' "note: inbox_size is 2" + +# Fill inbox beyond capacity +for i in $(seq 3 10); do + "$LLMEM" note "note number $i" --root "$PROJECT" >/dev/null 2>/dev/null +done +run "note capacity" "$LLMEM" reflect --root "$PROJECT" +ok "d['data']['inbox']['size'] <= 7" "note: respects capacity (<=7)" + +# ── 5. Remember ────────────────────────────────────────────────────────────── +echo -e "${WHITE} remember${RESET}" + +run "remember rust" "$LLMEM" remember "rust" --level project --root "$PROJECT" +ok "len(d['data']['memories']) >= 1" "remember: finds rust memories" +ok "d['data']['token_estimate'] >= 0" "remember: has token estimate" + +run "remember obscure" "$LLMEM" remember "xyzzy999" --level project --root "$PROJECT" +ok "'memories' in d['data']" "remember: returns memories array" + +# ── 6. Reflect ─────────────────────────────────────────────────────────────── +echo -e "${WHITE} reflect${RESET}" + +run "reflect" "$LLMEM" reflect --root "$PROJECT" +ok "len(d['data']['memories']) == 5" "reflect: 5 memories" +ok "d['data']['inbox']['size'] > 0" "reflect: inbox non-empty" + +# ── 7. Learn ───────────────────────────────────────────────────────────────── +echo -e "${WHITE} learn${RESET}" + +run "learn codebase" "$LLMEM" learn "$REPO_ROOT" --root "$PROJECT" +ok "d['data']['chunks'] > 0" "learn: extracted chunks" +ok "d['data']['files'] > 0" "learn: found files" + +# ── 8. Consolidate ─────────────────────────────────────────────────────────── +echo -e "${WHITE} consolidate${RESET}" + +run "consolidate dry-run" "$LLMEM" consolidate --dry-run --root "$PROJECT" +eq "d['data']['dry_run']" 'true' "consolidate dry-run: flag set" +ok "d['data']['promoted'] > 0" "consolidate dry-run: would promote" + +run "consolidate" "$LLMEM" consolidate --root "$PROJECT" +eq "d['data']['dry_run']" 'false' "consolidate: not dry run" +ok "d['data']['promoted'] > 0" "consolidate: promoted items" +eq "d['data']['decayed']" '0' "consolidate: no immediate decay (bug fix)" + +# Verify inbox drained +run "post-consolidate reflect" "$LLMEM" reflect --root "$PROJECT" +eq "d['data']['inbox']['size']" '0' "consolidate: inbox drained" +ok "len(d['data']['memories']) > 5" "consolidate: memories grew" + +# ── 9. Forget ──────────────────────────────────────────────────────────────── +echo -e "${WHITE} forget${RESET}" + +run "forget" "$LLMEM" forget feedback_stdin-mem.md --root "$PROJECT" +eq "d['data']['action']" '"forgotten"' "forget: action is forgotten" + +run_fail "forget nonexistent" "$LLMEM" forget nonexistent.md --root "$PROJECT" +eq "d['ok']" 'false' "forget nonexistent: ok is false" + +# ── 10. Context ────────────────────────────────────────────────────────────── +echo -e "${WHITE} context${RESET}" + +run "ctx show (empty)" "$LLMEM" ctx show +ok "d['data']['context'] is None" "ctx show: null before switch" + +run "ctx switch" "$LLMEM" ctx switch "$PROJECT" +ok "'myproject' in d['data']['context']" "ctx switch: set to project" + +run "ctx show (after)" "$LLMEM" ctx show +ok "d['data']['context'] is not None" "ctx show: non-null after switch" + +# ── 11. Server ─────────────────────────────────────────────────────────────── +echo -e "${WHITE} server${RESET}" + +"$SERVER" >/dev/null 2>&1 & +SERVER_PID=$! +sleep 1 + +run "server /health" curl -sf http://127.0.0.1:3179/health +eq "d['status']" '"ok"' "server health: status ok" +ok "'version' in d" "server health: has version" + +run "server /search match" curl -sf "http://127.0.0.1:3179/search?q=rust" +ok "len(d) >= 1" "server search: finds results" + +run "server /search empty" curl -sf "http://127.0.0.1:3179/search?q=xyzzy999" +eq "len(d)" '0' "server search: empty for no match" + +run "server /search top_k" curl -sf "http://127.0.0.1:3179/search?q=a&top_k=1" +ok "len(d) <= 1" "server search: respects top_k" + +run "server /search level" curl -sf "http://127.0.0.1:3179/search?q=rust&level=project" +ok "all(r['level']=='project' for r in d)" "server search: level filter" + +run "server /reload" curl -sf http://127.0.0.1:3179/reload +eq "d['status']" '"reloaded"' "server reload: status reloaded" + +kill "$SERVER_PID" 2>/dev/null || true +wait "$SERVER_PID" 2>/dev/null || true +SERVER_PID="" + +# ── Report ─────────────────────────────────────────────────────────────────── + +echo "" +echo -e "${DIM} ────────────────────────────────────────────────────────────${RESET}" +echo "" + +# Find max name length for alignment +MAX_NAME=0 +for r in "${RESULTS[@]}"; do + IFS='|' read -r status ms name <<< "$r" + len=${#name} + (( len > MAX_NAME )) && MAX_NAME=$len +done + +# Calculate max ms for bar scaling +MAX_MS=1 +for r in "${RESULTS[@]}"; do + IFS='|' read -r status ms name <<< "$r" + (( ms > MAX_MS )) && MAX_MS=$ms +done + +BAR_WIDTH=24 + +for r in "${RESULTS[@]}"; do + IFS='|' read -r status ms name <<< "$r" + + if [ "$status" = "PASS" ]; then + icon="${GREEN}${RESET}" + else + icon="${RED}${RESET}" + fi + + # Timing bar (only for bench'd items with ms > 0) + bar="" + timing="" + if [ "$ms" -gt 0 ]; then + filled=$(( (ms * BAR_WIDTH) / MAX_MS )) + (( filled < 1 )) && filled=1 + empty=$((BAR_WIDTH - filled)) + bar="${BAR_FG}" + for ((b=0; b