feat: add benchmarks, examples, tests, and scripts

.claude/CLAUDE.md

@@ -16,6 +16,29 @@ cargo build --release
 
 The compiled binary will be at `./target/release/mettatron`
 
+### Profile-Guided Optimization (PGO) Build
+
+For maximum performance (20%+ improvement), use Profile-Guided Optimization:
+
+```bash
+# Step 1: Build instrumented binary
+cargo clean
+RUSTFLAGS="-Cprofile-generate=/tmp/pgo-data -Ctarget-cpu=native" cargo build --release
+
+# Step 2: Collect profile data (run representative workloads)
+./target/release/mettatron examples/mmverify/demo0/verify_demo0.metta
+
+# Step 3: Merge profile data
+llvm-profdata merge -o /tmp/pgo-data/merged.profdata /tmp/pgo-data/*.profraw
+
+# Step 4: Build optimized binary
+cargo clean
+RUSTFLAGS="-Cprofile-use=/tmp/pgo-data/merged.profdata -Ctarget-cpu=native" cargo build --release
+```
+
+PGO optimizes code layout for better instruction cache locality and branch prediction.
+Benchmark results show 20%+ speedup on heavy workloads like mmverify.
+
 ### Running the Evaluator
 
 ```bash

.config/nextest.toml

@@ -0,0 +1,21 @@
+# Nextest configuration for MeTTaTron
+# See: https://nexte.st/docs/configuration/
+
+[profile.default]
+# Default timeout for all tests - 60s with terminate after 2 slow warnings
+slow-timeout = { period = "60s", terminate-after = 2 }
+
+# Retry failed tests once (helps with flaky tests)
+retries = 1
+
+# Output configuration
+fail-fast = false
+
+[profile.default.junit]
+path = "target/nextest/default/junit.xml"
+
+[profile.ci]
+# CI profile with stricter settings
+slow-timeout = { period = "60s", terminate-after = 1 }
+retries = 0
+fail-fast = true

.gitignore

@@ -3,7 +3,11 @@ target/
 perf.data
 perf.data.*
 *.perf.data
+perf*.data
+*.data
 flamegraph*.svg
+*.so
+*.sw?
 
 examples/sandbox.rs
-dev-notes.md
+dev-notes.md

CHANGELOG.md

@@ -7,6 +7,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [Unreleased]
+
+### Added - Module System (Phase 10)
+- **Module Operations:**
+  - `include` - Load and evaluate MeTTa files with module caching
+  - `import!` - Import modules with aliasing and selective imports
+  - `bind!` - Register tokens for runtime substitution (HE-compatible)
+  - `mod-space!` - Get a module's space for querying
+  - `print-mods!` - Print information about loaded modules
+
+- **I/O Operations:**
+  - `println!` - Print values to console with format string support
+  - `trace!` - Debug printing that returns its value
+  - `nop` - No-operation (returns Unit)
+
+- **String Operations:**
+  - `repr` - Get string representation of atoms
+  - `format-args` - String formatting with `{}` placeholders
+
+- **Utility Operations:**
+  - `empty` - Return no results (empty set)
+  - `get-metatype` - Return metatype (Symbol, Expression, Grounded, Variable)
+
+- **Package Management:**
+  - TOML manifest support (`metta.toml`)
+  - Version constraint parsing (semver compatible)
+  - Dependency declaration and tracking
+
+- **CLI:**
+  - `--strict-mode` flag to control transitive imports
+
+### Added - Testing Infrastructure
+- Integration test suite for module system (`tests/module_system.rs`)
+- Test fixtures in `tests/fixtures/`
+- 88 new unit tests across module, I/O, string, and utility operations
+- Package management tests
+
+### Added - Documentation
+- Module System Guide (`docs/guides/MODULE_SYSTEM_GUIDE.md`)
+- Updated Built-in Functions Reference with 15 newly implemented functions
+
+### Changed
+- Simplified `--strict-mode` to only control transitive imports (user decision)
+- Environment now supports module caching and tokenizer integration
+- Removed `export!` special form - use `metta.toml` `[exports]` section for visibility control
+
+---
+
 ## [0.2.0] - 2025-11-27
 
 ### Added

Cargo.toml

@@ -58,6 +58,38 @@ harness = false
 name = "mork_evaluation"
 harness = false
 
+[[bench]]
+name = "simd_micro"
+harness = false
+
+[[bench]]
+name = "mmverify_benchmark"
+harness = false
+
+[[bench]]
+name = "memoization_benchmark"
+harness = false
+
+[[bench]]
+name = "bytecode_comparison"
+harness = false
+
+[[bench]]
+name = "jit_comparison"
+harness = false
+
+[[bench]]
+name = "jit_tier1_benchmarks"
+harness = false
+
+[[bench]]
+name = "jit_optimization_benchmarks"
+harness = false
+
+[[bench]]
+name = "iter_rules_bench"
+harness = false
+
 [dependencies]
 # MORK - MeTTa Optimal Reduction Kernel for pattern matching
 mork = { path = "../MORK/kernel", features = ["interning"] }
@@ -92,26 +124,25 @@ ropey = "1.6"
 # LRU cache for pattern caching optimization
 lru = "0.12"
 
-# DashMap - Concurrent HashMap for lock-free bytecode caching
-dashmap = "6.1"
-
-# GxHash - Fast, SIMD-accelerated hashing for bytecode cache keys
-gxhash = "3.4"
-
 # SmallVec - Stack-allocated vector for optimizing pattern matching bindings
 smallvec = "1.11"
 
 # lasso - Fast, thread-safe string interning for symbol tables (optional)
 # Provides O(1) symbol comparison instead of O(n) string comparison
 lasso = { version = "0.7", features = ["multi-threaded"], optional = true }
 
+# Itertools - Lazy iterator combinators for cartesian products and nondeterminism
+itertools = "0.14"
+
 # liblevenshtein - Fuzzy string matching for "Did you mean?" suggestions
 # liblevenshtein = { path = "../liblevenshtein-rust", features = ["pathmap-backend"] }
-liblevenshtein = { version = "^0.7", features = ["pathmap-backend"] }
+liblevenshtein = { version = "^0.7", features = ["pathmap-backend", "simd"] }
 
-# Module system dependencies
-serde = { version = "1.0", features = ["derive"] }
+# TOML - Package manifest parsing (metta.toml)
 toml = "0.8"
+# Serde - Serialization for package manifest
+serde = { version = "1.0", features = ["derive"] }
+# Semver - Version constraint parsing and validation
 semver = "1.0"
 
 tracing = "0.1"
@@ -120,9 +151,6 @@ tracing-subscriber = "0.3"
 # Note: jemalloc is already enabled globally via PathMap's features = ["jemalloc"]
 # PathMap sets tikv-jemallocator as the global allocator, providing 100-1000× parallel speedup
 
-# Itertools - Lazy iterator combinators for cartesian products and nondeterminism
-itertools = "0.14"
-
 # Cranelift JIT - Native code generation for hot bytecode paths
 # Provides 2-5x additional speedup over bytecode VM for arithmetic chains
 # Always included - tiered compilation is always enabled
@@ -132,6 +160,12 @@ cranelift-jit = "0.126"
 cranelift-module = "0.126"
 cranelift-native = "0.126"
 
+# DashMap - Lock-free concurrent hash map for tiered compilation cache
+dashmap = "6.1"
+
+# GxHash - Ultra-fast SIMD-accelerated hash for expression tracking
+gxhash = "3.4"
+
 # Rayon - Work-stealing thread pool for background compilation
 rayon = "1.10"
 
@@ -204,6 +238,10 @@ harness = false
 name = "e2e_throughput"
 harness = false
 
+[[bench]]
+name = "eval_profiling"
+harness = false
+
 [profile.release]
 opt-level = 3
 lto = true

README.md

@@ -24,7 +24,8 @@ MeTTaTron is a direct evaluator for the MeTTa language featuring lazy evaluation
 - **MORK/PathMap integration** - Efficient pattern matching with MORK zipper optimization
 - **REPL mode** - Interactive evaluation environment
 - **CLI and library** - Use as a command-line tool or integrate into your Rust projects
-- **Comprehensive tests** - 287 tests covering all language features
+- **Module system** - Include files, import modules, token binding, and export control
+- **Comprehensive tests** - 800+ tests covering all language features
 - **Nondeterministic evaluation** - Multiply-defined patterns with Cartesian product semantics
 
 ## Prerequisites
@@ -211,11 +212,40 @@ MeTTa uses S-expression syntax similar to Lisp:
 - **`if`** - Conditional: `(if cond then else)` with lazy branch evaluation
 - **`match`** - Pattern matching: `(match space pattern template)` queries atom space
 - **`quote`** - Prevent evaluation: `(quote expr)` returns expr unevaluated
+  - **Shorthand**: `'expr` is equivalent to `(quote expr)` *(MeTTaTron extension)*
 - **`eval`** - Force evaluation: `(eval expr)` evaluates quoted expressions
 - **`error`** - Create error: `(error msg details)`
 - **`catch`** - Error recovery: `(catch expr default)` returns default if expr errors
 - **`is-error`** - Error check: `(is-error expr)` returns true if expr is an error
 
+### Module System
+
+MeTTaTron provides a comprehensive module system for code organization:
+
+**Module Operations:**
+- **`include`** - Load and evaluate a MeTTa file: `(include "path/to/module.metta")`
+- **`import!`** - Import modules with optional aliasing: `(import! &self "module.metta")`
+- **`bind!`** - Register tokens for runtime substitution: `(bind! my-token 42)`
+- **`mod-space!`** - Get a module's space: `(mod-space! "module.metta")`
+- **`print-mods!`** - Print all loaded modules: `(print-mods!)`
+
+**Example - Module Usage:**
+```lisp
+; math_utils.metta
+(= (square $x) (* $x $x))
+
+; main.metta
+(include "math_utils.metta")
+!(square 5)  ; Returns 25
+```
+
+**Visibility Control:** Use `metta.toml` `[exports]` section to declare public symbols.
+
+**Strict Mode:**
+Run with `--strict-mode` to disable transitive imports (each module must explicitly declare dependencies).
+
+See **`docs/guides/MODULE_SYSTEM_GUIDE.md`** for complete documentation.
+
 ### Type System
 
 The evaluator includes basic type system support with type assertions and type checking:

benches/algebraic_status_duplicate_detection.rs

@@ -44,13 +44,13 @@ fn create_test_facts(n: usize) -> Vec<MettaValue> {
 fn create_test_rules(n: usize) -> Vec<Rule> {
     let mut rules = Vec::new();
     for i in 0..n {
-        rules.push(Rule {
-            lhs: MettaValue::SExpr(vec![
+        rules.push(Rule::new(
+            MettaValue::SExpr(vec![
                 MettaValue::Atom("pattern".to_string()),
                 MettaValue::Long(i as i64),
             ]),
-            rhs: MettaValue::Atom(format!("result-{}", i)),
-        });
+            MettaValue::Atom(format!("result-{}", i)),
+        ));
     }
     rules
 }

benches/branch_comparison.rs

@@ -34,13 +34,13 @@ fn generate_facts(n: usize) -> Vec<MettaValue> {
 fn generate_rules(n: usize) -> Vec<Rule> {
     let mut rules = Vec::new();
     for i in 0..n {
-        rules.push(Rule {
-            lhs: MettaValue::SExpr(vec![
+        rules.push(Rule::new(
+            MettaValue::SExpr(vec![
                 MettaValue::Atom("rule".to_string()),
                 MettaValue::Long(i as i64),
             ]),
-            rhs: MettaValue::Atom(format!("result-{}", i)),
-        });
+            MettaValue::Atom(format!("result-{}", i)),
+        ));
     }
     rules
 }
@@ -233,13 +233,13 @@ fn bench_rule_matching(c: &mut Criterion) {
 
         // Add fibonacci-like rules
         for i in 0..*rule_count {
-            env.add_rule(Rule {
-                lhs: MettaValue::SExpr(vec![
+            env.add_rule(Rule::new(
+                MettaValue::SExpr(vec![
                     MettaValue::Atom("fib".to_string()),
                     MettaValue::Long(i as i64),
                 ]),
-                rhs: MettaValue::Long((i * 2) as i64),
-            });
+                MettaValue::Long((i * 2) as i64),
+            ));
         }
 
         // Benchmark lookup

benches/bulk_operations.rs

@@ -19,13 +19,13 @@ fn generate_facts(n: usize) -> Vec<MettaValue> {
 fn generate_rules(n: usize) -> Vec<Rule> {
     let mut rules = Vec::new();
     for i in 0..n {
-        rules.push(Rule {
-            lhs: MettaValue::SExpr(vec![
+        rules.push(Rule::new(
+            MettaValue::SExpr(vec![
                 MettaValue::Atom("rule".to_string()),
                 MettaValue::Long(i as i64),
             ]),
-            rhs: MettaValue::Atom(format!("result-{}", i)),
-        });
+            MettaValue::Atom(format!("result-{}", i)),
+        ));
     }
     rules
 }