Add CI simulation test for genesis generation to finalization

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "lean-quickstart"]
 	path = lean-quickstart
 	url = git@github.com:blockblaz/lean-quickstart.git
+
 [submodule "leanSpec"]
 	path = leanSpec
 	url = https://github.com/leanEthereum/leanSpec

build.zig

@@ -63,6 +63,7 @@ pub fn build(b: *Builder) !void {
     const prover = std.meta.stringToEnum(ProverChoice, prover_option) orelse .dummy;
 
     const build_rust_lib_steps = build_rust_project(b, "rust", prover);
+    const zkvm_host_cmd = build_rust_project(b, "rust", prover);
 
     // LTO option (disabled by default for faster builds)
     const enable_lto = b.option(bool, "lto", "Enable Link Time Optimization (slower builds, smaller binaries)") orelse false;
@@ -285,6 +286,31 @@ pub fn build(b: *Builder) !void {
     zeam_beam_node.addImport("@zeam/api", zeam_api);
     zeam_beam_node.addImport("@zeam/key-manager", zeam_key_manager);
 
+    // add zeam-cli library module (for testing and external use)
+    const zeam_cli = b.addModule("@zeam/cli", .{
+        .target = target,
+        .optimize = optimize,
+        .root_source_file = b.path("pkgs/cli/src/lib.zig"),
+    });
+    zeam_cli.addImport("ssz", ssz);
+    zeam_cli.addImport("build_options", build_options_module);
+    zeam_cli.addImport("simargs", simargs);
+    zeam_cli.addImport("xev", xev);
+    zeam_cli.addImport("@zeam/database", zeam_database);
+    zeam_cli.addImport("@zeam/utils", zeam_utils);
+    zeam_cli.addImport("@zeam/params", zeam_params);
+    zeam_cli.addImport("@zeam/types", zeam_types);
+    zeam_cli.addImport("@zeam/configs", zeam_configs);
+    zeam_cli.addImport("@zeam/state-transition", zeam_state_transition);
+    zeam_cli.addImport("@zeam/state-proving-manager", zeam_state_proving_manager);
+    zeam_cli.addImport("@zeam/network", zeam_network);
+    zeam_cli.addImport("@zeam/node", zeam_beam_node);
+    zeam_cli.addImport("@zeam/api", zeam_api);
+    zeam_cli.addImport("metrics", metrics);
+    zeam_cli.addImport("multiformats", multiformats);
+    zeam_cli.addImport("enr", enr);
+    zeam_cli.addImport("yaml", yaml);
+
     const zeam_spectests = b.addModule("zeam_spectests", .{
         .target = target,
         .optimize = optimize,
@@ -392,13 +418,22 @@ pub fn build(b: *Builder) !void {
     const integration_build_options_module = integration_build_options.createModule();
     cli_integration_tests.root_module.addImport("build_options", integration_build_options_module);
 
-    // Add CLI constants module to integration tests
-    const cli_constants = b.addModule("cli_constants", .{
-        .root_source_file = b.path("pkgs/cli/src/constants.zig"),
-        .target = target,
-        .optimize = optimize,
-    });
-    cli_integration_tests.root_module.addImport("cli_constants", cli_constants);
+    // Add all dependencies needed by integration tests
+    cli_integration_tests.root_module.addImport("@zeam/cli", zeam_cli);
+    cli_integration_tests.root_module.addImport("@zeam/node", zeam_beam_node);
+    cli_integration_tests.root_module.addImport("@zeam/utils", zeam_utils);
+    cli_integration_tests.root_module.addImport("@zeam/configs", zeam_configs);
+    cli_integration_tests.root_module.addImport("@zeam/network", zeam_network);
+    cli_integration_tests.root_module.addImport("enr", enr);
+    cli_integration_tests.root_module.addImport("@zeam/state-transition", zeam_state_transition);
+    cli_integration_tests.root_module.addImport("@zeam/api", zeam_api);
+    cli_integration_tests.root_module.addImport("xev", xev);
+    cli_integration_tests.root_module.addImport("multiformats", multiformats);
+    cli_integration_tests.root_module.addImport("yaml", yaml);
+    cli_integration_tests.root_module.addImport("ssz", ssz);
+    cli_integration_tests.root_module.addImport("@zeam/types", zeam_types);
+    cli_integration_tests.root_module.addImport("@zeam/database", zeam_database);
+    addRustGlueLib(b, cli_integration_tests, target, prover);
 
     // Add error handler module to integration tests
     const error_handler_module = b.addModule("error_handler", .{
@@ -533,10 +568,11 @@ pub fn build(b: *Builder) !void {
     spectests.root_module.addImport("@zeam/state-transition", zeam_state_transition);
     spectests.root_module.addImport("ssz", ssz);
 
-    manager_tests.step.dependOn(&build_rust_lib_steps.step);
-
-    network_tests.step.dependOn(&build_rust_lib_steps.step);
-    node_tests.step.dependOn(&build_rust_lib_steps.step);
+    manager_tests.step.dependOn(&zkvm_host_cmd.step);
+    cli_tests.step.dependOn(&zkvm_host_cmd.step);
+    network_tests.step.dependOn(&zkvm_host_cmd.step);
+    node_tests.step.dependOn(&zkvm_host_cmd.step);
+    cli_integration_tests.step.dependOn(&zkvm_host_cmd.step);
     transition_tests.step.dependOn(&build_rust_lib_steps.step);
     addRustGlueLib(b, transition_tests, target, prover);
 

pkgs/cli/src/lib.zig

@@ -0,0 +1,20 @@
+// CLI package library - exposes types and functions for testing and external use
+// This module provides access to CLI internals without relative imports
+
+// Re-export types from main.zig
+pub const NodeCommand = @import("main.zig").NodeCommand;
+
+// Re-export types and functions from node.zig
+const node_module = @import("node.zig");
+pub const Node = node_module.Node;
+pub const NodeOptions = node_module.NodeOptions;
+pub const buildStartOptions = node_module.buildStartOptions;
+
+// Re-export api_server module
+pub const api_server = @import("api_server.zig");
+
+// Re-export constants module
+pub const constants = @import("constants.zig");
+
+// Re-export error handler module
+pub const error_handler = @import("error_handler.zig");

pkgs/cli/src/main.zig

@@ -488,17 +488,9 @@ fn mainInner() !void {
                 .database_path = leancmd.@"data-dir",
             };
 
+            try node.buildStartOptions(allocator, leancmd, &start_options);
             defer start_options.deinit(allocator);
 
-            node.buildStartOptions(allocator, leancmd, &start_options) catch |err| {
-                ErrorHandler.logErrorWithDetails(err, "build node start options", .{
-                    .node_id = leancmd.@"node-id",
-                    .validator_config = leancmd.validator_config,
-                    .custom_genesis = leancmd.custom_genesis,
-                });
-                return err;
-            };
-
             var lean_node: node.Node = undefined;
             lean_node.init(allocator, &start_options) catch |err| {
                 ErrorHandler.logErrorWithOperation(err, "initialize lean node");

pkgs/cli/src/node.zig

@@ -72,6 +72,7 @@ pub const NodeOptions = struct {
         allocator.free(self.bootnodes);
         allocator.free(self.validator_indices);
         allocator.free(self.local_priv_key);
+        allocator.free(self.genesis_spec.validator_pubkeys);
     }
 };
 
@@ -344,20 +345,41 @@ pub fn buildStartOptions(allocator: std.mem.Allocator, node_cmd: NodeCommand, op
     if (bootnodes.len == 0) {
         return error.InvalidNodesConfig;
     }
-    const genesis_spec = try configs.genesisConfigFromYAML(parsed_config, node_cmd.override_genesis_time);
+
+    // Parse genesis configuration (time and validator count)
+    const genesis_config = try configs.genesisConfigFromYAML(parsed_config, node_cmd.override_genesis_time);
+
+    // Generate validator keys using key manager
+    const key_manager_lib = @import("@zeam/key-manager");
+    var key_manager = try key_manager_lib.getTestKeyManager(allocator, genesis_config.validator_count, 10000);
+    defer key_manager.deinit();
+
+    // Extract all validator public keys
+    const validator_pubkeys = try key_manager.getAllPubkeys(allocator, genesis_config.validator_count);
+    errdefer allocator.free(validator_pubkeys);
+
+    // Create the full GenesisSpec
+    const genesis_spec = types.GenesisSpec{
+        .genesis_time = genesis_config.genesis_time,
+        .validator_pubkeys = validator_pubkeys,
+    };
 
     const validator_indices = try validatorIndicesFromYAML(allocator, opts.node_key, parsed_validators);
     errdefer allocator.free(validator_indices);
     if (validator_indices.len == 0) {
         return error.InvalidValidatorConfig;
     }
     const local_priv_key = try getPrivateKeyFromValidatorConfig(allocator, opts.node_key, parsed_validator_config);
+    errdefer allocator.free(local_priv_key);
+
+    const node_key_index = try nodeKeyIndexFromYaml(opts.node_key, parsed_validator_config);
 
+    // All operations succeeded, now transfer ownership (no try statements after this point)
     opts.bootnodes = bootnodes;
     opts.validator_indices = validator_indices;
     opts.local_priv_key = local_priv_key;
     opts.genesis_spec = genesis_spec;
-    opts.node_key_index = try nodeKeyIndexFromYaml(opts.node_key, parsed_validator_config);
+    opts.node_key_index = node_key_index;
 }
 
 /// Parses the nodes from a YAML configuration.

pkgs/cli/test/README.md

@@ -0,0 +1,175 @@
+# Zeam Integration Tests
+
+This directory contains integration tests for the Zeam lean consensus client.
+
+## Tests
+
+### 1. `beam_integration_test.zig`
+Tests the beam command with mock network and SSE event streaming.
+
+### 2. `genesis_to_finalization_test.zig`
+Two-node test that spawns zeam nodes directly and monitors them to finalization.
+
+### 3. `lean_quickstart_integration_test.zig`
+**Full lean-quickstart integration test (Option B)**. Uses the lean-quickstart scripts for genesis generation and node spawning.
+
+---
+
+## lean-quickstart Integration Test
+
+### Overview
+
+The `lean_quickstart_integration_test.zig` implements a full end-to-end test using the [lean-quickstart](https://github.com/blockblaz/lean-quickstart) tooling. This test:
+
+1. Uses lean-quickstart's `generate-genesis.sh` (PK's eth-beacon-genesis Docker tool)
+2. Uses lean-quickstart's `spin-node.sh` to spawn zeam nodes
+3. Monitors nodes for finalization via SSE events
+4. Cleans up using SIGTERM signals (mimics Ctrl+C)
+
+### Required Changes to lean-quickstart
+
+For the test to work properly, the following changes are needed in the `lean-quickstart` submodule:
+
+#### **File: `client-cmds/zeam-cmd.sh`**
+
+The zeam command needs these additions:
+
+```bash
+# Extract genesis time from config.yaml to ensure both nodes use same genesis time
+genesisTime=$(yq eval '.GENESIS_TIME' "$configDir/config.yaml")
+
+node_binary="$scriptDir/../zig-out/bin/zeam node \
+      --custom_genesis $configDir \
+      --validator_config $validatorConfig \
+      --override_genesis_time $genesisTime \     # ← ADD THIS
+      --network-dir $dataDir/$item/network \      # ← ADD THIS
+      --data-dir $dataDir/$item \
+      --node-id $item \
+      --node-key $configDir/$item.key \
+      --metrics_enable \                          # ← ADD THIS
+      --metrics_port $metricsPort"                # ← FIX: Use underscore not hyphen
+```
+
+**Required changes:**
+1. **Add `--override_genesis_time`**: Extract from config.yaml and pass to zeam
+2. **Add `--network-dir`**: Each node needs isolated network directory
+3. **Add `--metrics_enable`**: Required flag to enable metrics endpoint
+4. **Fix `--metrics_port`**: Must use underscore `_` not hyphen `-`
+
+#### **File: `client-cmds/zeam-cmd.sh` - Set Binary Mode**
+
+```bash
+# Use binary mode by default since lean-quickstart is a submodule in zeam repo
+node_setup="binary"  # ← Change from "docker" to "binary"
+```
+
+#### **File: `spin-node.sh` - Optional Improvements**
+
+These are optional but recommended:
+
+1. **Make sudo optional** (line 102-110):
+```bash
+# Only use sudo if explicitly requested
+if [ -n "$useSudo" ]; then
+    cmd="sudo rm -rf $itemDataDir/*"
+else
+    cmd="rm -rf $itemDataDir/*"
+fi
+```
+
+2. **Use $scriptDir for relative paths** (line 113, 120):
+```bash
+source "$scriptDir/parse-vc.sh"            # ← Add $scriptDir/
+sourceCmd="source $scriptDir/client-cmds/$client-cmd.sh"  # ← Add $scriptDir/
+```
+
+### Requirements
+
+1. **Docker**: For PK's `eth-beacon-genesis` tool (genesis generation)
+2. **yq**: YAML processor for parsing configuration
+3. **zeam binary**: Must be built (`zig build`) before running test
+   - Binary should be at `zig-out/bin/zeam`
+4. **bash**: The lean-quickstart scripts use bash
+
+### Platform Compatibility
+
+#### **Linux (GitPod, CI)**
+✅ **Fully working** - All tests pass including finalization
+
+#### **macOS**
+⚠️ **Partial support** - Tests run but validator activation issue:
+- Genesis generation: ✅ Works
+- Node spawning: ✅ Works
+- Node connectivity: ✅ Works
+- Validator activation: ❌ **Only validator 0 activates, validator 1 doesn't**
+- Finalization: ❌ Fails (due to only 50% stake active)
+
+**macOS Issue:**
+This appears to be a platform-specific bug in zeam's validator initialization or libp2p networking layer. Both `genesis_to_finalization_test.zig` and `lean_quickstart_integration_test.zig` exhibit the same behavior on macOS.
+
+**Known symptoms on macOS:**
+- Only validator index 0 produces blocks and votes
+- Validator index 1 never appears in logs
+- No justification beyond genesis
+- No finalization occurs
+- "Address already in use" panics may occur
+
+**Workaround:** Run tests on Linux for full functionality.
+
+### Running the Test
+
+```bash
+# Build zeam first
+zig build
+
+# Run all integration tests
+zig build simtest
+
+# On Linux, expect: All tests pass including finalization
+# On macOS, expect: Tests run but timeout (validator issue)
+```
+
+### Test Configuration
+
+The test creates a network with:
+- 2 validators (1 per node)
+- Metrics ports: 9669 (zeam_0), 9670 (zeam_1)
+- QUIC ports: 9100 (zeam_0), 9101 (zeam_1)
+- 600 second timeout for finalization
+- Test directory: `test_lean_quickstart_network/`
+
+### Architecture
+
+```
+Test Process
+  ├─ Generate genesis via generate-genesis.sh (Docker)
+  ├─ Spawn zeam_0 via spin-node.sh
+  │    └─ spin-node.sh spawns zeam binary in background
+  │         └─ zeam node runs with validator 0
+  ├─ Spawn zeam_1 via spin-node.sh
+  │    └─ spin-node.sh spawns zeam binary in background
+  │         └─ zeam node runs with validator 1
+  ├─ Monitor finalization via SSE events
+  └─ Cleanup via SIGTERM (triggers script trap)
+```
+
+### Key Implementation Details
+
+1. **Working Directory**: Process.Child.cwd set to `lean-quickstart/` so relative paths work
+2. **Environment**: `NETWORK_DIR` passed as relative path (`../test_network`)
+3. **Signal Handling**: Uses SIGTERM (not SIGKILL) to trigger lean-quickstart's cleanup trap
+4. **Process Management**: Custom `NodeProcess` struct manages child process lifecycle
+
+### Known Issues
+
+1. **macOS validator activation**: Only validator 0 active (platform-specific zeam bug)
+2. **bash compatibility**: `wait -n` on line 173 of spin-node.sh fails on macOS bash 3.2 (but non-fatal)
+3. **SSE connection crashes**: Race condition in api_server.zig when connection closes
+
+### Future Work
+
+- [ ] Debug macOS validator activation issue
+- [ ] Fix SSE connection close race condition  
+- [ ] Consider adding conditional test execution based on platform
+- [ ] Add metrics to track validator participation percentage
+