From a7c3b5b959a1e473a73408f4d03bc8ac07cd5766 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Thu, 2 Apr 2026 16:59:03 +0800 Subject: [PATCH] feat: add Shadow network simulator automation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run-shadow.sh: orchestrator (genesis → shadow.yaml → shadow) - generate-shadow-yaml.sh: generates shadow.yaml from validator-config.yaml using each client's client-cmds/-cmd.sh - shadow-devnet/genesis/validator-config.yaml: 4 zeam nodes with Shadow IPs - generate-genesis.sh: add --genesis-time flag for fixed timestamps - README: add Shadow Network Simulator section --- README.md | 70 +++++++ generate-genesis.sh | 49 +++-- generate-shadow-yaml.sh | 207 ++++++++++++++++++++ run-shadow.sh | 169 ++++++++++++++++ shadow-devnet/genesis/validator-config.yaml | 43 ++++ 5 files changed, 525 insertions(+), 13 deletions(-) create mode 100755 generate-shadow-yaml.sh create mode 100755 run-shadow.sh create mode 100644 shadow-devnet/genesis/validator-config.yaml diff --git a/README.md b/README.md index 953bdec..7bc5d73 100644 --- a/README.md +++ b/README.md @@ -1099,6 +1099,76 @@ Ansible mode is ideal for: - Multi-host deployments - Infrastructure as Code workflows +## Shadow Network Simulator + +[Shadow](https://shadow.github.io/) is a discrete-event network simulator that runs real application binaries over a simulated network. It is useful for reproducible multi-node testing without real hardware or cloud resources. + +### Requirements + +- **Shadow** (v3.x): [Install guide](https://shadow.github.io/docs/guide/install.html) +- **yq**: YAML processor (same as regular devnet) +- **Docker**: Required for genesis generation + +### Quick Start + +From the client repo root (with lean-quickstart as a submodule): + +```sh +cd lean-quickstart +./run-shadow.sh +``` + +This single command: +1. Generates genesis with a fixed Shadow-compatible timestamp (`946684860`) +2. Generates `shadow.yaml` from `shadow-devnet/genesis/validator-config.yaml` +3. Runs Shadow simulation (default: 360 seconds) + +### Options + +```sh +# Custom simulation time +./run-shadow.sh --stop-time 600s + +# Force regenerate hash-sig keys +./run-shadow.sh --forceKeyGen + +# Use a different genesis directory +./run-shadow.sh --genesis-dir /path/to/custom/genesis +``` + +### Configuration + +Shadow devnet configuration lives in `shadow-devnet/genesis/validator-config.yaml`. It uses virtual IPs (`100.0.0.x`) required by Shadow's simulated network: + +```yaml +validators: + - name: "zeam_0" + enrFields: + ip: "100.0.0.1" + quic: 9001 + # ... +``` + +To test a different client, change the node name prefixes (e.g., `ream_0`) and ensure a matching `client-cmds/-cmd.sh` exists. + +### How It Works + +- **`run-shadow.sh`** — orchestrator: genesis → shadow.yaml → `shadow` execution +- **`generate-shadow-yaml.sh`** — reads `validator-config.yaml`, sources each client's `client-cmds/-cmd.sh` to build per-node command lines, and emits `shadow.yaml` +- **`generate-genesis.sh --genesis-time 946684860`** — produces genesis with Shadow's virtual clock epoch (Jan 1, 2000 + 60s warmup) + +Shadow's virtual clock starts at Unix timestamp `946684800` (Jan 1, 2000). Genesis time is set to `946684860` (epoch + 60s) to give nodes time to initialize. + +### Checking Results + +```sh +# Check consensus status +grep 'new_head\|finalized' shadow.data/hosts/*/*.stderr | tail -20 + +# Full node logs +cat shadow.data/hosts/zeam-0/zeam.1000.stderr +``` + ## Client branches Clients can maintain their own branches to integrated and use binay with their repos as the static targets (check `git diff main zeam_repo`, it has two nodes, both specified to run `zeam` for sim testing in zeam using the quickstart generated genesis). diff --git a/generate-genesis.sh b/generate-genesis.sh index cd29b23..83d0cda 100755 --- a/generate-genesis.sh +++ b/generate-genesis.sh @@ -15,7 +15,7 @@ PK_DOCKER_IMAGE="ethpandaops/eth-beacon-genesis:pk910-leanchain" # ======================================== show_usage() { cat << EOF -Usage: $0 [--mode local|ansible] [--offset ] [--forceKeyGen] +Usage: $0 [--mode local|ansible] [--offset ] [--genesis-time ] [--forceKeyGen] Generate genesis configuration files using PK's eth-beacon-genesis tool. Generates: config.yaml, validators.yaml, nodes.yaml, genesis.json, genesis.ssz, and .key files @@ -30,12 +30,15 @@ Options: - local: GENESIS_TIME = now + 30 seconds (default) - ansible: GENESIS_TIME = now + 360 seconds (default) --offset Override genesis time offset in seconds (overrides mode defaults) + --genesis-time Use exact genesis timestamp (unix seconds). Overrides --mode and --offset. + Useful for Shadow simulator (e.g., 946684860) or replay scenarios. --forceKeyGen Force regeneration of hash-sig validator keys Examples: $0 local-devnet/genesis # Local mode (30s offset) $0 ansible-devnet/genesis --mode ansible # Ansible mode (360s offset) $0 ansible-devnet/genesis --mode ansible --offset 600 # Custom 600s offset + $0 shadow-devnet/genesis --genesis-time 946684860 # Shadow simulator (fixed epoch) Generated Files: - config.yaml Auto-generated with GENESIS_TIME, VALIDATOR_COUNT, shuffle, and config.activeEpoch @@ -89,6 +92,7 @@ VALIDATOR_CONFIG_FILE="$GENESIS_DIR/validator-config.yaml" SKIP_KEY_GEN="true" DEPLOYMENT_MODE="local" # Default to local mode GENESIS_TIME_OFFSET="" # Will be set based on mode or --offset flag +EXACT_GENESIS_TIME="" # If set, use this exact timestamp (ignores mode/offset) shift while [[ $# -gt 0 ]]; do case "$1" in @@ -118,6 +122,19 @@ while [[ $# -gt 0 ]]; do exit 1 fi ;; + --genesis-time) + if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then + if ! [[ "$2" =~ ^[0-9]+$ ]]; then + echo "❌ Error: --genesis-time requires a positive integer (unix timestamp)" + exit 1 + fi + EXACT_GENESIS_TIME="$2" + shift 2 + else + echo "❌ Error: --genesis-time requires a value (unix timestamp)" + exit 1 + fi + ;; *) shift ;; @@ -338,21 +355,27 @@ echo "" # ======================================== echo "🔧 Step 2: Generating config.yaml..." -# Calculate genesis time based on deployment mode or explicit offset +# Calculate genesis time based on deployment mode, explicit offset, or exact timestamp # Default offsets: Local mode: 30 seconds, Ansible mode: 360 seconds -TIME_NOW="$(date +%s)" -if [ -n "$GENESIS_TIME_OFFSET" ]; then - # Use explicit offset if provided - : -elif [ "$DEPLOYMENT_MODE" == "local" ]; then - GENESIS_TIME_OFFSET=30 +if [ -n "$EXACT_GENESIS_TIME" ]; then + # Use exact genesis time (e.g., for Shadow simulator) + GENESIS_TIME="$EXACT_GENESIS_TIME" + echo " Using exact genesis time: $GENESIS_TIME" else - GENESIS_TIME_OFFSET=360 + TIME_NOW="$(date +%s)" + if [ -n "$GENESIS_TIME_OFFSET" ]; then + # Use explicit offset if provided + : + elif [ "$DEPLOYMENT_MODE" == "local" ]; then + GENESIS_TIME_OFFSET=30 + else + GENESIS_TIME_OFFSET=360 + fi + GENESIS_TIME=$((TIME_NOW + GENESIS_TIME_OFFSET)) + echo " Deployment mode: $DEPLOYMENT_MODE" + echo " Genesis time offset: ${GENESIS_TIME_OFFSET}s" + echo " Genesis time: $GENESIS_TIME" fi -GENESIS_TIME=$((TIME_NOW + GENESIS_TIME_OFFSET)) -echo " Deployment mode: $DEPLOYMENT_MODE" -echo " Genesis time offset: ${GENESIS_TIME_OFFSET}s" -echo " Genesis time: $GENESIS_TIME" # Sum all individual validator counts from validator-config.yaml TOTAL_VALIDATORS=$(yq eval '.validators[].count' "$VALIDATOR_CONFIG_FILE" | awk '{sum+=$1} END {print sum}') diff --git a/generate-shadow-yaml.sh b/generate-shadow-yaml.sh new file mode 100755 index 0000000..34d483e --- /dev/null +++ b/generate-shadow-yaml.sh @@ -0,0 +1,207 @@ +#!/bin/bash +set -e + +# generate-shadow-yaml.sh — Generate shadow.yaml from validator-config.yaml +# +# Multi-client: reuses existing client-cmds/-cmd.sh to get node_binary. +# Works for zeam, ream, lantern, gean, or any client with a *-cmd.sh file. +# +# Usage: +# ./generate-shadow-yaml.sh --project-root [--stop-time 360s] [--output shadow.yaml] + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +show_usage() { + cat << EOF +Usage: $0 --project-root [--stop-time 360s] [--output shadow.yaml] + +Generate a Shadow network simulator configuration (shadow.yaml) from validator-config.yaml. + +Arguments: + genesis-dir Path to genesis directory containing validator-config.yaml + +Options: + --project-root Project root directory (parent of lean-quickstart). Required. + --stop-time