feat: add Shadow network simulator automation

README.md

@@ -1104,6 +1104,76 @@ Ansible mode is ideal for:
 - Multi-host deployments
 - Infrastructure as Code workflows
 
+## Shadow Network Simulator
+
+[Shadow](https://shadow.github.io/) is a discrete-event network simulator that runs real application binaries over a simulated network. It is useful for reproducible multi-node testing without real hardware or cloud resources.
+
+### Requirements
+
+- **Shadow** (v3.x): [Install guide](https://shadow.github.io/docs/guide/install.html)
+- **yq**: YAML processor (same as regular devnet)
+- **Docker**: Required for genesis generation
+
+### Quick Start
+
+From the client repo root (with lean-quickstart as a submodule):
+
+```sh
+cd lean-quickstart
+./run-shadow.sh
+```
+
+This single command:
+1. Generates genesis with a fixed Shadow-compatible timestamp (`946684860`)
+2. Generates `shadow.yaml` from `shadow-devnet/genesis/validator-config.yaml`
+3. Runs Shadow simulation (default: 360 seconds)
+
+### Options
+
+```sh
+# Custom simulation time
+./run-shadow.sh --stop-time 600s
+
+# Force regenerate hash-sig keys
+./run-shadow.sh --forceKeyGen
+
+# Use a different genesis directory
+./run-shadow.sh --genesis-dir /path/to/custom/genesis
+```
+
+### Configuration
+
+Shadow devnet configuration lives in `shadow-devnet/genesis/validator-config.yaml`. It uses virtual IPs (`100.0.0.x`) required by Shadow's simulated network:
+
+```yaml
+validators:
+  - name: "zeam_0"
+    enrFields:
+      ip: "100.0.0.1"
+      quic: 9001
+    # ...
+```
+
+To test a different client, change the node name prefixes (e.g., `ream_0`) and ensure a matching `client-cmds/<client>-cmd.sh` exists.
+
+### How It Works
+
+- **`run-shadow.sh`** — orchestrator: genesis → shadow.yaml → `shadow` execution
+- **`generate-shadow-yaml.sh`** — reads `validator-config.yaml`, sources each client's `client-cmds/<client>-cmd.sh` to build per-node command lines, and emits `shadow.yaml`
+- **`generate-genesis.sh --genesis-time 946684860`** — produces genesis with Shadow's virtual clock epoch (Jan 1, 2000 + 60s warmup)
+
+Shadow's virtual clock starts at Unix timestamp `946684800` (Jan 1, 2000). Genesis time is set to `946684860` (epoch + 60s) to give nodes time to initialize.
+
+### Checking Results
+
+```sh
+# Check consensus status
+grep 'new_head\|finalized' shadow.data/hosts/*/*.stderr | tail -20
+
+# Full node logs
+cat shadow.data/hosts/zeam-0/zeam.1000.stderr
+```
+
 ## Client branches
 
 Clients can maintain their own branches to integrated and use binay with their repos as the static targets (check `git diff main zeam_repo`, it has two nodes, both specified to run `zeam` for sim testing in zeam using the quickstart generated genesis).

generate-genesis.sh

@@ -15,7 +15,7 @@ PK_DOCKER_IMAGE="ethpandaops/eth-beacon-genesis:pk910-leanchain"
 # ========================================
 show_usage() {
     cat << EOF
-Usage: $0 <genesis-directory> [--mode local|ansible] [--offset <seconds>] [--forceKeyGen] [--validator-config <path>]
+Usage: $0 <genesis-directory> [--mode local|ansible] [--offset <seconds>] [--genesis-time <timestamp>] [--forceKeyGen] [--validator-config <path>]
 
 Generate genesis configuration files using PK's eth-beacon-genesis tool.
 Generates: config.yaml, validators.yaml, nodes.yaml, genesis.json, genesis.ssz, and .key files
@@ -30,13 +30,16 @@ Options:
                        - local: GENESIS_TIME = now + 30 seconds (default)
                        - ansible: GENESIS_TIME = now + 360 seconds (default)
   --offset <seconds>   Override genesis time offset in seconds (overrides mode defaults)
+  --genesis-time <ts>  Use exact genesis timestamp (unix seconds). Overrides --mode and --offset.
+                       Useful for Shadow simulator (e.g., 946684860) or replay scenarios.
   --forceKeyGen        Force regeneration of hash-sig validator keys
   --validator-config   Path to a custom validator-config.yaml (default: <genesis-directory>/validator-config.yaml)
 
 Examples:
   $0 local-devnet/genesis                      # Local mode (30s offset)
   $0 ansible-devnet/genesis --mode ansible     # Ansible mode (360s offset)
   $0 ansible-devnet/genesis --mode ansible --offset 600  # Custom 600s offset
+  $0 shadow-devnet/genesis --genesis-time 946684860      # Shadow simulator (fixed epoch)
 
 Generated Files:
   - config.yaml        Auto-generated with GENESIS_TIME, VALIDATOR_COUNT, shuffle, and config.activeEpoch
@@ -90,6 +93,7 @@ VALIDATOR_CONFIG_FILE="$GENESIS_DIR/validator-config.yaml"
 SKIP_KEY_GEN="true"
 DEPLOYMENT_MODE="local"  # Default to local mode
 GENESIS_TIME_OFFSET=""   # Will be set based on mode or --offset flag
+EXACT_GENESIS_TIME=""    # If set, use this exact timestamp (ignores mode/offset)
 shift
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -119,6 +123,19 @@ while [[ $# -gt 0 ]]; do
                 exit 1
             fi
             ;;
+        --genesis-time)
+            if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+                if ! [[ "$2" =~ ^[0-9]+$ ]]; then
+                    echo "❌ Error: --genesis-time requires a positive integer (unix timestamp)"
+                    exit 1
+                fi
+                EXACT_GENESIS_TIME="$2"
+                shift 2
+            else
+                echo "❌ Error: --genesis-time requires a value (unix timestamp)"
+                exit 1
+            fi
+            ;;
         --validator-config)
             if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
                 VALIDATOR_CONFIG_FILE="$2"
@@ -348,21 +365,27 @@ echo ""
 # ========================================
 echo "🔧 Step 2: Generating config.yaml..."
 
-# Calculate genesis time based on deployment mode or explicit offset
+# Calculate genesis time based on deployment mode, explicit offset, or exact timestamp
 # Default offsets: Local mode: 30 seconds, Ansible mode: 360 seconds
-TIME_NOW="$(date +%s)"
-if [ -n "$GENESIS_TIME_OFFSET" ]; then
-    # Use explicit offset if provided
-    :
-elif [ "$DEPLOYMENT_MODE" == "local" ]; then
-    GENESIS_TIME_OFFSET=30
+if [ -n "$EXACT_GENESIS_TIME" ]; then
+    # Use exact genesis time (e.g., for Shadow simulator)
+    GENESIS_TIME="$EXACT_GENESIS_TIME"
+    echo "   Using exact genesis time: $GENESIS_TIME"
 else
-    GENESIS_TIME_OFFSET=360
+    TIME_NOW="$(date +%s)"
+    if [ -n "$GENESIS_TIME_OFFSET" ]; then
+        # Use explicit offset if provided
+        :
+    elif [ "$DEPLOYMENT_MODE" == "local" ]; then
+        GENESIS_TIME_OFFSET=30
+    else
+        GENESIS_TIME_OFFSET=360
+    fi
+    GENESIS_TIME=$((TIME_NOW + GENESIS_TIME_OFFSET))
+    echo "   Deployment mode: $DEPLOYMENT_MODE"
+    echo "   Genesis time offset: ${GENESIS_TIME_OFFSET}s"
+    echo "   Genesis time: $GENESIS_TIME"
 fi
-GENESIS_TIME=$((TIME_NOW + GENESIS_TIME_OFFSET))
-echo "   Deployment mode: $DEPLOYMENT_MODE"
-echo "   Genesis time offset: ${GENESIS_TIME_OFFSET}s"
-echo "   Genesis time: $GENESIS_TIME"
 
 # Sum all individual validator counts from validator-config.yaml
 TOTAL_VALIDATORS=$(yq eval '.validators[].count' "$VALIDATOR_CONFIG_FILE" | awk '{sum+=$1} END {print sum}')

generate-shadow-yaml.sh

@@ -0,0 +1,207 @@
+#!/bin/bash
+set -e
+
+# generate-shadow-yaml.sh — Generate shadow.yaml from validator-config.yaml
+#
+# Multi-client: reuses existing client-cmds/<client>-cmd.sh to get node_binary.
+# Works for zeam, ream, lantern, gean, or any client with a *-cmd.sh file.
+#
+# Usage:
+#   ./generate-shadow-yaml.sh <genesis-dir> --project-root <path> [--stop-time 360s] [--output shadow.yaml]
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+show_usage() {
+    cat << EOF
+Usage: $0 <genesis-dir> --project-root <path> [--stop-time 360s] [--output shadow.yaml]
+
+Generate a Shadow network simulator configuration (shadow.yaml) from validator-config.yaml.
+
+Arguments:
+  genesis-dir          Path to genesis directory containing validator-config.yaml
+
+Options:
+  --project-root <path>  Project root directory (parent of lean-quickstart). Required.
+  --stop-time <time>     Shadow simulation stop time (default: 360s)
+  --output <path>        Output shadow.yaml path (default: <project-root>/shadow.yaml)
+
+This script is client-agnostic. It reads node names from validator-config.yaml,
+extracts the client name from the node prefix (e.g., zeam_0 → zeam), and sources
+the corresponding client-cmds/<client>-cmd.sh to generate per-node arguments.
+EOF
+    exit 1
+}
+
+# ========================================
+# Parse arguments
+# ========================================
+if [ -z "$1" ] || [ "${1:0:1}" == "-" ]; then
+    show_usage
+fi
+
+GENESIS_DIR="$(cd "$1" && pwd)"
+shift
+
+PROJECT_ROOT=""
+STOP_TIME="360s"
+OUTPUT_FILE=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --project-root)
+            if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+                PROJECT_ROOT="$(cd "$2" && pwd)"
+                shift 2
+            else
+                echo "❌ Error: --project-root requires a path"
+                exit 1
+            fi
+            ;;
+        --stop-time)
+            if [ -n "$2" ]; then
+                STOP_TIME="$2"
+                shift 2
+            else
+                echo "❌ Error: --stop-time requires a value"
+                exit 1
+            fi
+            ;;
+        --output)
+            if [ -n "$2" ]; then
+                OUTPUT_FILE="$2"
+                shift 2
+            else
+                echo "❌ Error: --output requires a path"
+                exit 1
+            fi
+            ;;
+        *)
+            echo "❌ Unknown option: $1"
+            show_usage
+            ;;
+    esac
+done
+
+if [ -z "$PROJECT_ROOT" ]; then
+    echo "❌ Error: --project-root is required"
+    show_usage
+fi
+
+if [ -z "$OUTPUT_FILE" ]; then
+    OUTPUT_FILE="$PROJECT_ROOT/shadow.yaml"
+fi
+
+VALIDATOR_CONFIG="$GENESIS_DIR/validator-config.yaml"
+if [ ! -f "$VALIDATOR_CONFIG" ]; then
+    echo "❌ Error: validator-config.yaml not found at $VALIDATOR_CONFIG"
+    exit 1
+fi
+
+# ========================================
+# Read nodes from validator-config.yaml
+# ========================================
+node_names=($(yq eval '.validators[].name' "$VALIDATOR_CONFIG"))
+node_count=${#node_names[@]}
+
+if [ "$node_count" -eq 0 ]; then
+    echo "❌ Error: No validators found in $VALIDATOR_CONFIG"
+    exit 1
+fi
+
+echo "🔧 Generating shadow.yaml for $node_count nodes..."
+
+# ========================================
+# Write shadow.yaml preamble
+# ========================================
+cat > "$OUTPUT_FILE" << EOF
+# Auto-generated Shadow network simulator configuration
+# Generated from: $VALIDATOR_CONFIG
+# Nodes: ${node_names[*]}
+
+general:
+  model_unblocked_syscall_latency: true
+  stop_time: $STOP_TIME
+
+experimental:
+  native_preemption_enabled: true
+
+network:
+  graph:
+    type: 1_gbit_switch
+
+hosts:
+EOF
+
+# ========================================
+# Generate per-node host entries
+# ========================================
+for i in "${!node_names[@]}"; do
+    item="${node_names[$i]}"
+
+    # Extract client name from node prefix (zeam_0 → zeam, leanspec_0 → leanspec)
+    IFS='_' read -r -a elements <<< "$item"
+    client="${elements[0]}"
+
+    # DNS-valid hostname: underscores → hyphens (Shadow requirement)
+    hostname="${item//_/-}"
+
+    # Extract IP from validator-config
+    ip=$(yq eval ".validators[$i].enrFields.ip" "$VALIDATOR_CONFIG")
+
+    # Set up environment for parse-vc.sh and client-cmd.sh
+    # These scripts expect: $item, $configDir, $dataDir, $scriptDir, $validatorConfig
+    export scriptDir="$SCRIPT_DIR"
+    export configDir="$GENESIS_DIR"
+    export dataDir="$PROJECT_ROOT/shadow.data/hosts/$hostname"
+    export validatorConfig="$VALIDATOR_CONFIG"
+
+    # Source parse-vc.sh to extract per-node config (quicPort, metricsPort, apiPort, etc.)
+    # parse-vc.sh uses $item and $configDir
+    source "$SCRIPT_DIR/parse-vc.sh"
+
+    # Source client-cmd.sh to get node_binary
+    node_setup="binary"
+    client_cmd="$SCRIPT_DIR/client-cmds/${client}-cmd.sh"
+    if [ ! -f "$client_cmd" ]; then
+        echo "❌ Error: Client command script not found: $client_cmd"
+        echo "   Available clients:"
+        ls "$SCRIPT_DIR/client-cmds/"*-cmd.sh 2>/dev/null | sed 's/.*\//  /' | sed 's/-cmd.sh//'
+        exit 1
+    fi
+    source "$client_cmd"
+
+    # node_binary is now set by the client-cmd.sh script
+    # Convert relative paths to absolute paths for Shadow
+    # Extract the binary path (first word) and args (rest)
+    binary_path=$(echo "$node_binary" | awk '{print $1}')
+    binary_args=$(echo "$node_binary" | sed "s|^[^ ]*||")
+
+    # Make binary path absolute
+    if [[ "$binary_path" != /* ]]; then
+        binary_path="$(cd "$(dirname "$binary_path")" 2>/dev/null && pwd)/$(basename "$binary_path")" 2>/dev/null || binary_path="$PROJECT_ROOT/${binary_path#./}"
+    fi
+
+    # Make all path args absolute: replace $configDir, $dataDir references with absolute paths
+    # The client-cmd.sh already uses $configDir and $dataDir which we set to absolute paths
+
+    # Write host entry
+    cat >> "$OUTPUT_FILE" << EOF
+  $hostname:
+    network_node_id: 0
+    ip_addr: $ip
+    processes:
+    - path: $binary_path
+      args: >-
+       $binary_args
+      start_time: 1s
+      expected_final_state: running
+
+EOF
+
+    echo "   ✅ $item → $hostname ($ip) [$client]"
+done
+
+echo ""
+echo "📄 Shadow config written to: $OUTPUT_FILE"
+echo "   Stop time: $STOP_TIME"
+echo "   Nodes: $node_count"