diff --git a/.hack/orbstack.sh b/.hack/orbstack.sh new file mode 100755 index 000000000..b95e0a3f6 --- /dev/null +++ b/.hack/orbstack.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# +# Provision an OrbStack Linux machine to run benchmarkoor with a copy-on-write +# datadir method (overlayfs). macOS has no overlayfs/zfs/schelk, so the large +# stateful prestates (e.g. config.state-actor-eest.full.amsterdam.stateful.yaml, +# ~77 GB) can't use anything but `copy` on the host — which is infeasible at that +# size. An OrbStack Linux machine has a real kernel (overlayfs works) and we run +# a DEDICATED dockerd inside it so benchmarkoor, dockerd, and the overlay mounts +# all share one filesystem namespace (the merged dir must be bind-mountable into +# the client containers — the host's shared Docker engine can't see VM-local +# overlay mounts). +# +# Prereqs: OrbStack installed (https://orbstack.dev). The repo is auto-shared +# into the machine at the same path, so no copying is needed. +# +# Usage: +# .hack/orbstack.sh [machine-name] # default: bench +# +# After it finishes the binary is at /bin/benchmarkoor (also symlinked onto +# PATH, so a bare `benchmarkoor` works). Run it INSIDE the machine as root +# (overlayfs mount needs CAP_SYS_ADMIN), e.g.: +# orb -m bench sudo benchmarkoor build \ +# --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --force +# orb -m bench sudo benchmarkoor run \ +# --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --limit-instance-id=geth +set -euo pipefail + +MACHINE="${1:-bench}" +DISTRO="${ORBSTACK_DISTRO:-ubuntu}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if ! command -v orb >/dev/null 2>&1; then + echo "error: 'orb' not found — install OrbStack first (https://orbstack.dev)" >&2 + exit 1 +fi + +echo "==> Repo root: ${REPO_ROOT}" +echo "==> OrbStack machine: ${MACHINE} (${DISTRO})" + +# 1. Create the machine if it doesn't already exist (idempotent). +if orb list 2>/dev/null | awk '{print $1}' | grep -qx "${MACHINE}"; then + echo "==> Machine '${MACHINE}' already exists, reusing it." +else + echo "==> Creating machine '${MACHINE}'..." + orb create "${DISTRO}" "${MACHINE}" +fi + +# 2. Provision inside the machine: dedicated dockerd + Go toolchain, then build +# benchmarkoor natively. Everything below runs as root in the machine. +orb -m "${MACHINE}" sudo bash -s -- "${REPO_ROOT}" <<'PROVISION' +set -euo pipefail +REPO_ROOT="$1" + +echo "==> apt: docker.io + golang-go + make + git" +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +# make + git let you run the repo's Makefile targets (e.g. `make build-core`, +# `make test-core`) inside the machine; git also stamps the version/commit. +apt-get install -y -qq docker.io golang-go make git + +echo "==> Starting the machine's own dockerd" +systemctl enable --now docker +systemctl is-active --quiet docker || { echo "dockerd failed to start" >&2; exit 1; } + +# A dedicated, VM-local docker daemon is the whole point — confirm it. +docker version --format ' dockerd {{.Server.Version}}' >/dev/null + +echo "==> Go: $(go version)" +case "$(go version)" in + *go1.2[3-9]*|*go1.[3-9][0-9]*|*go[2-9].*) : ;; + *) echo "warning: Go < 1.23 detected; benchmarkoor needs >= 1.23. Install a newer Go." >&2 ;; +esac + +echo "==> Building benchmarkoor (make build-core)" +cd "${REPO_ROOT}" +GOCACHE=/root/.cache/go-build GOPATH=/root/go make build-core +# The binary lands in ${REPO_ROOT}/bin/benchmarkoor. Symlink it onto PATH so a +# bare `benchmarkoor` works from anywhere and a later `make build-core` (rebuild +# after a code change) is picked up automatically — no stale copy. +ln -sf "${REPO_ROOT}/bin/benchmarkoor" /usr/local/bin/benchmarkoor +benchmarkoor --help >/dev/null \ + && echo "==> benchmarkoor built at ${REPO_ROOT}/bin/benchmarkoor (symlinked onto PATH at /usr/local/bin/benchmarkoor)" + +echo "==> Verifying overlayfs works in this machine" +d="$(mktemp -d)"; mkdir -p "$d"/{low,up,work,merged}; echo ok > "$d/low/f" +mount -t overlay overlay -o "lowerdir=$d/low,upperdir=$d/up,workdir=$d/work" "$d/merged" +docker run --rm -v "$d/merged:/m" alpine cat /m/f >/dev/null && echo "==> overlayfs + dockerd bind-mount: OK" +umount "$d/merged"; rm -rf "$d" +PROVISION + +cat < Done. The '${MACHINE}' machine is ready (dedicated dockerd + overlayfs + benchmarkoor). + +benchmarkoor is built at ${REPO_ROOT}/bin/benchmarkoor (symlinked onto PATH). +Run it inside the machine as root (the repo is shared at ${REPO_ROOT}): + + orb -m ${MACHINE} sudo benchmarkoor build \\ + --config ${REPO_ROOT}/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --force + + orb -m ${MACHINE} sudo benchmarkoor run \\ + --config ${REPO_ROOT}/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --limit-instance-id=geth + +Tear down with: orb delete ${MACHINE} +EOF diff --git a/Dockerfile.eest-filler b/Dockerfile.eest-filler new file mode 100644 index 000000000..dee864909 --- /dev/null +++ b/Dockerfile.eest-filler @@ -0,0 +1,60 @@ +# Dockerfile.eest-filler +# +# Builds the toolchain image referenced by `builder.eest_payloads.fill_image`: +# it carries only `uv` + python + the build toolchain — NOT the execution-specs +# repo. benchmarkoor clones execution-specs at build time (at the configured +# `eest_ref`, caching it on disk) and mounts the checkout into this container at +# /eest, then runs `uv run fill-stateful`. uv builds the venv into the mounted +# checkout on first use (cached across runs). This keeps the EEST version in +# config and changeable without rebuilding the image. +# +# fill-stateful drives a *live* EL client over testing_buildBlockV1, so — unlike +# the t8n-based `fill` — it needs NO evmone/eels transition-tool binary. +# +# Build: +# docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +# # (podman build works identically) +# +# Then point your config at it: +# builder: +# eest_payloads: +# fill_image: ghcr.io/your-org/eest-fill-stateful:latest +# eest_ref: forks/amsterdam # optional; defaults to forks/amsterdam + +ARG PYTHON_VERSION=3.12 + +FROM python:${PYTHON_VERSION}-slim-bookworm + +# uv: copy the static binary from the official image (recommended pattern). +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# Runtime deps: git (fill-stateful reads its commit hash from the checkout, and +# uv resolves git deps); ca-certificates for HTTPS to PyPI; build-essential plus +# clang/libclang-dev/pkg-config for EEST dependencies without a prebuilt wheel. +# In particular `ethereum-execution[optimized]` pulls in rust-pyspec-glue, whose +# native mdbx-sys crate has no arm64 wheel and is compiled from source: its build +# script runs bindgen, which dlopens libclang. Cargo itself is bootstrapped by uv +# at build time, so only the clang toolchain has to be present in the image. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + git \ + ca-certificates \ + build-essential \ + clang \ + libclang-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Point bindgen/clang-sys straight at the bookworm-default LLVM 14 libclang so it +# never falls back to its "couldn't find any valid shared libraries" failure. The +# base image is pinned to bookworm (above), so the LLVM major version is stable. +ENV LIBCLANG_PATH=/usr/lib/llvm-14/lib + +ENV UV_NO_PROGRESS=1 + +# The execution-specs checkout is bind-mounted here by benchmarkoor at run time; +# the relative test paths (e.g. tests/benchmark/stateful) resolve against it. +WORKDIR /eest + +# Standalone default; benchmarkoor overrides this with the full fill-stateful argv. +CMD ["uv", "--version"] diff --git a/_typos.toml b/_typos.toml index f250b3b86..6e94fac16 100644 --- a/_typos.toml +++ b/_typos.toml @@ -4,3 +4,11 @@ extend-exclude = ["go.mod"] [default.extend-words] PANC = "PANC" ERRO = "ERRO" +BULD = "BULD" + +# EIP-7928 block access lists. typos splits the identifier "BALs" into "BA"+"Ls" +# and flags "BA" as a typo of "BY"/"BE"; allow the whole identifier (and the +# singular "BAL") so the BAL terminology in comments/config passes. +[default.extend-identifiers] +BAL = "BAL" +BALs = "BALs" diff --git a/cmd/benchmarkoor/build.go b/cmd/benchmarkoor/build.go index 79ee8e1e0..d8ba8c59c 100644 --- a/cmd/benchmarkoor/build.go +++ b/cmd/benchmarkoor/build.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/signal" + "sort" "strings" "syscall" @@ -18,30 +19,52 @@ import ( ) var ( - buildTargetFilter []string - buildForce bool + buildTargetFilter []string + buildStateActorTargets []string + buildEESTPayloadTargets []string + buildSkipStateActor bool + buildSkipEESTPayloads bool + buildForce bool ) var buildCmd = &cobra.Command{ Use: "build", - Short: "Build client datadirs declared in builder.state_actor.targets", - Long: `Materialise each datadir declared under builder.state_actor.targets -by invoking state-actor (https://github.com/ethereum/state-actor) via the -configured container runtime. Builds are decoupled from "benchmarkoor run": -this command produces datadirs on disk that subsequent runs consume via -their normal datadir.* providers.`, + Short: "Build datadirs and fixtures declared under the builder.* config blocks", + Long: `Run each configured builder: + + - builder.state_actor materialises pre-populated client datadirs by invoking + state-actor (https://github.com/ethereum/state-actor). + - builder.eest_payloads generates stateful EEST benchmark fixtures by running + fill-stateful against a filler client booted on a snapshot. + +Builds are decoupled from "benchmarkoor run": this command produces artifacts on +disk that subsequent runs consume via their normal datadir.* / test source providers. +Builders run in declaration order (state_actor before eest_payloads) so a fixture +build can consume a datadir produced earlier in the same invocation.`, RunE: runBuild, } func init() { rootCmd.AddCommand(buildCmd) buildCmd.Flags().StringSliceVar(&buildTargetFilter, "target", nil, - "Only build targets whose name matches (comma-separated or repeated)") + "Only build targets whose name matches, across all builders (comma-separated or repeated)") + buildCmd.Flags().StringSliceVar(&buildStateActorTargets, "limit-state-actor-target", nil, + "Only build builder.state_actor targets whose name matches (comma-separated or repeated)") + buildCmd.Flags().StringSliceVar(&buildEESTPayloadTargets, "limit-eest-payload-target", nil, + "Only build builder.eest_payloads targets whose name matches (comma-separated or repeated)") + buildCmd.Flags().BoolVar(&buildSkipStateActor, "skip-state-actor-build", false, + "Skip the builder.state_actor builder entirely") + buildCmd.Flags().BoolVar(&buildSkipEESTPayloads, "skip-eest-payload-build", false, + "Skip the builder.eest_payloads builder entirely") buildCmd.Flags().BoolVar(&buildForce, "force", false, "Remove each target's output_dir before building") } func runBuild(_ *cobra.Command, _ []string) error { + // Match the `benchmarkoor run` log format (🔵 prefix); container output is + // streamed in the same 🟣 client-log style for a consistent look. + log.SetFormatter(&consistentFormatter{prefix: "🔵"}) + if len(cfgFiles) == 0 { return fmt.Errorf("config file is required (use --config)") } @@ -55,18 +78,33 @@ func runBuild(_ *cobra.Command, _ []string) error { return fmt.Errorf("validating config: %w", err) } - if cfg.Builder == nil || cfg.Builder.StateActor == nil || len(cfg.Builder.StateActor.Targets) == 0 { - return fmt.Errorf("builder.state_actor.targets is empty or unset; nothing to build") + if cfg.Builder == nil || + (cfg.Builder.StateActor == nil && cfg.Builder.EESTPayloads == nil) { + return fmt.Errorf("no builders configured; nothing to build") } - targets, err := selectTargets(cfg.Builder.StateActor.Targets, buildTargetFilter) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + installSignalHandler(cancel) + + builders, stop, err := buildBuilders(ctx, cfg) if err != nil { return err } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + defer stop() + + if len(builders) == 0 { + return fmt.Errorf("all configured builders were skipped; nothing to build") + } + return runBuilders(ctx, builders) +} + +// installSignalHandler cancels the context on the first SIGINT/SIGTERM and +// force-exits on the second. +func installSignalHandler(cancel context.CancelFunc) { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) @@ -78,45 +116,128 @@ func runBuild(_ *cobra.Command, _ []string) error { sig = <-sigCh log.WithField("signal", sig).Fatal("Received second signal, forcing exit") }() +} - runtime := cfg.GetStateActorContainerRuntime() +// buildBuilders constructs every configured builder, creating and starting a +// container manager per distinct runtime. The returned stop func stops all +// managers and must be deferred by the caller. +func buildBuilders(ctx context.Context, cfg *config.Config) ([]builder.Builder, func(), error) { + managers := make(map[string]docker.ContainerManager, 2) + + stop := func() { + for _, mgr := range managers { + // Best-effort teardown of the shared eest_payloads build network + // (created lazily during the fill). RemoveNetwork errors when it was + // never created (e.g. no eest build ran) — expected, so log at Debug. + // Use a fresh context so cleanup still runs if the build ctx was + // cancelled (e.g. SIGINT). + if err := mgr.RemoveNetwork(context.Background(), builder.EESTBuildNetwork); err != nil { + log.WithError(err).Debug("Build network not removed (likely never created)") + } + + if err := mgr.Stop(); err != nil { + log.WithError(err).Warn("Failed to stop container manager") + } + } + } - var mgr docker.ContainerManager + getManager := func(runtime string) (docker.ContainerManager, error) { + if mgr, ok := managers[runtime]; ok { + return mgr, nil + } - switch runtime { - case "podman": - mgr, err = podman.NewManager(log) - default: - mgr, err = docker.NewManager(log) + mgr, err := newContainerManager(runtime) + if err != nil { + return nil, err + } + + if err := mgr.Start(ctx); err != nil { + return nil, fmt.Errorf("starting %s container manager: %w", runtime, err) + } + + managers[runtime] = mgr + + return mgr, nil } - if err != nil { - return fmt.Errorf("creating container manager: %w", err) + var builders []builder.Builder + + if cfg.Builder.StateActor != nil && buildSkipStateActor { + log.Info("Skipping builder.state_actor (--skip-state-actor-build)") + } + + if cfg.Builder.StateActor != nil && !buildSkipStateActor { + runtime := cfg.GetStateActorContainerRuntime() + + mgr, err := getManager(runtime) + if err != nil { + stop() + + return nil, nil, err + } + + builders = append(builders, builder.NewStateActorBuilder(log, cfg.Builder.StateActor, runtime, mgr)) } - if err := mgr.Start(ctx); err != nil { - return fmt.Errorf("starting container manager: %w", err) + if cfg.Builder.EESTPayloads != nil && buildSkipEESTPayloads { + log.Info("Skipping builder.eest_payloads (--skip-eest-payload-build)") } - defer func() { - if err := mgr.Stop(); err != nil { - log.WithError(err).Warn("Failed to stop container manager") + if cfg.Builder.EESTPayloads != nil && !buildSkipEESTPayloads { + runtime := cfg.GetEESTPayloadsContainerRuntime() + + mgr, err := getManager(runtime) + if err != nil { + stop() + + return nil, nil, err } - }() - b := builder.NewStateActorBuilder(log, cfg.Builder.StateActor, runtime, mgr) + cacheDir, err := cfg.ResolveCacheDir() + if err != nil { + stop() + + return nil, nil, err + } + + builders = append(builders, + builder.NewEESTPayloadsBuilder(log, cfg.Builder.EESTPayloads, runtime, mgr, cacheDir)) + } + + return builders, stop, nil +} + +// newContainerManager creates a container manager for the given runtime. +func newContainerManager(runtime string) (docker.ContainerManager, error) { + switch runtime { + case "podman": + return podman.NewManager(log) + default: + return docker.NewManager(log) + } +} + +// buildResult captures the outcome of a single target build. +type buildResult struct { + builder string + name string + client string + outputDir string + skipped bool + err error +} - type result struct { - name string - client string - outputDir string - skipped bool - err error +// runBuilders selects and builds the requested targets across all builders, +// preserving declaration order, then prints a summary. +func runBuilders(ctx context.Context, builders []builder.Builder) error { + targets, err := selectTargets(builders, buildTargetFilter, limitFilters(builders)) + if err != nil { + return err } - results := make([]result, 0, len(targets)) + results := make([]buildResult, 0, len(targets)) - for _, t := range targets { + for _, sel := range targets { select { case <-ctx.Done(): return ctx.Err() @@ -124,97 +245,214 @@ func runBuild(_ *cobra.Command, _ []string) error { } log.WithFields(logrus.Fields{ - "target": t.EffectiveName(), - "client": t.Client, - "output_dir": t.OutputDir, + "target": sel.info.Name, + "client": sel.info.Client, + "output_dir": sel.info.OutputDir, }).Info("Building target") - skipped, buildErr := b.Build(ctx, t.EffectiveName(), builder.BuildOptions{Force: buildForce}) + skipped, buildErr := sel.builder.Build(ctx, sel.info.Name, builder.BuildOptions{Force: buildForce}) - results = append(results, result{ - name: t.EffectiveName(), - client: t.Client, - outputDir: t.OutputDir, + results = append(results, buildResult{ + builder: sel.builder.Name(), + name: sel.info.Name, + client: sel.info.Client, + outputDir: sel.info.OutputDir, skipped: skipped, err: buildErr, }) if buildErr != nil { - log.WithError(buildErr).WithField("target", t.EffectiveName()).Error("Build failed") + log.WithError(buildErr).WithField("target", sel.info.Name).Error("Build failed") } } - // Summary. + return summarise(results) +} + +// summarise logs the per-target outcome grouped under each builder and returns +// an error if any target failed. Builders are emitted in first-seen +// (declaration) order so state_actor appears before eest_payloads. +func summarise(results []buildResult) error { var failed []string - log.Info("Build summary:") + var order []string + + byBuilder := make(map[string][]buildResult, 2) for _, r := range results { - var status string + if _, seen := byBuilder[r.builder]; !seen { + order = append(order, r.builder) + } + + byBuilder[r.builder] = append(byBuilder[r.builder], r) - switch { - case r.err != nil: - status = "ERR " + if r.err != nil { failed = append(failed, r.name) - case r.skipped: - status = "SKIP" - default: - status = "OK " } + } - log.WithFields(logrus.Fields{ - "target": r.name, - "client": r.client, - "output_dir": r.outputDir, - }).Infof(" %s %s", status, r.name) + for _, b := range order { + log.Infof("Build summary [%s]:", b) + + for _, r := range byBuilder[b] { + var status string + + switch { + case r.err != nil: + status = "ERR " + case r.skipped: + status = "SKIP" + default: + status = "OK " + } + + log.WithFields(logrus.Fields{ + "builder": r.builder, + "target": r.name, + "client": r.client, + "output_dir": r.outputDir, + }).Infof(" %s %s", status, r.name) + } } if len(failed) > 0 { - return fmt.Errorf("%d target(s) failed: %s", - len(failed), strings.Join(failed, ", ")) + return fmt.Errorf("%d target(s) failed: %s", len(failed), strings.Join(failed, ", ")) } return nil } -// selectTargets filters `all` by the names in `filter`, preserving the -// order targets were declared in. An empty filter returns all targets. -// Unmatched filter values produce an error so typos surface immediately. -func selectTargets(all []config.StateActorTarget, filter []string) ([]config.StateActorTarget, error) { - if len(filter) == 0 { - return all, nil +// selectedTarget pairs a target with the builder that owns it. +type selectedTarget struct { + builder builder.Builder + info builder.TargetInfo +} + +// builderFilter is a per-builder `--limit--target` filter: the wanted +// target names plus the flag name, used for a precise "matched nothing" error. +type builderFilter struct { + flag string + values []string +} + +// limitFilters maps each present builder to its `--limit--target` +// filter. A builder absent from `builders` (e.g. skipped via --skip-*-build) is +// omitted, so its limit flag is silently ignored rather than erroring as +// "matched no targets". +func limitFilters(builders []builder.Builder) map[string]builderFilter { + all := map[string]builderFilter{ + builder.StateActorBuilderName: {flag: "--limit-state-actor-target", values: buildStateActorTargets}, + builder.EESTPayloadsBuilderName: {flag: "--limit-eest-payload-target", values: buildEESTPayloadTargets}, + } + + out := make(map[string]builderFilter, len(builders)) + + for _, b := range builders { + if f, ok := all[b.Name()]; ok { + out[b.Name()] = f + } + } + + return out +} + +// selectTargets flattens all builders' targets in declaration order and filters +// them. A target is selected when it passes both the global `--target` filter +// and the per-builder filter for the builder that owns it (keyed by Builder +// Name()). An empty filter imposes no restriction. Unmatched filter values +// produce an error so typos surface immediately — the global filter is checked +// against every target name, each per-builder filter against only that builder's +// target names. +func selectTargets( + builders []builder.Builder, global []string, perBuilder map[string]builderFilter, +) ([]selectedTarget, error) { + globalWanted := nameSet(global) + + var out []selectedTarget + + for _, b := range builders { + bf := perBuilder[b.Name()] + builderWanted := nameSet(bf.values) + + for _, info := range b.Targets() { + if len(globalWanted) > 0 && !globalWanted[info.Name] { + continue + } + + if len(builderWanted) > 0 && !builderWanted[info.Name] { + continue + } + + out = append(out, selectedTarget{builder: b, info: info}) + } + } + + if err := checkFiltersMatched(builders, globalWanted, perBuilder); err != nil { + return nil, err } - wanted := make(map[string]bool, len(filter)) - for _, f := range filter { - f = strings.TrimSpace(f) - if f != "" { - wanted[f] = true + return out, nil +} + +// nameSet builds a lookup set from filter values, trimming blanks. +func nameSet(values []string) map[string]bool { + set := make(map[string]bool, len(values)) + + for _, v := range values { + if v = strings.TrimSpace(v); v != "" { + set[v] = true } } - out := make([]config.StateActorTarget, 0, len(all)) - matched := make(map[string]bool, len(wanted)) + return set +} + +// checkFiltersMatched verifies every filter value names an existing target, +// returning an error listing any that matched nothing. +func checkFiltersMatched( + builders []builder.Builder, global map[string]bool, perBuilder map[string]builderFilter, +) error { + allNames := make(map[string]bool) + namesByBuilder := make(map[string]map[string]bool, len(builders)) + + for _, b := range builders { + names := make(map[string]bool) - for _, t := range all { - name := t.EffectiveName() - if wanted[name] { - out = append(out, t) - matched[name] = true + for _, info := range b.Targets() { + names[info.Name] = true + allNames[info.Name] = true + } + + namesByBuilder[b.Name()] = names + } + + if missing := unmatched(global, allNames); len(missing) > 0 { + return errors.New("--target filter matched no targets: " + strings.Join(missing, ", ")) + } + + for builderName, bf := range perBuilder { + if missing := unmatched(nameSet(bf.values), namesByBuilder[builderName]); len(missing) > 0 { + return fmt.Errorf( + "%s matched no %s targets: %s", bf.flag, builderName, strings.Join(missing, ", "), + ) } } + return nil +} + +// unmatched returns the wanted names absent from available, sorted for a stable +// error message. +func unmatched(wanted, available map[string]bool) []string { var missing []string for name := range wanted { - if !matched[name] { + if !available[name] { missing = append(missing, name) } } - if len(missing) > 0 { - return nil, errors.New("--target filter matched no targets: " + strings.Join(missing, ", ")) - } + sort.Strings(missing) - return out, nil + return missing } diff --git a/cmd/benchmarkoor/build_test.go b/cmd/benchmarkoor/build_test.go new file mode 100644 index 000000000..262c25565 --- /dev/null +++ b/cmd/benchmarkoor/build_test.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "errors" + "io" + "os" + "testing" + + "github.com/ethpandaops/benchmarkoor/pkg/builder" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMain initializes the package-level logger (nil until main() runs) so tests +// that exercise functions which log (e.g. summarise) don't nil-panic. +func TestMain(m *testing.M) { + log = logrus.New() + log.SetOutput(io.Discard) + + os.Exit(m.Run()) +} + +// fakeBuilder is a minimal builder.Builder for exercising selectTargets. +type fakeBuilder struct { + name string + targets []builder.TargetInfo +} + +func (f *fakeBuilder) Name() string { return f.name } + +func (f *fakeBuilder) Targets() []builder.TargetInfo { return f.targets } + +func (f *fakeBuilder) Build(context.Context, string, builder.BuildOptions) (bool, error) { + return false, nil +} + +func TestSummarise(t *testing.T) { + t.Run("all OK/SKIP returns no error", func(t *testing.T) { + err := summarise([]buildResult{ + {builder: "state-actor", name: "geth", skipped: false}, + {builder: "state-actor", name: "reth", skipped: true}, + {builder: "eest-payloads", name: "fill-geth", skipped: false}, + }) + assert.NoError(t, err) + }) + + t.Run("any failure aggregates into an error naming the failed targets", func(t *testing.T) { + err := summarise([]buildResult{ + {builder: "state-actor", name: "geth"}, + {builder: "eest-payloads", name: "fill-besu", err: errors.New("boom")}, + {builder: "eest-payloads", name: "fill-reth", err: errors.New("kaboom")}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "2 target(s) failed") + assert.Contains(t, err.Error(), "fill-besu") + assert.Contains(t, err.Error(), "fill-reth") + // A successful target must not appear in the failed list. + assert.NotContains(t, err.Error(), "geth") + }) +} + +func TestLimitFilters(t *testing.T) { + // limitFilters reads the package-level flag vars; save and restore them. + savedSA, savedEEST := buildStateActorTargets, buildEESTPayloadTargets + t.Cleanup(func() { buildStateActorTargets, buildEESTPayloadTargets = savedSA, savedEEST }) + + buildStateActorTargets = []string{"nethermind"} + buildEESTPayloadTargets = []string{"payload-generator-nethermind"} + + sa := &fakeBuilder{name: builder.StateActorBuilderName} + eest := &fakeBuilder{name: builder.EESTPayloadsBuilderName} + + t.Run("both present yields both filters", func(t *testing.T) { + got := limitFilters([]builder.Builder{sa, eest}) + require.Len(t, got, 2) + assert.Equal(t, []string{"nethermind"}, got[builder.StateActorBuilderName].values) + assert.Equal(t, []string{"payload-generator-nethermind"}, got[builder.EESTPayloadsBuilderName].values) + }) + + t.Run("absent (skipped) builder's limit is dropped", func(t *testing.T) { + // state_actor skipped → only eest builder present. + got := limitFilters([]builder.Builder{eest}) + require.Len(t, got, 1) + _, hasSA := got[builder.StateActorBuilderName] + assert.False(t, hasSA, "skipped builder's limit must be omitted") + assert.Equal(t, []string{"payload-generator-nethermind"}, got[builder.EESTPayloadsBuilderName].values) + }) +} + +func TestSelectTargets(t *testing.T) { + // state_actor builds per-client snapshots; eest_payloads fills per named target. + stateActor := &fakeBuilder{ + name: builder.StateActorBuilderName, + targets: []builder.TargetInfo{ + {Name: "geth", Client: "geth"}, + {Name: "nethermind", Client: "nethermind"}, + }, + } + eest := &fakeBuilder{ + name: builder.EESTPayloadsBuilderName, + targets: []builder.TargetInfo{ + {Name: "payload-generator-geth", Client: "geth"}, + {Name: "payload-generator-nethermind", Client: "nethermind"}, + }, + } + builders := []builder.Builder{stateActor, eest} + + filters := func(sa, eestF []string) map[string]builderFilter { + return map[string]builderFilter{ + builder.StateActorBuilderName: {flag: "--limit-state-actor-target", values: sa}, + builder.EESTPayloadsBuilderName: {flag: "--limit-eest-payload-target", values: eestF}, + } + } + + tests := []struct { + name string + global []string + sa []string + eest []string + want []string + wantErr bool + errSubstr string + }{ + { + name: "no filters selects every target", + want: []string{"geth", "nethermind", "payload-generator-geth", "payload-generator-nethermind"}, + }, + { + name: "global --target filters across builders", + global: []string{"nethermind", "payload-generator-nethermind"}, + want: []string{"nethermind", "payload-generator-nethermind"}, + }, + { + name: "limit-eest-payload-target restricts only eest; state_actor unrestricted", + eest: []string{"payload-generator-nethermind"}, + want: []string{"geth", "nethermind", "payload-generator-nethermind"}, + }, + { + name: "limit-state-actor-target restricts only state_actor; eest unrestricted", + sa: []string{"nethermind"}, + want: []string{"nethermind", "payload-generator-geth", "payload-generator-nethermind"}, + }, + { + name: "both per-builder filters narrow each builder", + sa: []string{"nethermind"}, + eest: []string{"payload-generator-nethermind"}, + want: []string{"nethermind", "payload-generator-nethermind"}, + }, + { + name: "global and per-builder compose (intersection)", + global: []string{"nethermind", "payload-generator-geth", "payload-generator-nethermind"}, + eest: []string{"payload-generator-nethermind"}, + want: []string{"nethermind", "payload-generator-nethermind"}, + }, + { + name: "unknown global target errors", + global: []string{"besu"}, + wantErr: true, + errSubstr: "--target filter matched no targets: besu", + }, + { + name: "unknown state-actor target errors with builder-scoped message", + sa: []string{"besu"}, + wantErr: true, + errSubstr: "--limit-state-actor-target matched no state-actor targets: besu", + }, + { + name: "eest name given to state-actor filter errors (scoped per builder)", + sa: []string{"payload-generator-nethermind"}, + wantErr: true, + errSubstr: "--limit-state-actor-target matched no state-actor targets: payload-generator-nethermind", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := selectTargets(builders, tt.global, filters(tt.sa, tt.eest)) + + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + + return + } + + require.NoError(t, err) + + names := make([]string, 0, len(got)) + for _, sel := range got { + names = append(names, sel.info.Name) + } + + assert.Equal(t, tt.want, names) + }) + } +} diff --git a/cmd/benchmarkoor/cleanup.go b/cmd/benchmarkoor/cleanup.go index 19ee20853..a1b5d379b 100644 --- a/cmd/benchmarkoor/cleanup.go +++ b/cmd/benchmarkoor/cleanup.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "github.com/ethpandaops/benchmarkoor/pkg/builder" "github.com/ethpandaops/benchmarkoor/pkg/cpufreq" "github.com/ethpandaops/benchmarkoor/pkg/datadir" "github.com/ethpandaops/benchmarkoor/pkg/docker" @@ -48,6 +49,12 @@ type managedVolume struct { mgr docker.ContainerManager } +// managedNetwork associates a network name with the manager that owns it. +type managedNetwork struct { + name string + mgr docker.ContainerManager +} + func runCleanup(cmd *cobra.Command, args []string) error { ctx := context.Background() @@ -95,6 +102,22 @@ func performCleanup(ctx context.Context, managers []docker.ContainerManager, for } } + // Detect the shared eest_payloads build network, left behind after a build. + var networks []managedNetwork + + for _, mgr := range managers { + exists, err := mgr.NetworkExists(ctx, builder.EESTBuildNetwork) + if err != nil { + log.WithError(err).Warn("Failed to check for the build network") + + continue + } + + if exists { + networks = append(networks, managedNetwork{name: builder.EESTBuildNetwork, mgr: mgr}) + } + } + // List orphaned ZFS resources. zfsResources, err := datadir.ListOrphanedZFSResources(ctx) if err != nil { @@ -113,8 +136,8 @@ func performCleanup(ctx context.Context, managers []docker.ContainerManager, for log.WithError(err).Warn("Failed to list CPU frequency state files") } - if len(containers) == 0 && len(volumes) == 0 && len(zfsResources) == 0 && - len(overlayMounts) == 0 && len(cpufreqStateFiles) == 0 { + if len(containers) == 0 && len(volumes) == 0 && len(networks) == 0 && + len(zfsResources) == 0 && len(overlayMounts) == 0 && len(cpufreqStateFiles) == 0 { log.Info("No benchmarkoor resources found") return nil @@ -137,6 +160,14 @@ func performCleanup(ctx context.Context, managers []docker.ContainerManager, for } } + if len(networks) > 0 { + fmt.Printf("\nNetworks to be removed (%d):\n", len(networks)) + + for _, n := range networks { + fmt.Printf(" - %s\n", n.name) + } + } + if len(zfsResources) > 0 { fmt.Printf("\nZFS resources to be removed (%d):\n", len(zfsResources)) @@ -200,6 +231,15 @@ func performCleanup(ctx context.Context, managers []docker.ContainerManager, for } } + // Remove networks (after their containers are gone). + for _, n := range networks { + log.WithField("network", n.name).Info("Removing network") + + if err := n.mgr.RemoveNetwork(ctx, n.name); err != nil { + log.WithError(err).WithField("network", n.name).Warn("Failed to remove network") + } + } + // Remove ZFS resources (clones first, then snapshots). if len(zfsResources) > 0 { if err := datadir.CleanupOrphanedZFSResources(ctx, log, zfsResources); err != nil { diff --git a/cmd/benchmarkoor/run.go b/cmd/benchmarkoor/run.go index 3ce952ace..7a64dab45 100644 --- a/cmd/benchmarkoor/run.go +++ b/cmd/benchmarkoor/run.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "os/signal" - "path/filepath" "strings" "syscall" "time" @@ -93,10 +92,15 @@ func runBenchmark(cmd *cobra.Command, args []string) error { } } - if dir := cfg.Runner.Directories.TmpCacheDir; dir != "" { - if err := fsutil.MkdirAll(dir, 0755, resultsOwner); err != nil { - return fmt.Errorf("creating tmp_cachedir %q: %w", dir, err) - } + // Resolve the shared cache dir (global.directories.cachedir, default + // ~/.cache/benchmarkoor) and ensure it exists. + cacheDir, err := cfg.ResolveCacheDir() + if err != nil { + return err + } + + if err := fsutil.MkdirAll(cacheDir, 0755, resultsOwner); err != nil { + return fmt.Errorf("creating cachedir %q: %w", cacheDir, err) } // Use consistent log format when client logs go to stdout. @@ -108,7 +112,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error { // instance's benchmarkoor.log, capturing logs from before RunInstance. // Created after the formatter is set so the buffered output matches the // per-instance log format. - preRunLogBuffer, err := runner.NewBufferHook(log.Formatter, cfg.Runner.Directories.TmpCacheDir) + preRunLogBuffer, err := runner.NewBufferHook(log.Formatter, cacheDir) if err != nil { return fmt.Errorf("creating pre-run log buffer: %w", err) } @@ -231,16 +235,6 @@ func runBenchmark(cmd *cobra.Command, args []string) error { var exec executor.Executor if cfg.Runner.Benchmark.Tests.Source.IsConfigured() { - cacheDir := cfg.Runner.Directories.TmpCacheDir - if cacheDir == "" { - var err error - - cacheDir, err = getExecutorCacheDir() - if err != nil { - return fmt.Errorf("getting cache directory: %w", err) - } - } - // Pass suite metadata to executor only when labels are present. var suiteMetadata *config.MetadataConfig if len(cfg.Runner.Benchmark.Tests.Metadata.Labels) > 0 { @@ -276,15 +270,6 @@ func runBenchmark(cmd *cobra.Command, args []string) error { // Create CPU frequency manager if CPU frequency settings are configured. var cpufreqMgr cpufreq.Manager if needsCPUFreqManager(cfg) { - cacheDir := cfg.Runner.Directories.TmpCacheDir - if cacheDir == "" { - var err error - cacheDir, err = getExecutorCacheDir() - if err != nil { - return fmt.Errorf("getting cache directory: %w", err) - } - } - cpufreqMgr = cpufreq.NewManager(log, cacheDir, cfg.GetCPUSysfsPath()) if err := cpufreqMgr.Start(ctx); err != nil { return fmt.Errorf("starting cpufreq manager: %w", err) @@ -328,7 +313,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error { GenesisURLs: cfg.Runner.Client.Config.Genesis, DataDirs: cfg.Runner.Client.DataDirs, TmpDataDir: cfg.Runner.Directories.TmpDataDir, - TmpCacheDir: cfg.Runner.Directories.TmpCacheDir, + CacheDir: cacheDir, TestFilter: cfg.Runner.Benchmark.Tests.Filter, FullConfig: cfg, StopAfterPrerun: stopAfterPrerun, @@ -391,16 +376,6 @@ func runBenchmark(cmd *cobra.Command, args []string) error { return nil } -// getExecutorCacheDir returns the cache directory for the executor. -func getExecutorCacheDir() (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("getting home directory: %w", err) - } - - return filepath.Join(homeDir, ".cache", "benchmarkoor"), nil -} - // needsCPUFreqManager returns true if any instance has CPU frequency settings configured. func needsCPUFreqManager(cfg *config.Config) bool { // Check global resource limits. diff --git a/config.example.yaml b/config.example.yaml index b92371d5e..0ec1ce07a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -6,6 +6,11 @@ global: log_level: ${LOG_LEVEL:-info} + # Optional directory configurations shared by the build and run commands. + # directories: + # # On-disk cache: executor git/archive clones (run) and the EEST repo clone + # # (build). Defaults to ~/.cache/benchmarkoor. + # cachedir: ${TMP_DIR:-/tmp}/benchmarkoor-cache runner: # Container runtime: "docker" (default) or "podman". @@ -24,8 +29,7 @@ runner: # directories: # # Directory for temporary datadir copies (defaults to system temp). # tmp_datadir: ${TMP_DIR:-/tmp}/benchmarkoor - # # Directory for executor cache - git clones, etc (defaults to ~/.cache/benchmarkoor). - # tmp_cachedir: ${TMP_DIR:-/tmp}/benchmarkoor-cache + # # (The shared cache dir lives under global.directories.cachedir.) # Optional: Override path to drop_caches file (default: /proc/sys/vm/drop_caches). # Useful when running in containers where the file is mounted at a different path. # drop_caches_path: /proc/sys/vm/drop_caches @@ -266,6 +270,49 @@ runner: # client: reth # output_dir: /srv/state/reth-spec # # inherits the top-level spec/spec_file and every field from `config` +# +# # Optional: generate stateful EEST benchmark fixtures with fill-stateful. +# # Boots a filler client on a copy of a snapshot datadir (e.g. a state_actor +# # output_dir above), records engine-API payloads, and writes fixtures that +# # `run` consumes via tests.source.eest_fixtures. Runs after state_actor. +# # Only geth supports fill-stateful's testing_buildBlockV1 today. Build the +# # fill image from the repo's Dockerfile.eest-filler (execution-specs #2637): +# # docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +# eest_payloads: +# fill_image: ghcr.io/your-org/eest-fill-stateful:latest +# # Instead of a pre-built fill_image, point at a Dockerfile and benchmarkoor +# # builds it (with the container runtime) at build time, tagging it fill_image +# # if set or a default local tag. One of fill_image / fill_dockerfile is required. +# # fill_dockerfile: Dockerfile.eest-filler +# # pull_policy: always # always | if-not-present | never +# # container_runtime: docker # defaults to runner.container_runtime, then docker +# # jwt: # Engine API secret shared with the filler (default: built-in) +# # fill_command: [uv, run, fill-stateful] # argv prefix inside fill_image (default) +# # EEST checkout cloned at build time and mounted into the fill container at +# # /eest (fill_image is just the uv/python toolchain). Cached on disk; re-cloned +# # only when the ref changes. Both default as shown. +# # eest_repo: https://github.com/ethereum/execution-specs.git +# # eest_ref: forks/amsterdam # branch, tag, or commit to check out +# config: # shared per-target defaults; targets override when set +# filler_image: ethpandaops/geth:master +# fork: Osaka +# gas_benchmark_values: [10, 30] # millions of gas to parametrise against +# # fixed_opcode_count parametrises by opcode count (thousands) instead of gas; +# # mutually exclusive with gas_benchmark_values. [] passes the flag bare, which +# # uses the fill image's .fixed_opcode_counts.json default. +# # fixed_opcode_count: [0.5, 1, 2] +# datadir_method: copy # copy | overlayfs | fuse-overlayfs | zfs | direct | schelk +# targets: +# - name: compute-geth +# filler_client: geth # only geth is supported today +# source_dir: /srv/state/geth-5g # PRISTINE snapshot (never mutated) +# genesis_file: /srv/state/geth-5g/genesis.json # chain config for the filler boot +# output_dir: /srv/fixtures/compute +# tests: +# - tests/benchmark/compute # pytest paths inside the fill image +# # filter: bn128 # optional pytest -k expression +# # force: true # wipe output_dir before filling +# # address_stubs_file: /etc/benchmarkoor/stubs.json # for stub-dependent tests # Optional: API server for authentication and user management. # When configured, the UI can integrate with the API for login, admin, and role-based access. diff --git a/docs/configuration.md b/docs/configuration.md index 5e5184423..40c38efd1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,6 +6,8 @@ This document describes all configuration options for benchmarkoor. The [config. - [Overview](#overview) - [Environment Variables](#environment-variables) + - [Config-local variables (`global.env`)](#config-local-variables-globalenv) + - [Environment Variable Overrides](#environment-variable-overrides) - [Configuration Merging](#configuration-merging) - [Global Settings](#global-settings) - [Runner Settings](#runner-settings) @@ -52,6 +54,23 @@ runner: results_dir: ${RESULTS_DIR:-./results} ``` +### Config-local variables (`global.env`) + +`global.env` declares variables inside the config itself, available to the same `${VAR}` / `${VAR:-default}` substitution everywhere in the file. This keeps a config self-contained — no need to `export` a value before running — while preserving the single-point-of-edit indirection: + +```yaml +global: + env: + STATE_DIR: /tmp/benchmarkoor/state-actor/simple-amsterdam-compute +builder: + state_actor: + targets: + - client: geth + output_dir: ${STATE_DIR}/geth # → /tmp/benchmarkoor/state-actor/simple-amsterdam-compute/geth +``` + +Resolution order for any `${VAR}` is **shell environment → `global.env` → inline `:-default`**. A real environment variable of the same name therefore still wins, so `global.env` acts as a per-config default that CI or an ad-hoc `VAR=… benchmarkoor …` invocation can override. A `global.env` value may itself reference the shell environment (e.g. `${BASE:-/tmp}/state-actor`); values do not reference one another. + ### Environment Variable Overrides Configuration values can also be overridden via environment variables with the `BENCHMARKOOR_` prefix. The variable name is derived from the config path using underscores: @@ -83,6 +102,10 @@ The `global` section contains application-wide settings. ```yaml global: log_level: info + env: + STATE_DIR: /tmp/benchmarkoor/state-actor/my-config + directories: + cachedir: ~/.cache/benchmarkoor ``` ### Options @@ -90,6 +113,8 @@ global: | Option | Type | Default | Description | |--------|------|---------|-------------| | `log_level` | string | `info` | Logging level: `debug`, `info`, `warn`, `error` | +| `env` | map[string]string | – | Config-local variables for `${VAR}` substitution; a per-config default that a shell env var of the same name still overrides. See [Config-local variables](#config-local-variables-globalenv). | +| `directories.cachedir` | string | `~/.cache/benchmarkoor` | On-disk cache shared by both commands: executor git/archive clones (`run`) and the EEST repo clone (`build`). | ## Runner Settings @@ -104,7 +129,6 @@ runner: run_timeout: 4h directories: tmp_datadir: /tmp/benchmarkoor - tmp_cachedir: /tmp/benchmarkoor-cache drop_caches_path: /proc/sys/vm/drop_caches cpu_sysfs_path: /sys/devices/system/cpu ``` @@ -118,8 +142,7 @@ runner: | `container_network` | string | `benchmarkoor` | Container network name | | `cleanup_on_start` | bool | `false` | Remove leftover containers/networks on startup | | `run_timeout` | string | - | Global timeout for the entire run covering all instances, setup, and teardown. Uses Go duration format (e.g., `4h`, `30m`). See [Runner Run Timeout](#runner-run-timeout) | -| `directories.tmp_datadir` | string | system temp | Directory for temporary datadir copies | -| `directories.tmp_cachedir` | string | `~/.cache/benchmarkoor` | Directory for executor cache (git clones, etc.) | +| `directories.tmp_datadir` | string | system temp | Directory for temporary datadir copies. (The shared cache dir is `global.directories.cachedir`.) | | `drop_caches_path` | string | `/proc/sys/vm/drop_caches` | Path to Linux drop_caches file (for containerized environments) | | `cpu_sysfs_path` | string | `/sys/devices/system/cpu` | Base path for CPU sysfs files (for containerized environments where `/sys` is read-only and the host path is bind-mounted elsewhere, e.g., `/host_sys_cpu`) | | `metadata.labels` | map[string]string | - | Arbitrary key-value labels attached to the run (see [Metadata Labels](#metadata-labels)) | @@ -944,6 +967,8 @@ When both `retry_new_payloads_syncing_state` and `retry_new_payloads_failed_stat Some clients (e.g., Erigon) may still be performing internal initialization or syncing after their RPC endpoint becomes available. The `bootstrap_fcu` option sends an `engine_forkchoiceUpdatedV3` call in a retry loop after RPC is ready, using the latest block hash from `eth_getBlockByNumber("latest")`. The client accepting the FCU with `VALID` status confirms it has finished syncing and is ready for test execution. +> **Besu** accepts the bootstrap FCU on an isolated snapshot node only with `--p2p-enabled=true`: its synchronizer must run to register the post-merge head as in-sync, otherwise besu answers `SYNCING` to every FCU. Set `extra_args: [--p2p-enabled=true]` on the besu instance (`--max-peers=0` + `--discovery-enabled=false` keep it isolated, with zero real peers). + **Shorthand** (uses defaults: `max_retries: 30`, `backoff: 1s`): ```yaml @@ -1158,6 +1183,8 @@ runner: | `restart` | string | No | - | Container restart policy | | `environment` | map | No | - | Additional environment variables | | `genesis` | string | No | From `runner.client.config.genesis` | Override genesis file URL | +| `genesis_fork_override` | map | No | - | Activate forks at given timestamps by patching a geth-format genesis at boot. See [Genesis Fork & EIP Overrides](#genesis-fork--eip-overrides) | +| `genesis_eip_override` | object | No | - | Activate EIPs at a timestamp by patching a parity/nethermind chainspec at boot. See [Genesis Fork & EIP Overrides](#genesis-fork--eip-overrides) | | `datadir` | object | No | From `runner.client.datadirs` | Instance-specific data directory config | | `drop_memory_caches` | string | No | From `runner.client.config` | Instance-specific cache drop setting | | `rollback_strategy` | string | No | From `runner.client.config` | Instance-specific rollback strategy | @@ -1172,6 +1199,45 @@ runner: | `bootstrap_fcu` | bool/object | No | From `runner.client.config` | Instance-specific bootstrap FCU setting | | `opcode_extraction` | object | No | From `runner.client.config` | Instance-specific opcode extraction setting (replaces global) | +#### Genesis Fork & EIP Overrides + +These options let an instance activate a fork that is not scheduled in the genesis it boots from — for example, running Amsterdam payloads against an Osaka snapshot. benchmarkoor patches the genesis file in-memory at boot, before mounting it; the source genesis on disk is never modified, and untouched fields (including large integers) round-trip verbatim, so the genesis block hash is unchanged. + +Use these only for clients that read their fork schedule from the genesis file. **geth and erigon do not** — they read the fork schedule from the datadir, so a patched genesis is ignored. For those, use the client's own fork-override flag instead (e.g. `--override.amsterdam=` in `extra_args`). + +**`genesis_fork_override`** — for geth-format genesis files (besu, reth, ethrex). A map of fork name to activation timestamp. For each entry it sets `config.Time`, and if the genesis has a `blobSchedule` that lacks the fork, it inherits the schedule of the latest preceding fork (so the new fork carries a blob schedule, as geth-family clients require). + +```yaml +runner: + instances: + - id: besu + client: besu + genesis: /path/to/osaka-chainspec.json # used as-is + genesis_fork_override: + amsterdam: 1 # sets config.amsterdamTime=1, inherits blobSchedule.amsterdam +``` + +**`genesis_eip_override`** — for parity/nethermind-format chainspecs, which schedule forks per-EIP rather than by fork name. It sets `params.eipTransitionTimestamp` for each listed EIP to the given (hex-encoded) timestamp. The EIP list is devnet-specific, so it lives in config. + +```yaml +runner: + instances: + - id: nethermind + client: nethermind + genesis: /path/to/osaka-parity-chainspec.json # used as-is + genesis_eip_override: + timestamp: 1 + eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] +``` + +| Option | Type | Description | +|--------|------|-------------| +| `genesis_fork_override` | map[string]uint | Fork name → activation timestamp (unix seconds). geth-format genesis only. | +| `genesis_eip_override.timestamp` | uint | Activation timestamp (unix seconds) applied to every listed EIP. | +| `genesis_eip_override.eips` | []uint | EIP numbers to activate, e.g. `[7928, 8037]`. parity/nethermind chainspec only. | + +Applying an override to the wrong genesis format is an error (a geth-format override needs a top-level `config` object; an EIP override needs a top-level `params` object). + ## Resource Limits Resource limits can be configured globally (`runner.client.config.resource_limits`) or per-instance (`runner.instances[].resource_limits`). Instance-level settings override global defaults. @@ -1382,9 +1448,12 @@ runner: ## Builder -The `builder` section configures tools that pre-populate client datadirs on disk. Today the only builder is `state-actor` (https://github.com/ethereum/state-actor), which writes per-client genesis state directly in each EL's native on-disk format — geth Pebble, reth MDBX, besu/nethermind RocksDB — bypassing the client's normal genesis-replay path. +The `builder` section configures tools that pre-populate benchmark inputs on disk. There are two builders: + +- **`state_actor`** (https://github.com/ethereum/state-actor) writes per-client genesis state directly in each EL's native on-disk format — geth Pebble, reth MDBX, besu/nethermind RocksDB — bypassing the client's normal genesis-replay path. +- **`eest_payloads`** generates stateful EEST benchmark fixtures by running [`fill-stateful`](https://github.com/ethereum/execution-specs/pull/2637) against a filler client booted on a pre-populated snapshot (typically one produced by `state_actor`). The fixtures are replayed by `benchmarkoor run`. -Builds are **decoupled from `benchmarkoor run`**: invoke `benchmarkoor build` to materialise datadirs, then run benchmarks against them via the regular `datadir.method: copy|zfs|schelk|…` providers. A missing datadir at `run` time is an error — it is never auto-built. +Builds are **decoupled from `benchmarkoor run`**: invoke `benchmarkoor build` to materialise the artifacts, then run benchmarks against them via the regular `datadir.method: copy|zfs|schelk|…` providers and test-source config. A missing datadir at `run` time is an error — it is never auto-built. When both builders are configured, they run in declaration order (`state_actor` before `eest_payloads`) so a fixture build can consume a datadir produced earlier in the same `benchmarkoor build` invocation. ### `builder.state_actor` options @@ -1403,8 +1472,8 @@ builder: container_runtime: docker # docker | podman (default: inherits runner.container_runtime, then docker) # spec source — top-level, shared across every target. # Pick at most one of: - # spec: | # inline YAML; benchmarkoor writes it to a temp file before invoking state-actor - # ... + # spec: # structured YAML (or a `|` block scalar); written to a temp file before invoking state-actor + # entities: [ ... ] # spec_file: /etc/benchmarkoor/state-spec.yaml # absolute host path config: # shared per-target defaults; targets override when set seed: 1 @@ -1422,7 +1491,7 @@ builder: | `images` | map[string]string | – | Per-client docker images for state-actor. Every active target's client must have an entry; state-actor needs a different cgo build per client (reth → MDBX, besu → RocksDB JNI, nethermind → .NET RocksDB). | | `pull_policy` | string | `always` | One of `always`, `if-not-present`, `never`. | | `container_runtime` | string | runner's runtime, then `docker` | Container runtime for the build container. | -| `spec` | string (YAML) | – | Inline state spec body (see [state-actor SPEC.md](https://github.com/ethereum/state-actor/blob/main/docs/SPEC.md)). Materialised to a temp file at build time. Mutually exclusive with `spec_file`. | +| `spec` | YAML mapping or string | – | Inline state spec body (see [state-actor SPEC.md](https://github.com/ethereum/state-actor/blob/main/docs/SPEC.md)). Write it as **structured YAML** (a mapping — your editor highlights it) or as a `\|` block scalar; both materialise to the same temp spec file at build time. Mutually exclusive with `spec_file`. | | `spec_file` | string | – | Absolute host path to a state spec YAML. Bind-mounted read-only into the build container. Mutually exclusive with `spec`. | | `config` | object | – | Shared defaults for the per-target build parameters. See below. | | `targets` | []object | – | Required when invoking `benchmarkoor build`. See below. | @@ -1455,7 +1524,7 @@ The fields below mirror `builder.state_actor.config`; any field set here overrid | Option | Type | Default | Applies to | Description | |---|---|---|---|---| | `name` | string | `client` | all | Human-readable name. Used by `--target` to filter. Must be unique across targets; defaults to the `client` field when omitted. | -| `client` | string | – | all | One of `geth`, `reth`, `besu`, `nethermind`. State-actor does not support `erigon` or `nimbus`. | +| `client` | string | – | all | One of `geth`, `reth`, `besu`, `nethermind`, `ethrex`. State-actor does not support `erigon` or `nimbus`. | | `output_dir` | string | – | all | Absolute host path. If the directory already contains entries, that target is **skipped** (no error) — pass `--force` (CLI) or set `force: true` here to wipe and rebuild. For geth, state-actor writes into `/geth/chaindata`. | | `target_size` | string | from `config` | all | Advisory size budget for auto-generated state, e.g. `5GB`, `500MB` (base-1024). Required for the target when no spec is configured; when a spec is configured (top-level or default), `target_size` is optional and acts as a headroom budget that state-actor fills past the spec's projected cost. | | `force` | bool | `false` | all | Per-target override of the CLI `--force` flag: wipes `output_dir` before building so state-actor sees a clean directory. Useful when most targets should skip-if-built but specific ones should always rebuild. | @@ -1469,22 +1538,42 @@ The fields below mirror `builder.state_actor.config`; any field set here overrid | `binary_trie` | bool | from `config`, then `false` | geth | EIP-7864 binary trie. Set `false` to opt out of a global default. Rejected (after resolution) for non-geth. | | `group_depth` | int | from `config`, then `8` (state-actor) | geth + binary_trie | Binary-trie serialisation unit. Range 1..8. Requires effective `binary_trie=true`. | -State-actor itself only writes the genesis block; subsequent blocks come from running a client against the produced datadir. See [state-actor RUNBOOK.md](https://github.com/ethereum/state-actor/blob/main/docs/RUNBOOK.md) for the per-client boot recipes (e.g. geth needs `--db.engine=pebble`; reth needs `--debug.skip-genesis-validation`; besu needs `--data-storage-format=BONSAI`). +State-actor itself only writes the genesis block; subsequent blocks come from running a client against the produced datadir. See [state-actor RUNBOOK.md](https://github.com/ethereum/state-actor/blob/main/docs/RUNBOOK.md) for the per-client boot recipes (e.g. geth needs `--db.engine=pebble`; reth needs `--debug.skip-genesis-validation`; besu needs `--data-storage-format=BONSAI`; ethrex needs `--skip-genesis-validation` and ≥ v16.0.0). ### Running ```bash -# Build every target declared under builder.state_actor.targets +# Build every target declared under builder.state_actor.targets / builder.eest_payloads.targets benchmarkoor build --config build.yaml -# Build only specific targets (by name) +# Build only specific targets by name, across all builders benchmarkoor build --config build.yaml --target geth-5g --target reth-spec +# Limit a single builder's targets (the other builder is unrestricted) +benchmarkoor build --config build.yaml --limit-state-actor-target nethermind +benchmarkoor build --config build.yaml --limit-eest-payload-target payload-generator-nethermind + +# Build just one client end-to-end: its snapshot, then its fill +benchmarkoor build --config build.yaml \ + --limit-state-actor-target nethermind \ + --limit-eest-payload-target payload-generator-nethermind + # Overwrite existing output_dir contents benchmarkoor build --config build.yaml --force ``` -The command exits non-zero if any target fails; successful targets are still left in place on partial failure. A final summary lists each target with `OK ` (built), `SKIP` (output_dir already populated), or `ERR ` (failed). `--force` wipes each target's `output_dir` before building, bypassing the skip behaviour. +| Flag | Description | +|---|---| +| `--target` | Filter by target `name` across **all** builders (comma-separated or repeated). | +| `--limit-state-actor-target` | Filter only `builder.state_actor` targets; `eest_payloads` is left unrestricted. | +| `--limit-eest-payload-target` | Filter only `builder.eest_payloads` targets; `state_actor` is left unrestricted. | +| `--skip-state-actor-build` | Skip the `builder.state_actor` builder entirely (only `eest_payloads` runs). | +| `--skip-eest-payload-build` | Skip the `builder.eest_payloads` builder entirely (only `state_actor` runs). | +| `--force` | Wipe each selected target's `output_dir` before building (bypasses the skip-if-populated behaviour). | + +A target is built when it passes the global `--target` filter **and** the per-builder limit for the builder that owns it; an unset filter imposes no restriction. Any filter value that names no existing target is a hard error (typos surface immediately — the per-builder limits are checked against only that builder's target names). `--skip-*-build` removes a whole builder; a skipped builder's `--limit-*-target` is then ignored. Skipping every configured builder is an error. + +The command exits non-zero if any target fails; successful targets are still left in place on partial failure. A final summary lists each target with `OK ` (built), `SKIP` (output_dir already populated), or `ERR ` (failed). ### Examples @@ -1533,23 +1622,146 @@ builder: archive: false # overrides config.archive=true (besu doesn't support archive) ``` -Inline spec — write the YAML directly in the config: +Inline spec — write the YAML directly in the config (structured, so editors highlight it; a `|` block scalar works too): ```yaml builder: state_actor: images: geth: ghcr.io/ethereum/state-actor:latest - spec: | - genesis: - chain_id: 1337 - gas_limit: 30000000 + spec: + entities: + - kind: eoa + name: bloated-eoa + approximate_size_bytes: 2_000_000_000 # … rest of the state spec targets: - client: geth output_dir: /srv/state/geth-spec ``` +### `builder.eest_payloads` options + +`eest_payloads` generates **stateful** EEST benchmark fixtures: it boots a filler EL client on a *writable copy* of a pre-populated snapshot datadir, runs `fill-stateful` against the live client (recording engine-API payloads anchored to the snapshot's head block), and writes the fixtures to each target's `output_dir`. `fill-stateful` itself does not manage datadirs — benchmarkoor boots the filler and snapshots it. + +> **Filler client:** `geth` (`ethpandaops/geth:master`) is the production-ready filler. `nethermind` (`nethermindeth/nethermind:master`) also works — it implements `testing_buildBlockV1` with correct EIP-7928 block-access-lists, and `fill-stateful`'s per-test rewind falls back to `debug_resetHead` for it (nethermind has no `debug_setHead`). `besu` works too with an image carrying the merged `TestingBuildBlockV1` coinbase fix (e.g. `ethpandaops/besu:bal-devnet-7`); benchmarkoor auto-pins its session priority fee. +> +> **Fill image:** there is no published `fill-stateful` image yet (the command lands in execution-specs [#2637](https://github.com/ethereum/execution-specs/pull/2637)). Build one from the repo's `Dockerfile.eest-filler` (it bundles `uv` + execution-specs) and point `fill_image` at it: +> ```bash +> docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +> ``` + +```yaml +builder: + eest_payloads: + fill_image: ghcr.io/your-org/eest-fill-stateful:latest # image carrying `uv run fill-stateful` + # fill_dockerfile: Dockerfile.eest-filler # build the fill image instead of pulling one + pull_policy: always # always | if-not-present | never (default: always) + container_runtime: docker # docker | podman (default: inherits runner.container_runtime, then docker) + # jwt: # Engine API secret, shared with the filler (default: benchmarkoor's DefaultJWT) + # fill_command: [uv, run, fill-stateful] # argv prefix inside fill_image (this is the default) + # eest_repo: https://github.com/ethereum/execution-specs.git # cloned + mounted at /eest (default) + # eest_ref: forks/amsterdam # branch, tag, or commit to check out (default: forks/amsterdam) + config: # shared per-target defaults; targets override when set + filler_image: ethpandaops/geth:master + fork: Osaka + gas_benchmark_values: [10, 30] # millions of gas to parametrise against + # fixed_opcode_count: [0.5, 1, 2] # thousands of opcodes; mutually exclusive with gas_benchmark_values + datadir_method: copy # copy | overlayfs | fuse-overlayfs | zfs | direct | schelk + targets: + - name: compute-geth + filler_client: geth + source_dir: /srv/state/geth-archive # PRISTINE snapshot (never mutated; a writable copy is filled) + # geth boots from the datadir; to fill a fork that activates after the + # snapshot, pass --override. here (besu/nethermind use `genesis` + + # genesis_fork_override / genesis_eip_override instead): + # filler_extra_args: [--override.amsterdam=1] + output_dir: /srv/fixtures/compute + tests: + - tests/benchmark/compute # pytest paths inside the fill image + filter: bn128 # optional pytest -k expression +``` + +| Option | Type | Default | Description | +|---|---|---|---| +| `fill_image` | string | – | Pre-built container image carrying the uv/python toolchain that runs `fill-stateful`. Required unless `fill_dockerfile` is set. | +| `fill_dockerfile` | string | – | Path to a Dockerfile (e.g. `Dockerfile.eest-filler`) that benchmarkoor builds with the container runtime at build time, instead of pulling a pre-built image. Tagged `fill_image` when set, else `benchmarkoor-eest-fill:local`. Requires the runtime's `build` CLI (docker/podman) on the host. One of `fill_image` / `fill_dockerfile` is required. | +| `pull_policy` | string | `always` | One of `always`, `if-not-present`, `never`. Applies to both the fill image and the filler image (ignored for a locally built `fill_dockerfile`). | +| `container_runtime` | string | runner's runtime, then `docker` | Container runtime for the filler + fill containers. | +| `jwt` | string | benchmarkoor's `DefaultJWT` | Engine API JWT secret; shared between the filler client and `fill-stateful`. | +| `fill_command` | []string | `[uv, run, fill-stateful]` | argv prefix invoked inside `fill_image` before the `fill-stateful` flags. Override if your image exposes the command differently. | +| `eest_repo` | string | `https://github.com/ethereum/execution-specs.git` | execution-specs repo cloned for filling. | +| `eest_ref` | string | `forks/amsterdam` | Branch, tag, or commit of `eest_repo`. benchmarkoor always clones the repo at this ref into an on-disk cache at build time and mounts the checkout into the fill container at `/eest` (the `fill_image` carries only the uv/python toolchain, not the repo), so the EEST version is config-driven and changeable without rebuilding the image. The clone is cached and re-fetched only when the ref changes; `uv` builds the venv into the mounted checkout on first use (cached across runs). | +| `config` | object | – | Shared defaults for the per-target parameters. See below. | +| `targets` | []object | – | Required when invoking `benchmarkoor build`. See below. | + +### `builder.eest_payloads.config` options + +Every field below is also available per-target; a non-nil/non-empty value on a target overrides the default. Use this block to avoid repeating shared knobs (`fork`, `tests`, `filter`, `address_stubs`, …) across targets that build the same suite. (Only the identity/locator fields — `name`, `filler_client`, `source_dir`, `output_dir`, `genesis`, `genesis_fork_override`, `genesis_eip_override` — are target-only and never hoisted.) + +| Option | Type | Default | Description | +|---|---|---|---| +| `filler_image` | string | – | Docker image for the filler client (e.g. `ethpandaops/geth:master`). | +| `fork` | string | – | Fork to fill against, e.g. `Osaka` (passed to `fill-stateful --fork`). | +| `tests` | string[] | – | pytest paths inside the fill image, e.g. `tests/benchmark/compute`. Required after resolution — set here or per-target. | +| `filter` | string | – | pytest `-k` expression (substring/node-id selection). | +| `marker` | string | – | pytest `-m` marker expression, orthogonal to `filter`'s `-k`, e.g. `repricing` / `not repricing`. | +| `address_stubs` | map | – | Inline `--address-stubs` map: stub name → arbitrary string fields (e.g. `addr`, `pkey`). Materialised to a temp JSON file at build time. Mutually exclusive with `address_stubs_file`. | +| `address_stubs_file` | string | – | **Absolute** host path to a `--address-stubs` JSON map. Mutually exclusive with `address_stubs`. | +| `gas_benchmark_values` | int[] | – | Gas budgets in millions, e.g. `[10, 30]`; joined into `--gas-benchmark-values`. Mutually exclusive with `fixed_opcode_count`. | +| `fixed_opcode_count` | float[] | – | Opcode counts in thousands, e.g. `[0.5, 1, 2]`; joined into `--fixed-opcode-count`. An empty list (`[]`) passes the flag bare, using the fill image's `.fixed_opcode_counts.json` default. Mutually exclusive with `gas_benchmark_values`. | +| `datadir_method` | string | `copy` | How the filler's writable copy of `source_dir` is prepared: `copy`, `overlayfs`, `fuse-overlayfs`, `zfs`, `direct`, `schelk`. Use `zfs`/`overlayfs` to avoid a full copy of a large snapshot. | +| `max_gas_per_test` | uint64 | – | Overrides the fork's transaction gas-limit cap (`--max-gas-per-test`). | +| `rpc_seed_key` | string | – | Pin the seed EOA for reproducible fills (`--rpc-seed-key`); otherwise one is generated and funded via CL withdrawal. | +| `filler_extra_args` | []string | – | Extra argv appended to the filler client command. | + +> **Address-stubs hoisting:** `address_stubs` / `address_stubs_file` hoist as a *unit* — a target that sets either form inherits neither from `config`, so their mutual exclusion is preserved. An inline `address_stubs` example: +> ```yaml +> address_stubs: +> bloated_eoa_10GB: +> addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" +> pkey: "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" +> ``` + +### `builder.eest_payloads.targets[]` options + +Identity/locator fields are target-only; the rest mirror `config` and are resolved with per-target precedence. + +| Option | Type | Default | Description | +|---|---|---|---| +| `name` | string | `filler_client` | Used by `--target` to filter. Must be unique across targets. | +| `filler_client` | string | – | Client booted as the filler: `geth`, `nethermind`, or `besu` (all implement `testing_buildBlockV1`). | +| `source_dir` | string | – | **Absolute** host path to the pristine snapshot datadir (e.g. a `state_actor` `output_dir`). Never mutated — a writable copy is filled. Existence is checked at build time. | +| `genesis` | string | – | **Absolute** host path to the genesis/chainspec the filler boots with (besu/nethermind read their fork schedule from it; passed via the client's genesis flag). Must match the chain config used to produce `source_dir`. geth/erigon boot from the datadir instead and need no `genesis`. | +| `genesis_fork_override` | map | – | Patch the geth-format `genesis` at filler boot to activate forks at given timestamps (`{amsterdam: 1}` → `config.amsterdamTime`, inheriting the blob schedule). For besu/reth/ethrex fillers. Same mechanism as the runner. Requires `genesis`. | +| `genesis_eip_override` | object | – | Patch a parity/nethermind `genesis` at filler boot, setting `params.eipTransitionTimestamp` for each listed EIP. Fields: `timestamp` (uint), `eips` ([]uint). For the nethermind filler. Requires `genesis`; mutually exclusive with `genesis_fork_override`. | +| `output_dir` | string | – | **Absolute** host path for the generated fixtures. Skipped if already populated unless `--force` / `force: true`. Written under `/blockchain_tests_stateful_engine/`. | +| `force` | bool | `false` | Per-target override of `--force`: wipe `output_dir` before filling. | +| `filler_image`, `fork`, `tests`, `filter`, `marker`, `address_stubs`, `address_stubs_file`, `gas_benchmark_values`, `fixed_opcode_count`, `datadir_method`, `max_gas_per_test`, `rpc_seed_key`, `filler_extra_args` | — | from `config` | Mirror `config` with per-target precedence — see the `config` table above. `tests`, `fork`, and `filler_image` are required after resolution (set on the target or in `config`). | + +### Replaying generated fixtures + +Point `benchmarkoor run` at the **pristine** snapshot (never the copy the filler mutated) and at the fixture output: + +```yaml +runner: + client: + datadirs: + geth: + source_dir: /srv/state/geth-archive # the pristine snapshot + method: zfs # or copy/overlayfs/… + benchmark: + tests: + source: + eest_fixtures: + local_fixtures_dir: /srv/fixtures/compute + fixtures_subdir: blockchain_tests_stateful_engine +``` + +> Stateful replay needs the new fixture format support — see benchmarkoor [#182](https://github.com/ethpandaops/benchmarkoor/pull/182). + +As a sanity check, each fixture's recorded `benchmarkGasUsed` should match benchmarkoor's measured `gas_used_total` for that test. + ## API Server See [API Server documentation](api.md) for the full reference on the `api` config section, including server settings, authentication, database, storage, endpoints, and UI integration. diff --git a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml new file mode 100644 index 000000000..2465579de --- /dev/null +++ b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml @@ -0,0 +1,302 @@ +# Building data directories and EEST FULL STATEFUL (bloatnet / repricing) payloads — AMSTERDAM fill. +# +# This is the "full" counterpart of config.state-actor-eest.simple.amsterdam.stateful.yaml: +# a large repricing prestate (jochem-brouwer's production-minimum spec, +# https://gist.github.com/jochem-brouwer/5da19ef1e96edcf645b3dd590b5604bd) that +# backs the tests/benchmark/stateful/bloatnet repricing benchmarks. The geth +# snapshot alone is ~77 GB on disk and takes 30-90 min to build. +# +# ── Run this in a Linux env with a COW datadir method (overlayfs/zfs/schelk) ── +# macOS has none of these, and `copy` is infeasible at 77 GB (copied per fill and +# per client per run). Provision an OrbStack Linux machine with .hack/orbstack.sh +# (dedicated dockerd + overlayfs + a native benchmarkoor build), then run as root +# INSIDE the machine (overlayfs mount needs CAP_SYS_ADMIN): +# +# .hack/orbstack.sh bench +# orb -m bench sudo benchmarkoor build --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --force +# orb -m bench sudo benchmarkoor run --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --limit-instance-id=geth +# +# Amsterdam is activated exactly as in the other configs: geth/erigon via the +# --override.amsterdam flag, besu/reth/ethrex via genesis_fork_override, +# nethermind via genesis_eip_override. (Per the assume-Amsterdam decision; the +# gist itself notes the repricing actually ships in glamsterdam.) +# +# NB: geth, nethermind, ethrex, reth and besu are enabled by default — each +# client needs its OWN multi-hundred-GB snapshot (state-actor builds one per +# target), ~1.7 TB for these five. erigon is left commented out: its state-actor +# build OOMs a 58 GB VM in the commitment phase — re-enable with more RAM. +# +# STATUS: validated end-to-end in the OrbStack VM — geth fills 225 bloatnet +# cases (0 failed) against this prestate, and the geth/nethermind genesis state +# roots match. The excluded tests (see the filter note on the geth target) need +# prestate/stub coverage this gist doesn't provide. +# +global: + log_level: info + env: + # Per-config dirs so configs that build different state-actor snapshots or + # EEST fixtures don't collide under /tmp. Override by exporting these. Each + # client writes under its own / subdir. + STATE_DIR: /tmp/benchmarkoor/state-actor/full-amsterdam-stateful + EEST_FIXTURES_DIR: /tmp/benchmarkoor/eest-fixtures/full-amsterdam-stateful + EEST_FIXTURES_RUNNER_SOURCE: geth # which filler's fixtures the runner replays + +builder: + ## Stage 1: Building the large repricing datadir using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + ethrex: ghcr.io/ethereum/state-actor-ethrex:main + reth: ghcr.io/ethereum/state-actor-reth:main + besu: ghcr.io/ethereum/state-actor-besu:main + # erigon: ghcr.io/ethereum/state-actor-erigon:main + pull_policy: always + config: + seed: 42 # gist uses --seed=42 (reproducible addresses) + fork: osaka # snapshot baseline (amsterdam is forked in at fill/run) + chain: 1337 + # gist minimum is 85M (an amsterdam max-size code deploy needs gas_limit + # >= 67M). 1000M (1 G) gives generous headroom for the benchmark setup + # blocks. NB: this does NOT make test_deep_branch fillable — its worst-case + # deep-MPT setup needs a single block > 1 G, and EIP-1559 caps the per-block + # ramp to +/-1/1024, so the limit can't jump there (verified at 1000M; deep + # stays excluded below). All 225 other bloatnet cases fill fine at any of + # these values; the funding fixes, not the gas_limit, are what matter. + gas_limit: 1000000000 + # Must exceed the prestate size (~77 GB) or state-actor truncates entities. + target_size: 200GB + # jochem-brouwer's production-minimum repricing prestate. The big cost is + # entity "storage-pattern-10gb" (335M slots ≈ 73 GB); the 150000-count knobs + # give 300M-gas cold-access headroom (count >= gas/2000). + spec: + entities: + # Pre-funded seed for fill-stateful — Hardhat/Anvil dev key #0 + # (pkey 0xac0974…ff80). Pinned below via rpc_seed_key. + - kind: eoa + name: fill-stateful-seed + address: 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 + balance: "0x33b2e3c91efc989409c0000" + # Sequential EOAs at 0x1000+ (test_account_access EXISTING_EOA). + - kind: contract + name: sequential-eoas-300m + template: sequential_eoas + address: 0x0000000000000000000000000000000000001000 + parameters: + count: 150000 + balance: "1000000000000000000" + # 10GB storage-pattern target at the bloated_eoa_10GB stub address + # (test_sload_bloated / test_sstore_bloated, existing_slots=True). ~73 GB. + - kind: contract + name: storage-pattern-10gb + template: storage_pattern + address: 0x87a6314da5ac8832f6e7a176c8fb133b19f5be04 + nonce: 1 + balance: "1000000000000000000" + parameters: + final: 335545082 + # Arachnid CREATE2 factory (required by create2_deploys + EEST fill). + - kind: contract + name: create2-factory + template: create2_factory + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + # 150000 CREATE2 contracts with UNIQUE 24KB runtime (cold-contract access + # / test_transaction_types). ~3.7 GB resident code during build. + - kind: contract + name: create2-deploys-300m + template: create2_deploys + parameters: + code_pattern: unique_jumpdest_pre_amsterdam + salt_count: 150000 + # Sender pool for test_ether_transfers_onchain_receivers, which derives + # senders as EOA(key=SENDER_BASE_KEY + i) and assumes them pre-funded. + # SENDER_BASE_KEY = keccak256("gas-repricings-private-key") on this EEST + # branch (NOT the gist's 0x1111…, which left every sender unfunded); the + # first sender is 0x4e5e4cbb…377a. 150000 funded. + - kind: contract + name: sender-pool-300m + template: sequential_pkey_eoas + parameters: + start_pkey: "0xe1e1d3457c4e69b29cba0f7e1f92ce080d4db56d221bed913b09b2753bd97c7a" + count: 150000 + balance: "1000000000000000000" + # Bittrex CREATE-preimage chain (test_account_access EXISTING_CONTRACT). + - kind: contract + name: bittrex-create-preimage-300m + template: create_preimage_deploys + parameters: + sender: "0xA3C1E324CA1CE40DB73ED6026C4A177F099B5770" + start_nonce: 2 + count: 150000 + runtime: "0x606060405236156100495763ffffffff7c01000000000000000000000000000000000000000000000000000000006000350416636ea056a98114610052578063c0ee0b8a14610092575b6100505b5b565b005b341561005a57fe5b61007e73ffffffffffffffffffffffffffffffffffffffff60043516602435610104565b604080519115158252519081900360200190f35b341561009a57fe5b604080516020600460443581810135601f810184900484028501840190955284845261005094823573ffffffffffffffffffffffffffffffffffffffff169460248035956064949293919092019181908401838280828437509496506101ef95505050505050565b005b6000805460408051602090810184905281517f3c18d31800000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff878116600483015292519290931692633c18d318926024808301939282900301818787803b151561017b57fe5b6102c65a03f1151561018957fe5b5050506040518051905073ffffffffffffffffffffffffffffffffffffffff1660003660006040516020015260405180838380828437820191505092505050602060405180830381856102c65a03f415156101e057fe5b50506040515190505b92915050565b5b5050505600a165627a7a723058204cdd69fdcf3cf6cbee9677fe380fa5f044048aa9e060ec5619a21ca5a5bd4cd10029" + storage_init: + "0x0": "0xa3c1e324ca1ce40db73ed6026c4a177f099b5770" + targets: + - client: geth + output_dir: ${STATE_DIR}/geth # ~338 GB on disk once built + - client: nethermind + output_dir: ${STATE_DIR}/nethermind # ~248 GB on disk once built + - client: ethrex + output_dir: ${STATE_DIR}/ethrex # ~436 GB; snapshot only (ethrex isn't a fill-stateful filler) + - client: reth + output_dir: ${STATE_DIR}/reth # ~353 GB; snapshot only (reth isn't a fill-stateful filler) + - client: besu + output_dir: ${STATE_DIR}/besu # ~329 GB + # - client: erigon # OOMs the state-actor build on a 58 GB VM (huge-storage commitment phase) + # output_dir: ${STATE_DIR}/erigon + + ## Stage 2: Building EEST bloatnet/repricing fixtures + eest_payloads: + fill_dockerfile: Dockerfile.eest-filler + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + # devnets/bal/7-bench carries the BAL benchmark tooling plus the fill-stateful + # fixes that let nethermind drive fill-stateful (debug_resetHead fallback, + # receipt `error` strip, and start_block-not-"latest" nonce/rewind reads). + # Geth filling is unaffected by them. + eest_ref: devnets/bal/7-bench + config: + fork: amsterdam + # Hardhat/Anvil dev key #0 → 0xf39Fd6…2266 (the pre-funded seed above). + rpc_seed_key: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + # The bloatnet prestate has ample gas headroom; start lower for faster fills. + gas_benchmark_values: [30] + # COW is mandatory at this size — overlayfs (Linux/OrbStack). On the host + # this would be `copy` (infeasible). See .hack/orbstack.sh. + datadir_method: overlayfs + # Both targets fill the SAME bloatnet set against the SAME prestate, so + # tests / filter / address_stubs are hoisted here once instead of repeated + # per target. Any target may still override an individual field. + tests: + - tests/benchmark/stateful/bloatnet + # Resolves @pytest.mark.stub_parametrize prefixes against pre-deployed + # state (e.g. bloated_eoa_10GB → 0x87a6…be04). + address_stubs: + bloated_eoa_10GB: + addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" + pkey: "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" + filter: >- + not erc20 + and not test_multi_opcode + and not test_setup_contracts + and not test_deep_branch + and not test_extcodesize_bytecode_sizes + and not test_sload_bloated_prefetch_miss + and not (test_sstore_variants and zero_to_nonzero) + and not (test_sload_bloated_multi_contract and existing_slots_True) + targets: + # Geth + - name: payload-generator-geth-stateful-full + filler_client: geth + filler_image: skylenet/geth:bal-devnet-7-amsterdam-override + source_dir: ${STATE_DIR}/geth + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 + output_dir: ${EEST_FIXTURES_DIR}/geth + + # Nethermind + - name: payload-generator-nethermind-stateful-full + filler_client: nethermind + filler_image: nethermindeth/nethermind:master + source_dir: ${STATE_DIR}/nethermind + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + output_dir: ${EEST_FIXTURES_DIR}/nethermind + +## Stage 3: Run the bloatnet/repricing benchmarks +runner: + client_logs_to_stdout: true + cleanup_on_start: false + directories: + # Scratch dir for the overlayfs/copy datadir prep (default: system temp). + # reth keeps state in one monolithic MDBX file that gets copied here, so on a + # host where /tmp is a small tmpfs (e.g. the OrbStack VM) set TMP_DATADIR to a + # real disk (e.g. /root/bench/tmp) or reth dies with "No space left on + # device". RocksDB/Pebble clients (geth/nethermind/besu/ethrex) are fine on + # the default. + tmp_datadir: ${TMP_DATADIR:-} + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR}/${EEST_FIXTURES_RUNNER_SOURCE} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + # rollback_strategy: container-recreate # recreate the container and rollback snapshot datadir + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + # Per-client genesis/chainspec the runner boots each instance from. geth + # needs none (it boots from the datadir + --override.amsterdam flag), so it + # is omitted; nethermind reads a parity-format chainspec and gets amsterdam + # via the per-instance genesis_eip_override below. + genesis: + nethermind: ${STATE_DIR}/nethermind/parity-chainspec.json + ethrex: ${STATE_DIR}/ethrex/ethrex-genesis.json + reth: ${STATE_DIR}/reth/chainspec.json + besu: ${STATE_DIR}/besu/besu-chainspec.json + # erigon: ${STATE_DIR}/erigon/chainspec.json + datadirs: + geth: + source_dir: ${STATE_DIR}/geth + method: overlayfs + nethermind: + source_dir: ${STATE_DIR}/nethermind + method: overlayfs + ethrex: + source_dir: ${STATE_DIR}/ethrex + method: overlayfs + # reth's overlay copies a monolithic MDBX file — on a tmpfs /tmp set + # TMP_DATADIR to a real disk (see runner.directories.tmp_datadir above). + reth: + source_dir: ${STATE_DIR}/reth + method: overlayfs + besu: + source_dir: ${STATE_DIR}/besu + method: overlayfs + # erigon: { source_dir: ${STATE_DIR}/erigon, method: overlayfs } + + instances: + - id: geth + client: geth + image: skylenet/geth:bal-devnet-7-amsterdam-override + extra_args: + - --override.amsterdam=1 + # Other clients need their own snapshot (state_actor target above) + their + # genesis/override wiring — mirror config.state-actor-eest.simple.amsterdam.stateful.yaml. + - id: reth + client: reth + image: ethpandaops/reth:bal-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: [--engine.slow-block-threshold=0] + retry_new_payloads_failed_state: { enabled: true, max_retries: 10, backoff: 1s } + - id: nethermind + client: nethermind + image: nethermindeth/nethermind:master + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + extra_args: [--Init.BaseDbPath=/data, --Blocks.ParallelExecution=true, --Blocks.ParallelExecutionBatchRead=true] + - id: besu + client: besu + image: ethpandaops/besu:bal-devnet-7 + genesis_fork_override: { amsterdam: 1 } + environment: { BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" } + extra_args: [--p2p-enabled=true, --Xplugin-rocksdb-high-spec-enabled=true, --Xbal-perfect-parallelization-enabled=true, --Xbal-state-root-enabled=true, --Xbal-prefetch-reading-enabled=true, --Xbal-prefetch-batch-size=8, --Xbal-processing-timeout=-1, --Xbal-state-root-timeout=-1] + - id: ethrex + client: ethrex + image: ethpandaops/ethrex:bal-devnet-7 + genesis_fork_override: { amsterdam: 1 } + extra_args: [--skip-genesis-validation, --no-precompile-cache] + # - id: erigon + # client: erigon + # image: ethpandaops/erigon:bal-devnet-7 + # extra_args: [--override.amsterdam=1, --fcu.background.commit=false, --exec.no-merge=true, --exec.no-prune=true, --exec.no-background-maintenance] + # bootstrap_fcu: { enabled: true, max_retries: 120, backoff: 30s } diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml new file mode 100644 index 000000000..afc71f696 --- /dev/null +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml @@ -0,0 +1,386 @@ +# Building data directories and EEST COMPUTE payloads — AMSTERDAM fill. +# +# state-actor generates OSAKA snapshot datadirs (stage 1); the eest_payloads stage +# fills AMSTERDAM tests/benchmark/compute fixtures by booting the geth filler on the +# osaka snapshot and activating amsterdam at block 0's timestamp + 1 via geth's +# --override.amsterdam=1 flag (filler_extra_args). So block 0 stays osaka and the +# first benchmark block (block 1) is amsterdam. +# +# Stateful (EIP-7928 / BAL) fixtures are built by the sibling config: +# config.state-actor-eest.simple.amsterdam.stateful.yaml +# Both configs build the same state-actor snapshot — build either one first. +# +# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml --force +# +# Running the benchmark (one instance per client): +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml --limit-instance-id=geth +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml --limit-instance-id=besu +# +# Each client activates amsterdam on the osaka snapshot one of three ways: +# - geth, erigon: CLI fork override (--override.amsterdam=) +# - besu, reth, ethrex: genesis_fork_override (benchmarkoor sets Time in +# the geth-format genesis at boot) +# - nethermind: genesis_eip_override (benchmarkoor sets the amsterdam +# EIPs' eipTransitionTimestamp in the parity chainspec) +# Snapshot block 0's timestamp is 0, so amsterdam is scheduled at ts 1. +# +global: + log_level: info + env: + # Per-config dirs so configs that build different state-actor snapshots or + # EEST fixtures don't collide under /tmp. Override by exporting these. Each + # client writes under its own / subdir. + STATE_DIR: /tmp/benchmarkoor/state-actor/simple-amsterdam-compute + EEST_FIXTURES_DIR: /tmp/benchmarkoor/eest-fixtures/simple-amsterdam-compute + EEST_FIXTURES_RUNNER_SOURCE: geth # which filler's fixtures the runner replays + +builder: + ## Stage 1: Building datadirs using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + reth: ghcr.io/ethereum/state-actor-reth:main + besu: ghcr.io/ethereum/state-actor-besu:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + ethrex: ghcr.io/ethereum/state-actor-ethrex:main + erigon: ghcr.io/ethereum/state-actor-erigon:main + pull_policy: always + config: + seed: 1234 + fork: osaka + chain: 1337 + gas_limit: 300000000 # 300M + target_size: 256MB + #target_size: 20GB + spec: + entities: + # Predeploy the Arachnid deterministic-deployment proxy at the canonical + # 0x4e59…956c. EEST's fill bootstrap deploys it via a keyless tx with a + # fixed 100k gas limit, which amsterdam's state-expansion gas (EIP-7928/ + # 8037, ~324k for this proxy) exceeds — so it must be predeployed. EEST + # detects the existing bytecode and skips the keyless deploy. + - kind: contract + template: create2_factory + name: deterministic-deployment-proxy + # Explicit canonical address: with a name set, the address would + # otherwise be name-derived instead of the Arachnid default. + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + # Pre-fund fill-stateful's seed account (rpc_seed_key=0x..01 → this + # address). With the seed already funded, fill-stateful skips the + # withdrawal funding block, so start_block == the snapshot block, whose + # persistent state debug_setHead can always rewind to between tests. + # Balance must be >= SEED_FUNDING_WEI (1e27); 1e28 gives headroom. + - kind: eoa + name: fill-stateful-seed + address: 0x7e5f4552091a69125d5dfcb7b8c2659029395bdf + balance: "10000000000000000000000000000" + nonce: 0 + targets: + - client: geth + output_dir: ${STATE_DIR}/geth + - client: reth + output_dir: ${STATE_DIR}/reth + - client: nethermind + output_dir: ${STATE_DIR}/nethermind + - client: besu + output_dir: ${STATE_DIR}/besu + - client: ethrex + output_dir: ${STATE_DIR}/ethrex + - client: erigon + output_dir: ${STATE_DIR}/erigon + + ## Stage 2: Building EEST compute fixtures + eest_payloads: + # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead + # of pulling a pre-built one. Alternatively set fill_image: . + fill_dockerfile: Dockerfile.eest-filler + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + # devnets/bal/7-bench carries the BAL benchmark tooling plus the fill-stateful + # fixes that let nethermind fill: the per-test rewind falls back to + # debug_resetHead (nethermind has no debug_setHead) and reads nonces at + # start_block, not "latest". Geth filling is unaffected by them. + eest_ref: devnets/bal/7-bench + config: + fork: amsterdam # fill amsterdam fixtures (snapshot stays osaka) + # Pin the seed key so its account (0x7e5f…bdf, privkey 1) can be pre-funded + # in the snapshot above — lets fill-stateful skip the withdrawal funding + # block so start_block == the snapshot block (stable per-test rewind). + rpc_seed_key: "0x0000000000000000000000000000000000000000000000000000000000000001" + gas_benchmark_values: [10] # Millions of gas to parametrise against + datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) + # Every target fills the same compute suite, so hoist it here. The -k + # filters stay per-target (each client excludes a slightly different set). + tests: + - tests/benchmark/compute # pytest paths inside the fill image + targets: + # Geth + - name: payload-generator-geth + filler_client: geth + filler_image: skylenet/geth:bal-devnet-7-amsterdam-override + source_dir: ${STATE_DIR}/geth + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 + output_dir: ${EEST_FIXTURES_DIR}/geth + # Exclude the currently-failing compute tests so the fill is green. Reasons: + # blockhash: 256-block chain exceeds geth pathdb rewind (per-test reset) + # blobhash: geth testing_buildBlockV1 panics on blob txs (no sidecar) + # the rest: amsterdam gas-repricing mismatches / sizing / setup issues + # NB: -k is substring-based, so this also drops the passing + # test_creates_collisions and test_storage_access_cold_benchmark. + filter: >- + not test_blockhash + and not test_blobhash + and not test_create + and not test_auth_transaction + and not test_unchunkified_bytecode + and not test_storage_access_cold + and not test_jumpdest_analysis + and not test_state_root_computation + and not test_contract_creation + and not test_contract_calling_many_addresses + and not test_selfdestruct_created + and not test_selfdestruct_initcode + and not test_block_full_data + and not test_prefetch_cold_storage + and not test_ext_account_query_cold + + # Besu + - name: payload-generator-besu + filler_client: besu + # besu bal-devnet-7 — now carries the TestingBuildBlockV1 coinbase fix + # (merged upstream) that fill-stateful needs to build correct EIP-7928 BALs. + filler_image: ethpandaops/besu:bal-devnet-7 + source_dir: ${STATE_DIR}/besu + genesis: ${STATE_DIR}/besu/besu-chainspec.json + # besu reads forks from the geth-format genesis; benchmarkoor patches + # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). + genesis_fork_override: { amsterdam: 1 } + output_dir: ${EEST_FIXTURES_DIR}/besu + # Identical exclusions to nethermind (keeps test_blockhash, which besu also + # fills). besu fills the same compute set as nethermind. + filter: >- + not test_blobhash + and not test_create + and not test_auth_transaction + and not test_unchunkified_bytecode + and not test_storage_access_cold + and not test_jumpdest_analysis + and not test_state_root_computation + and not test_contract_creation + and not test_contract_calling_many_addresses + and not test_selfdestruct_created + and not test_selfdestruct_initcode + and not test_block_full_data + and not test_prefetch_cold_storage + and not test_ext_account_query_cold + + # Nethermind + - name: payload-generator-nethermind + filler_client: nethermind + filler_image: nethermindeth/nethermind:master + source_dir: ${STATE_DIR}/nethermind + genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + output_dir: ${EEST_FIXTURES_DIR}/nethermind + # test_blockhash is kept here but excluded on the geth target: geth's + # pathdb can't rebuild on start_block after the test's 256-block chain + # (historical state unavailable), so geth corrupts; nethermind fills it. + filter: >- + not test_blobhash + and not test_create + and not test_auth_transaction + and not test_unchunkified_bytecode + and not test_storage_access_cold + and not test_jumpdest_analysis + and not test_state_root_computation + and not test_contract_creation + and not test_contract_calling_many_addresses + and not test_selfdestruct_created + and not test_selfdestruct_initcode + and not test_block_full_data + and not test_prefetch_cold_storage + and not test_ext_account_query_cold + +## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads +runner: + client_logs_to_stdout: true + cleanup_on_start: false + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR}/${EEST_FIXTURES_RUNNER_SOURCE} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + # Booting from a snapshot datadir leaves the Engine API forkchoice head + # un-established, so clients answer SYNCING to every newPayload. A bootstrap + # FCU to the snapshot head makes them canonical and ready to validate the + # replayed payloads (required for besu/reth/nethermind; geth is fine without). + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + genesis: + # nethermind reads a parity-format chainspec that schedules forks per-EIP, + # so amsterdam is activated via the per-instance genesis_eip_override below + # (not genesis_fork_override). The osaka chainspec is used as-is. + nethermind: ${STATE_DIR}/nethermind/parity-chainspec.json + #geth: ${STATE_DIR}/geth/geth-genesis.json + reth: ${STATE_DIR}/reth/chainspec.json + # besu/reth/ethrex have no --override.amsterdam flag — they read forks + # from the genesis. benchmarkoor patches it for them at boot via the + # per-instance genesis_fork_override below (sets amsterdamTime + inherits + # the amsterdam blobSchedule), so the osaka chainspec is used as-is. + besu: ${STATE_DIR}/besu/besu-chainspec.json + ethrex: ${STATE_DIR}/ethrex/ethrex-genesis.json + # erigon consumes a geth-format genesis via its init container. Filename + # follows the per-client state-actor convention (geth-genesis.json, + # ethrex-genesis.json); confirm/adjust once state-actor publishes the + # erigon image. + erigon: ${STATE_DIR}/erigon/chainspec.json + #erigon: ${STATE_DIR}/geth/geth-genesis.json + + datadirs: + geth: + source_dir: ${STATE_DIR}/geth + method: copy + reth: + source_dir: ${STATE_DIR}/reth + method: copy + nethermind: + source_dir: ${STATE_DIR}/nethermind + method: copy + besu: + source_dir: ${STATE_DIR}/besu + method: copy + ethrex: + source_dir: ${STATE_DIR}/ethrex + method: copy + erigon: + source_dir: ${STATE_DIR}/erigon + method: copy + + instances: + - id: geth + client: geth + image: skylenet/geth:bal-devnet-7-amsterdam-override + extra_args: + # Activate amsterdam at snapshot block 0's timestamp (0) + 1. The snapshot + # is osaka and geth boots from the datadir (its genesis is commented out + # above), so without this the amsterdam payloads (newPayloadV5) would be + # rejected. Needs the amsterdam-override image; a stock geth ignores it. + - --override.amsterdam=1 + - id: reth + client: reth + # bal-devnet-7 is the reth line with amsterdam/BAL support. reth reads forks + # from its chainspec, so patch amsterdam in at boot. + image: ethpandaops/reth:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 + extra_args: + # reth-bal-full knobs (mirror benchmarkoor-tests): "full" is the default + # BAL mode, so only the slow-block log threshold is set. + - --engine.slow-block-threshold=0 + # Booting from a snapshot, reth can answer the first newPayload before the + # forkchoice head is established; retry the non-SYNCING failure briefly. + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s + - id: nethermind + client: nethermind + image: nethermindeth/nethermind:master + # nethermind's parity chainspec schedules forks per-EIP, so activate the + # amsterdam-devnet-7 EIP set at snapshot block 0's ts (0) + 1. benchmarkoor + # patches params.eipTransitionTimestamp into the chainspec at boot. + genesis_eip_override: + timestamp: 1 + eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] + extra_args: + # benchmarkoor mounts the datadir at /data and the nethermind client + # passes --datadir=/data, but nethermind's BaseDbPath then defaults to + # /data/nethermind_db// — so it reads a fresh EMPTY state db and + # ignores the state-actor snapshot written directly under /data/{state, + # blocks,headers,blockInfos}. It then builds genesis from the empty-alloc + # chainspec, so HasStateForBlock(genesis) is false and every replayed + # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind + # read the snapshot directly (it then logs "Detected HalfPath key scheme"). + - --Init.BaseDbPath=/data + # nethermind-bal-full knobs (mirror benchmarkoor-tests). + - --Blocks.ParallelExecution=true + - --Blocks.ParallelExecutionBatchRead=true + - id: besu + client: besu + # bal-devnet-7 is the besu line with amsterdam/BAL (EIP-7928) support. + image: ethpandaops/besu:bal-devnet-7 + # besu reads forks from the genesis, so activate amsterdam at snapshot + # block 0's ts (0) + 1 by patching its chainspec at boot. + genesis_fork_override: + amsterdam: 1 + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" + extra_args: + # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: + # its synchronizer must run to register the post-merge head as in-sync, + # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false + # (the client defaults) keep it isolated — no real peers. + - --p2p-enabled=true + - --Xplugin-rocksdb-high-spec-enabled=true + # BAL "full" mode — mirrors besu-bal-full in ethpandaops/benchmarkoor-tests. + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=true + - --Xbal-prefetch-batch-size=8 + - --Xbal-processing-timeout=-1 + - --Xbal-state-root-timeout=-1 + - id: ethrex + client: ethrex + # bal-devnet-7 is the ethrex line with amsterdam/BAL support. ethrex reads + # forks from its genesis, so patch amsterdam in at boot. + image: ethpandaops/ethrex:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 + extra_args: + # --skip-genesis-validation: the state-actor snapshot commits a synthetic + # state root the emitted ethrex-genesis.json (empty alloc) can't reproduce. + - --skip-genesis-validation + # ethrex-bal-full knob (mirrors benchmarkoor-tests). + - --no-precompile-cache + - id: erigon + client: erigon + # MUST match the erigon that state-actor pins for the snapshot + # (Dockerfile.erigon builds erigon at the bal-devnet-7 commit and notes the + # daemon must use that exact binary). state-actor's `erigon init` encodes the + # MDBX chain config the bal-devnet-7 way (e.g. chainId as a JSON string); + # stable erigon (erigontech/erigon:latest, 3.4.x) panics reading it back with + # "cannot unmarshal \"1337\" into a *big.Int". ethpandaops/erigon:bal-devnet-7 + # reads it correctly and supports the amsterdam/BAL (EIP-7928) fork these + # benchmarks target. + image: ethpandaops/erigon:bal-devnet-7 + extra_args: + # erigon supports a geth-style fork override. Snapshot block 0 ts is 0, so + # activate amsterdam at ts 1 (the reference devnet uses its real ts). + # NB: genesis_fork_override does NOT work for erigon — like geth it reads + # the fork schedule from the datadir, not the genesis file, so the flag is + # required (a patched genesis is ignored and amsterdam stays inactive). + - --override.amsterdam=1 + # erigon-bal-full knobs (mirror benchmarkoor-tests): keep execution + # deterministic/online by disabling background merge/prune/maintenance and + # synchronous FCU commit so replayed payloads are validated immediately. + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + # erigon is slow to register the snapshot head as in-sync; give the bootstrap + # FCU a long runway (mirrors the reference erigon-bal-full). + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml new file mode 100644 index 000000000..a6b48b190 --- /dev/null +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml @@ -0,0 +1,326 @@ +# Building data directories and EEST STATEFUL (EIP-7928 / BAL) payloads — AMSTERDAM fill. +# +# state-actor generates OSAKA snapshot datadirs (stage 1); the eest_payloads stage +# fills AMSTERDAM tests/benchmark/stateful EIP-7928 (BAL) fixtures by booting the geth +# filler on the osaka snapshot and activating amsterdam at block 0's timestamp + 1 +# via geth's --override.amsterdam=1 flag (filler_extra_args). +# +# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml --force +# +# Running the benchmark (one instance per client): +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml --limit-instance-id=geth +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml --limit-instance-id=besu +# +# Each client activates amsterdam on the osaka snapshot one of three ways: +# - geth, erigon: CLI fork override (--override.amsterdam=) +# - besu, reth, ethrex: genesis_fork_override (benchmarkoor sets Time in +# the geth-format genesis at boot) +# - nethermind: genesis_eip_override (benchmarkoor sets the amsterdam +# EIPs' eipTransitionTimestamp in the parity chainspec) +# Snapshot block 0's timestamp is 0, so amsterdam is scheduled at ts 1. +# +global: + log_level: info + env: + # Per-config dirs so configs that build different state-actor snapshots or + # EEST fixtures don't collide under /tmp. Override by exporting these. Each + # client writes under its own / subdir. + STATE_DIR: /tmp/benchmarkoor/state-actor/simple-amsterdam-stateful + EEST_FIXTURES_DIR: /tmp/benchmarkoor/eest-fixtures/simple-amsterdam-stateful + EEST_FIXTURES_RUNNER_SOURCE: geth # which filler's fixtures the runner replays + +builder: + ## Stage 1: Building datadirs using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + reth: ghcr.io/ethereum/state-actor-reth:main + besu: ghcr.io/ethereum/state-actor-besu:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + ethrex: ghcr.io/ethereum/state-actor-ethrex:main + erigon: ghcr.io/ethereum/state-actor-erigon:main + pull_policy: always + config: + seed: 1234 + fork: osaka + chain: 1337 + gas_limit: 300000000 # 300M + target_size: 256MB + #target_size: 20GB + spec: + entities: + # Predeploy the Arachnid deterministic-deployment proxy at the canonical + # 0x4e59…956c. EEST's fill bootstrap deploys it via a keyless tx with a + # fixed 100k gas limit, which amsterdam's state-expansion gas (EIP-7928/ + # 8037, ~324k for this proxy) exceeds — so it must be predeployed. EEST + # detects the existing bytecode and skips the keyless deploy. + - kind: contract + template: create2_factory + name: deterministic-deployment-proxy + # Explicit canonical address: with a name set, the address would + # otherwise be name-derived instead of the Arachnid default. + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + # Pre-fund fill-stateful's seed account (rpc_seed_key=0x..01 → this + # address). With the seed already funded, fill-stateful skips the + # withdrawal funding block, so start_block == the snapshot block, whose + # persistent state debug_setHead can always rewind to between tests. + # Balance must be >= SEED_FUNDING_WEI (1e27); 1e28 gives headroom. + - kind: eoa + name: fill-stateful-seed + address: 0x7e5f4552091a69125d5dfcb7b8c2659029395bdf + balance: "10000000000000000000000000000" + nonce: 0 + targets: + - client: geth + output_dir: ${STATE_DIR}/geth + - client: reth + output_dir: ${STATE_DIR}/reth + - client: nethermind + output_dir: ${STATE_DIR}/nethermind + - client: besu + output_dir: ${STATE_DIR}/besu + - client: ethrex + output_dir: ${STATE_DIR}/ethrex + - client: erigon + output_dir: ${STATE_DIR}/erigon + + ## Stage 2: Building EEST stateful (EIP-7928 / BAL) fixtures + eest_payloads: + # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead + # of pulling a pre-built one. Alternatively set fill_image: . + fill_dockerfile: Dockerfile.eest-filler + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + # devnets/bal/7-bench carries the BAL benchmark tooling plus the fill-stateful + # fixes that let nethermind fill: the per-test rewind falls back to + # debug_resetHead (nethermind has no debug_setHead) and reads nonces at + # start_block, not "latest". Geth filling is unaffected by them. + eest_ref: devnets/bal/7-bench + config: + fork: amsterdam # fill amsterdam fixtures (snapshot stays osaka) + # Pin the seed key so its account (0x7e5f…bdf, privkey 1) can be pre-funded + # in the snapshot above — lets fill-stateful skip the withdrawal funding + # block so start_block == the snapshot block (stable per-test rewind). + rpc_seed_key: "0x0000000000000000000000000000000000000000000000000000000000000001" + gas_benchmark_values: [10] # Millions of gas to parametrise against + datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) + # Both targets fill the same stateful (BAL) suite, so hoist it here. + tests: + - tests/benchmark/stateful/eip7928_block_level_access_lists + targets: + # Geth + - name: payload-generator-geth-stateful + filler_client: geth + filler_image: skylenet/geth:bal-devnet-7-amsterdam-override + source_dir: ${STATE_DIR}/geth + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 + output_dir: ${EEST_FIXTURES_DIR}/geth + + # Nethermind + - name: payload-generator-nethermind-stateful + filler_client: nethermind + filler_image: nethermindeth/nethermind:master + source_dir: ${STATE_DIR}/nethermind + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + output_dir: ${EEST_FIXTURES_DIR}/nethermind + + # Besu — cross-client filler. ethpandaops/besu:bal-devnet-7 carries the + # merged TestingBuildBlockV1 coinbase fix needed to build correct EIP-7928 + # BALs (see config.state-actor-eest.simple.amsterdam.compute.yaml). + - name: payload-generator-besu-stateful + filler_client: besu + filler_image: ethpandaops/besu:bal-devnet-7 + source_dir: ${STATE_DIR}/besu + # besu reads forks from the geth-format genesis; benchmarkoor patches + # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). + genesis: ${STATE_DIR}/besu/besu-chainspec.json + genesis_fork_override: { amsterdam: 1 } + output_dir: ${EEST_FIXTURES_DIR}/besu + +## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads +runner: + client_logs_to_stdout: true + cleanup_on_start: false + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR}/${EEST_FIXTURES_RUNNER_SOURCE} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + rollback_strategy: container-recreate # recreate the container and rollback snapshot datadir + # Booting from a snapshot datadir leaves the Engine API forkchoice head + # un-established, so clients answer SYNCING to every newPayload. A bootstrap + # FCU to the snapshot head makes them canonical and ready to validate the + # replayed payloads (required for besu/reth/nethermind; geth is fine without). + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + genesis: + # nethermind reads a parity-format chainspec that schedules forks per-EIP, + # so amsterdam is activated via the per-instance genesis_eip_override below + # (not genesis_fork_override). The osaka chainspec is used as-is. + nethermind: ${STATE_DIR}/nethermind/parity-chainspec.json + #geth: ${STATE_DIR}/geth/geth-genesis.json + reth: ${STATE_DIR}/reth/chainspec.json + # besu/reth/ethrex have no --override.amsterdam flag — they read forks + # from the genesis. benchmarkoor patches it for them at boot via the + # per-instance genesis_fork_override below (sets amsterdamTime + inherits + # the amsterdam blobSchedule), so the osaka chainspec is used as-is. + besu: ${STATE_DIR}/besu/besu-chainspec.json + ethrex: ${STATE_DIR}/ethrex/ethrex-genesis.json + # erigon consumes a geth-format genesis via its init container. Filename + # follows the per-client state-actor convention (geth-genesis.json, + # ethrex-genesis.json); confirm/adjust once state-actor publishes the + # erigon image. + erigon: ${STATE_DIR}/erigon/chainspec.json + #erigon: ${STATE_DIR}/geth/geth-genesis.json + + datadirs: + geth: + source_dir: ${STATE_DIR}/geth + method: copy + reth: + source_dir: ${STATE_DIR}/reth + method: copy + nethermind: + source_dir: ${STATE_DIR}/nethermind + method: copy + besu: + source_dir: ${STATE_DIR}/besu + method: copy + ethrex: + source_dir: ${STATE_DIR}/ethrex + method: copy + erigon: + source_dir: ${STATE_DIR}/erigon + method: copy + + instances: + - id: geth + client: geth + image: skylenet/geth:bal-devnet-7-amsterdam-override + extra_args: + # Activate amsterdam at snapshot block 0's timestamp (0) + 1. The snapshot + # is osaka and geth boots from the datadir (its genesis is commented out + # above), so without this the amsterdam payloads (newPayloadV5) would be + # rejected. Needs the amsterdam-override image; a stock geth ignores it. + - --override.amsterdam=1 + - id: reth + client: reth + # bal-devnet-7 is the reth line with amsterdam/BAL support. reth reads forks + # from its chainspec, so patch amsterdam in at boot. + image: ethpandaops/reth:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 + extra_args: + # reth-bal-full knobs (mirror benchmarkoor-tests): "full" is the default + # BAL mode, so only the slow-block log threshold is set. + - --engine.slow-block-threshold=0 + # Booting from a snapshot, reth can answer the first newPayload before the + # forkchoice head is established; retry the non-SYNCING failure briefly. + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s + - id: nethermind + client: nethermind + image: nethermindeth/nethermind:master + # nethermind's parity chainspec schedules forks per-EIP, so activate the + # amsterdam-devnet-7 EIP set at snapshot block 0's ts (0) + 1. benchmarkoor + # patches params.eipTransitionTimestamp into the chainspec at boot. + genesis_eip_override: + timestamp: 1 + eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] + extra_args: + # benchmarkoor mounts the datadir at /data and the nethermind client + # passes --datadir=/data, but nethermind's BaseDbPath then defaults to + # /data/nethermind_db// — so it reads a fresh EMPTY state db and + # ignores the state-actor snapshot written directly under /data/{state, + # blocks,headers,blockInfos}. It then builds genesis from the empty-alloc + # chainspec, so HasStateForBlock(genesis) is false and every replayed + # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind + # read the snapshot directly (it then logs "Detected HalfPath key scheme"). + - --Init.BaseDbPath=/data + # nethermind-bal-full knobs (mirror benchmarkoor-tests). + - --Blocks.ParallelExecution=true + - --Blocks.ParallelExecutionBatchRead=true + - id: besu + client: besu + # bal-devnet-7 is the besu line with amsterdam/BAL (EIP-7928) support. + image: ethpandaops/besu:bal-devnet-7 + # besu reads forks from the genesis, so activate amsterdam at snapshot + # block 0's ts (0) + 1 by patching its chainspec at boot. + genesis_fork_override: + amsterdam: 1 + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" + extra_args: + # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: + # its synchronizer must run to register the post-merge head as in-sync, + # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false + # (the client defaults) keep it isolated — no real peers. + - --p2p-enabled=true + - --Xplugin-rocksdb-high-spec-enabled=true + # BAL "full" mode — mirrors besu-bal-full in ethpandaops/benchmarkoor-tests. + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=true + - --Xbal-prefetch-batch-size=8 + - --Xbal-processing-timeout=-1 + - --Xbal-state-root-timeout=-1 + - id: ethrex + client: ethrex + # bal-devnet-7 is the ethrex line with amsterdam/BAL support. ethrex reads + # forks from its genesis, so patch amsterdam in at boot. + image: ethpandaops/ethrex:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 + extra_args: + # --skip-genesis-validation: the state-actor snapshot commits a synthetic + # state root the emitted ethrex-genesis.json (empty alloc) can't reproduce. + - --skip-genesis-validation + # ethrex-bal-full knob (mirrors benchmarkoor-tests). + - --no-precompile-cache + - id: erigon + client: erigon + # MUST match the erigon that state-actor pins for the snapshot + # (Dockerfile.erigon builds erigon at the bal-devnet-7 commit and notes the + # daemon must use that exact binary). state-actor's `erigon init` encodes the + # MDBX chain config the bal-devnet-7 way (e.g. chainId as a JSON string); + # stable erigon (erigontech/erigon:latest, 3.4.x) panics reading it back with + # "cannot unmarshal \"1337\" into a *big.Int". ethpandaops/erigon:bal-devnet-7 + # reads it correctly and supports the amsterdam/BAL (EIP-7928) fork these + # benchmarks target. + image: ethpandaops/erigon:bal-devnet-7 + extra_args: + # erigon supports a geth-style fork override. Snapshot block 0 ts is 0, so + # activate amsterdam at ts 1 (the reference devnet uses its real ts). + # NB: genesis_fork_override does NOT work for erigon — like geth it reads + # the fork schedule from the datadir, not the genesis file, so the flag is + # required (a patched genesis is ignored and amsterdam stays inactive). + - --override.amsterdam=1 + # erigon-bal-full knobs (mirror benchmarkoor-tests): keep execution + # deterministic/online by disabling background merge/prune/maintenance and + # synchronous FCU commit so replayed payloads are validated immediately. + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + # erigon is slow to register the snapshot head as in-sync; give the bootstrap + # FCU a long runway (mirrors the reference erigon-bal-full). + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s diff --git a/examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml b/examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml new file mode 100644 index 000000000..64bc4fbfd --- /dev/null +++ b/examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml @@ -0,0 +1,206 @@ +# Building data directories and EEST COMPUTE payloads — OSAKA fill. +# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml --force +# +# Running benchmark using geth: +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml --limit-instance-id=geth +# +global: + log_level: info + env: + # Per-config dirs so configs that build different state-actor snapshots or + # EEST fixtures don't collide under /tmp. Override by exporting these. Each + # client writes under its own / subdir. + STATE_DIR: /tmp/benchmarkoor/state-actor/simple-osaka-compute + EEST_FIXTURES_DIR: /tmp/benchmarkoor/eest-fixtures/simple-osaka-compute + EEST_FIXTURES_RUNNER_SOURCE: geth # which filler's fixtures the runner replays + +builder: + ## Stage 1: Building datadirs using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + reth: ghcr.io/ethereum/state-actor-reth:main + besu: ghcr.io/ethereum/state-actor-besu:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + ethrex: ghcr.io/ethereum/state-actor-ethrex:main + erigon: ghcr.io/ethereum/state-actor-erigon:main + pull_policy: always + config: + seed: 1234 + fork: osaka + chain: 1337 + gas_limit: 300000000 # 300M + target_size: 256MB + #target_size: 20GB + spec: + entities: + - kind: eoa + name: bloated-eoa-2g + balance: "1000000000000000000" + nonce: 0 + code: "0xef01003333333333333333333333333333333333333333" + approximate_size_bytes: 2_000_000_000 + targets: + - client: geth + output_dir: ${STATE_DIR}/geth + - client: reth + output_dir: ${STATE_DIR}/reth + - client: nethermind + output_dir: ${STATE_DIR}/nethermind + - client: besu + output_dir: ${STATE_DIR}/besu + - client: ethrex + output_dir: ${STATE_DIR}/ethrex + - client: erigon + output_dir: ${STATE_DIR}/erigon + + ## Stage 2: Building EEST stateful fixtures + eest_payloads: + # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead + # of pulling a pre-built one. Alternatively set fill_image: . + fill_dockerfile: Dockerfile.eest-filler + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + eest_ref: forks/amsterdam + config: + fork: osaka + gas_benchmark_values: [10] # Millions of gas to parametrise against + datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) + # Every target fills the same compute subset, so hoist tests + filter here. + tests: + - tests/benchmark/compute # pytest paths inside the fill image + filter: bn128 # Quick subset for fast end-to-end iteration: only the bn128 compute tests. + # fill-stateful forces testing_buildBlockV1 + debug_setHead, so a filler must + # implement both. geth (ethpandaops/geth) does. besu/nethermind are plumbed + # in benchmarkoor (see fillerCommand in pkg/builder) but blocked upstream, so + # their targets are commented out below — uncomment to experiment once the + # upstream gaps close. + targets: + - name: payload-generator-geth + filler_client: geth + filler_image: ethpandaops/geth:master + source_dir: ${STATE_DIR}/geth + output_dir: ${EEST_FIXTURES_DIR}/geth + # besu: testing_buildBlockV1 (TESTING namespace, besu-eth/besu#9838), + # debug_setHead and the session-fee pin all work, but besu's self-built + # block fails its own engine_newPayloadV4 with a World-State-Root mismatch + # (upstream besu bug in testing_buildBlockV1). Re-enable when fixed. + # - name: payload-generator-besu + # filler_client: besu + # filler_image: ethpandaops/besu:main + # source_dir: ${STATE_DIR}/besu + # genesis: ${STATE_DIR}/besu/besu-chainspec.json + # output_dir: ${EEST_FIXTURES_DIR}/besu + # nethermind cross-client filler — uncomment to also fill via nethermind. + # - name: payload-generator-nethermind + # filler_client: nethermind + # filler_image: nethermindeth/nethermind:master + # source_dir: ${STATE_DIR}/nethermind + # genesis: ${STATE_DIR}/nethermind/parity-chainspec.json + # output_dir: ${EEST_FIXTURES_DIR}/nethermind + +## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads +runner: + client_logs_to_stdout: true + cleanup_on_start: false + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR}/${EEST_FIXTURES_RUNNER_SOURCE} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + # Booting from a snapshot datadir leaves the Engine API forkchoice head + # un-established, so clients answer SYNCING to every newPayload. A bootstrap + # FCU to the snapshot head makes them canonical and ready to validate the + # replayed payloads (required for besu/reth/nethermind; geth is fine without). + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + genesis: + nethermind: ${STATE_DIR}/nethermind/parity-chainspec.json + #geth: ${STATE_DIR}/geth/geth-genesis.json + reth: ${STATE_DIR}/reth/chainspec.json + besu: ${STATE_DIR}/besu/besu-chainspec.json + ethrex: ${STATE_DIR}/ethrex/ethrex-genesis.json + # erigon consumes a geth-format genesis via its init container. Filename + # follows the per-client state-actor convention (geth-genesis.json, + # ethrex-genesis.json); confirm/adjust once state-actor publishes the + # erigon image. + erigon: ${STATE_DIR}/erigon/chainspec.json + #erigon: ${STATE_DIR}/geth/geth-genesis.json + + datadirs: + geth: + source_dir: ${STATE_DIR}/geth + method: copy + reth: + source_dir: ${STATE_DIR}/reth + method: copy + nethermind: + source_dir: ${STATE_DIR}/nethermind + method: copy + besu: + source_dir: ${STATE_DIR}/besu + method: copy + ethrex: + source_dir: ${STATE_DIR}/ethrex + method: copy + erigon: + source_dir: ${STATE_DIR}/erigon + method: copy + + instances: + - id: geth + client: geth + image: ethpandaops/geth:master + - id: reth + client: reth + #image: ghcr.io/paradigmxyz/reth:latest + image: ghcr.io/paradigmxyz/reth:nightly@sha256:e528857e5e9ebc2c6cb99f28436e70ded38ca905629f00afc98d186e27d206e0 + - id: nethermind + client: nethermind + extra_args: + # benchmarkoor mounts the datadir at /data and the nethermind client + # passes --datadir=/data, but nethermind's BaseDbPath then defaults to + # /data/nethermind_db// — so it reads a fresh EMPTY state db and + # ignores the state-actor snapshot written directly under /data/{state, + # blocks,headers,blockInfos}. It then builds genesis from the empty-alloc + # chainspec, so HasStateForBlock(genesis) is false and every replayed + # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind + # read the snapshot directly (it then logs "Detected HalfPath key scheme"). + - --Init.BaseDbPath=/data + - id: besu + client: besu + # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: + # its synchronizer must run to register the post-merge head as in-sync, + # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false + # (the client defaults) keep it isolated — no real peers. + image: hyperledger/besu:26.6.1 + extra_args: + - --p2p-enabled=true + - id: ethrex + client: ethrex + # ethrex >= v16.0.0 is required: it's the first release with + # --skip-genesis-validation (lambdaclass/ethrex#6783), needed because the + # state-actor snapshot commits a synthetic state root that the emitted + # ethrex-genesis.json (empty alloc) can't reproduce. + image: ghcr.io/lambdaclass/ethrex:latest + extra_args: + - --skip-genesis-validation + - id: erigon + client: erigon + # MUST match the erigon that state-actor pins for the snapshot + # (Dockerfile.erigon builds erigon at the bal-devnet-7 commit and notes the + # daemon must use that exact binary). state-actor's `erigon init` encodes the + # MDBX chain config the bal-devnet-7 way (e.g. chainId as a JSON string); + # stable erigon (erigontech/erigon:latest, 3.4.x) panics reading it back with + # "cannot unmarshal \"1337\" into a *big.Int". ethpandaops/erigon:bal-devnet-7 + # reads it correctly and supports the amsterdam/BAL (EIP-7928) fork these + # benchmarks target. + image: ethpandaops/erigon:bal-devnet-7 diff --git a/pkg/api/local_test.go b/pkg/api/local_test.go index d2351223b..2c9f5b0c5 100644 --- a/pkg/api/local_test.go +++ b/pkg/api/local_test.go @@ -75,6 +75,25 @@ func TestLocalFileServer_ServeFile(t *testing.T) { assert.Contains(t, rec.Body.String(), `{"ok":true}`) }) + t.Run("serves dotfile directory (.eest-meta)", func(t *testing.T) { + metaDir := filepath.Join(root, "suites", "abc", ".eest-meta") + require.NoError(t, os.MkdirAll(metaDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(metaDir, "fixtures.ini"), + []byte("[environment]\npython = 3.12.13\n"), 0o644)) + + req := httptest.NewRequest( + http.MethodGet, + "/mydata/suites/abc/.eest-meta/fixtures.ini", nil, + ) + rec := httptest.NewRecorder() + + err := srv.ServeFile(rec, req, "mydata/suites/abc/.eest-meta/fixtures.ini") + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), "python = 3.12.13") + }) + t.Run("returns error for missing file", func(t *testing.T) { req := httptest.NewRequest( http.MethodGet, diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go new file mode 100644 index 000000000..ccccfbe61 --- /dev/null +++ b/pkg/builder/eest_payloads.go @@ -0,0 +1,1084 @@ +package builder + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/ethpandaops/benchmarkoor/pkg/client" + "github.com/ethpandaops/benchmarkoor/pkg/config" + "github.com/ethpandaops/benchmarkoor/pkg/datadir" + "github.com/ethpandaops/benchmarkoor/pkg/docker" + "github.com/ethpandaops/benchmarkoor/pkg/genesis" + "github.com/ethpandaops/benchmarkoor/pkg/gitrepo" + "github.com/sirupsen/logrus" +) + +// EESTPayloadsBuilderName is the value of the benchmarkoor.builder label +// set on every eest_payloads container, and the value returned by Name(). +const EESTPayloadsBuilderName = "eest-payloads" + +const ( + // EESTBuildNetwork is the docker/podman network shared by the filler + // client and the fill-stateful container so they can reach each other. + EESTBuildNetwork = "benchmarkoor-build" + + // fillerReadyTimeout bounds how long we wait for the filler client's RPC + // to answer — opening a large archive snapshot can take several minutes. + fillerReadyTimeout = 15 * time.Minute + + // fillerStopTimeoutSec is the graceful-stop window for the filler client. + fillerStopTimeoutSec = 30 + + // In-container paths used by the fill-stateful container. + fillJWTPath = "/jwt/jwtsecret" + fillOutputPath = "/out" + fillStubsPath = "/stubs.json" + // fillRepoPath is the fill image's WORKDIR (the execution-specs checkout); + // a config-selected EEST repo clone is mounted here when configured. + fillRepoPath = "/eest" + + // minerGasLimit is the huge gas limit the filler geth is started with so + // benchmark blocks of any size can be built (mirrors the fill-stateful docs). + minerGasLimit = "1000000000000" + + // besuDefaultPriorityFeeWei pins fill-stateful's session tip for the besu + // filler. besu's eth_maxPriorityFeePerGas returns 0 on a freshly-booted + // snapshot (no fee history), which fill-stateful rejects with "requires the + // backend to carry non-zero session fees". geth suggests a non-zero tip on + // its own, so this is only needed for besu. 1 gwei. + besuDefaultPriorityFeeWei = "1000000000" +) + +// EESTPayloadsBuilder generates stateful EEST benchmark fixtures. Per target +// it boots a filler EL client on a writable copy of a pre-populated snapshot +// datadir, runs fill-stateful against the live client, then tears it down. +type EESTPayloadsBuilder struct { + log logrus.FieldLogger + cfg *config.EESTPayloadsConfig + runtime string + mgr docker.ContainerManager + registry client.Registry + repoCache string + + // fillImage is resolved once per builder: built from fill_dockerfile or the + // configured fill_image. Guarded by fillImageOnce so a multi-target build + // only builds the image a single time. + fillImageOnce sync.Once + fillImage string + fillImageErr error +} + +// NewEESTPayloadsBuilder constructs a builder bound to a specific container +// manager. cacheDir is the shared on-disk cache (global.directories.cachedir); +// EEST repo clones are cached under /eest-repos so recurring builds +// of the same ref don't re-clone. The caller is expected to have Start()'d the +// manager and to Stop() it after the last Build() call. +func NewEESTPayloadsBuilder( + log logrus.FieldLogger, + cfg *config.EESTPayloadsConfig, + runtime string, + mgr docker.ContainerManager, + cacheDir string, +) *EESTPayloadsBuilder { + return &EESTPayloadsBuilder{ + log: log.WithField("component", "builder.eest_payloads"), + cfg: cfg, + runtime: runtime, + mgr: mgr, + registry: client.NewRegistry(), + repoCache: filepath.Join(cacheDir, "eest-repos"), + } +} + +// Name implements Builder. +func (b *EESTPayloadsBuilder) Name() string { + return EESTPayloadsBuilderName +} + +// Targets implements Builder. +func (b *EESTPayloadsBuilder) Targets() []TargetInfo { + out := make([]TargetInfo, 0, len(b.cfg.Targets)) + + for i := range b.cfg.Targets { + t := &b.cfg.Targets[i] + out = append(out, TargetInfo{ + Name: t.EffectiveName(), + Client: t.FillerClient, + OutputDir: t.OutputDir, + }) + } + + return out +} + +// Build implements Builder. +func (b *EESTPayloadsBuilder) Build(ctx context.Context, name string, opts BuildOptions) (bool, error) { + idx := b.findTargetIndex(name) + if idx < 0 { + return false, fmt.Errorf("no target named %q", name) + } + + resolved := b.cfg.ResolveTarget(idx) + target := &resolved + + // Keep only the target on the per-line logger; source_dir/output_dir/fork + // are reference details logged once below rather than suffixed onto every + // orchestration and streamed-client log line. + log := b.log.WithField("target", target.EffectiveName()) + + force := opts.Force || target.Force + + if !force { + populated, err := isPopulated(target.OutputDir) + if err != nil { + return false, err + } + + if populated { + log.Info("Skipping build: output_dir already populated " + + "(pass --force or set force: true on the target to rebuild)") + + return true, nil + } + } + + if err := b.checkInputs(target); err != nil { + return false, err + } + + if err := prepareOutputDir(target.OutputDir, force); err != nil { + return false, err + } + + return false, b.run(ctx, log, target) +} + +// findTargetIndex returns the index of the first target whose EffectiveName +// matches name, or -1 when nothing matches. +func (b *EESTPayloadsBuilder) findTargetIndex(name string) int { + for i := range b.cfg.Targets { + if b.cfg.Targets[i].EffectiveName() == name { + return i + } + } + + return -1 +} + +// checkInputs verifies the build-time inputs exist. Existence is checked +// here (not at config-validation time) because a state-actor target earlier +// in the same config may still need to produce source_dir. +func (b *EESTPayloadsBuilder) checkInputs(t *config.EESTPayloadTarget) error { + if info, err := os.Stat(t.SourceDir); err != nil { + return fmt.Errorf("source_dir %q: %w", t.SourceDir, err) + } else if !info.IsDir() { + return fmt.Errorf("source_dir %q is not a directory", t.SourceDir) + } + + if t.Genesis != "" { + if _, err := os.Stat(t.Genesis); err != nil { + return fmt.Errorf("genesis: %w", err) + } + } + + if t.AddressStubsFile != "" { + if _, err := os.Stat(t.AddressStubsFile); err != nil { + return fmt.Errorf("address_stubs_file: %w", err) + } + } + + return nil +} + +// run performs the orchestration: temp JWT, network, datadir copy, filler +// boot, fill-stateful, and teardown. +func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t *config.EESTPayloadTarget) error { + // Record the build details once; the per-line logger carries only target. + log.WithFields(logrus.Fields{ + "filler_client": t.FillerClient, + "source_dir": t.SourceDir, + "output_dir": t.OutputDir, + "fork": t.Fork, + }).Info("Generating EEST payloads") + + // Clone the EEST repo at the configured ref into the cache and mount it into + // the fill container at /eest. The fill image carries only the uv/python + // toolchain (no repo), so this is always done; the EEST version is + // config-driven (eest_repo / eest_ref). + repo, ref := b.cfg.ResolveEESTRepo(), b.cfg.ResolveEESTRef() + + eestRepoPath, err := gitrepo.CloneOrUpdate(ctx, log, repo, ref, b.repoCache) + if err != nil { + return fmt.Errorf("cloning EEST repo %s@%s: %w", repo, ref, err) + } + + sha, _ := gitrepo.HeadSHA(ctx, eestRepoPath) + log.WithFields(logrus.Fields{ + "repo": repo, "ref": ref, "commit": sha, "path": eestRepoPath, + }).Info("Using cloned EEST repo for fill") + + spec, err := b.registry.Get(client.ClientType(t.FillerClient)) + if err != nil { + return fmt.Errorf("resolving filler client %q: %w", t.FillerClient, err) + } + + jwtPath, cleanupJWT, err := writeTempJWT(b.cfg.JWT) + if err != nil { + return err + } + + defer cleanupJWT() + + if err := b.mgr.EnsureNetwork(ctx, EESTBuildNetwork); err != nil { + return fmt.Errorf("ensuring network %q: %w", EESTBuildNetwork, err) + } + + provider, err := datadir.NewProvider(b.log, t.DataDirMethod) + if err != nil { + return fmt.Errorf("creating datadir provider: %w", err) + } + + log.Info("Preparing writable copy of snapshot datadir") + + prepared, err := provider.Prepare(ctx, &datadir.ProviderConfig{ + SourceDir: t.SourceDir, + InstanceID: "eest-fill-" + t.EffectiveName(), + TmpDir: mountTempDir(), + }) + if err != nil { + return fmt.Errorf("preparing datadir copy: %w", err) + } + + defer func() { + if cleanupErr := prepared.Cleanup(); cleanupErr != nil { + log.WithError(cleanupErr).Warn("Failed to clean up datadir copy") + } + }() + + // genesis_fork_override / genesis_eip_override patch the boot genesis before + // the filler mounts it, to activate a fork the file doesn't schedule (e.g. + // amsterdam on an osaka snapshot) — identical to the runner. Used by fillers + // that read forks from the genesis (besu/reth/ethrex/nethermind). geth/erigon + // boot from the datadir and instead activate forks via --override. in + // filler_extra_args. + if len(t.GenesisForkOverride) > 0 || + (t.GenesisEIPOverride != nil && len(t.GenesisEIPOverride.EIPs) > 0) { + patched, cleanup, perr := patchFillerGenesis(log, t) + if perr != nil { + return perr + } + + defer cleanup() + + t.Genesis = patched + } + + // address_stubs defines the stub mapping inline; materialize it to a temp + // JSON file so the existing mount + --address-stubs path (keyed on + // AddressStubsFile) works unchanged — identical to the genesis patch above. + if len(t.AddressStubs) > 0 { + stubsPath, cleanup, serr := materializeAddressStubs(log, t) + if serr != nil { + return serr + } + + defer cleanup() + + t.AddressStubsFile = stubsPath + } + + // Stream the filler's logs for the lifetime of this build. + streamCtx, streamCancel := context.WithCancel(ctx) + defer streamCancel() + + fillerID, fillerIP, configCleanup, err := b.startFiller(ctx, streamCtx, log, t, spec, prepared.MountPath, jwtPath) + if err != nil { + return err + } + + // Order (defers run LIFO): stop the container first, then remove the temp + // config file it bind-mounted. The host file must outlive the container — + // removing it earlier makes /tmp/config.toml vanish inside the container + // (Docker Desktop syncs the bind mount by path), which geth reports as + // "open /tmp/config.toml: no such file or directory". + defer configCleanup() + defer b.stopFiller(log, fillerID) + + log.Info("Waiting for filler client RPC to become ready") + + version, err := b.waitForFillerReady(ctx, fillerID, fillerIP, spec.RPCPort()) + if err != nil { + return err + } + + snapshotHash, err := getLatestBlockHash(ctx, fillerIP, spec.RPCPort()) + if err != nil { + return fmt.Errorf("fetching snapshot block hash: %w", err) + } + + log.WithFields(logrus.Fields{ + "client_version": version, + "snapshot_block": snapshotHash, + }).Info("Filler client ready; running fill-stateful") + + return b.runFill(ctx, log, t, fillerIP, spec, jwtPath, snapshotHash, eestRepoPath) +} + +// waitForFillerReady blocks until the filler's RPC answers or the filler +// container exits, whichever happens first, returning the client version. +// +// A filler that dies before its RPC comes up (panic on boot, bad flags, OOM, +// corrupt datadir) would otherwise leave waitForRPC polling a dead endpoint for +// the full fillerReadyTimeout. Watching the container exit lets us fail fast +// with the exit code instead of hanging for 15 minutes. +func (b *EESTPayloadsBuilder) waitForFillerReady( + ctx context.Context, + containerID, ip string, + port int, +) (string, error) { + ctx, cancel := context.WithTimeout(ctx, fillerReadyTimeout) + defer cancel() + + exitCh, exitErrCh := b.mgr.WaitForContainerExit(ctx, containerID) + + type rpcResult struct { + version string + err error + } + + resultCh := make(chan rpcResult, 1) + + go func() { + version, err := waitForRPC(ctx, ip, port) + resultCh <- rpcResult{version: version, err: err} + }() + + select { + case res := <-resultCh: + if res.err != nil { + return "", fmt.Errorf("filler client never became ready: %w", res.err) + } + + return res.version, nil + case info := <-exitCh: + // cancel() unblocks the waitForRPC goroutine; the deferred cancel also + // covers it, but stopping the poll immediately is tidier. + cancel() + + return "", fmt.Errorf( + "filler client exited before RPC became ready "+ + "(exit code %d, oom_killed=%t)", + info.ExitCode, info.OOMKilled, + ) + case err := <-exitErrCh: + // A wait error from our own timeout/cancel isn't actionable — defer to + // the RPC goroutine, whose timeout message describes it better. + if err == nil || + errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) { + res := <-resultCh + + if res.err != nil { + return "", fmt.Errorf("filler client never became ready: %w", res.err) + } + + return res.version, nil + } + + cancel() + + return "", fmt.Errorf("watching filler container: %w", err) + } +} + +// startFiller boots the filler EL client and returns its container ID and IP. +// +// On success it returns a cleanup func that removes the temp config file; the +// caller must defer it for the lifetime of the container (the bind-mounted host +// file has to outlive geth's startup read of /tmp/config.toml). On error the +// cleanup is run here and a no-op is returned. +func (b *EESTPayloadsBuilder) startFiller( + ctx, streamCtx context.Context, + log logrus.FieldLogger, + t *config.EESTPayloadTarget, + spec client.Spec, + dataMount, jwtPath string, +) (id string, ip string, cleanup func(), err error) { + configCleanup := func() {} + + // Until the container is successfully handed off to the caller, any error + // return must remove the temp config file itself — the caller only defers + // the cleanup once startFiller succeeds. + defer func() { + if err != nil { + configCleanup() + } + }() + + if err = b.mgr.PullImage(ctx, t.FillerImage, b.cfg.PullPolicy); err != nil { + return "", "", nil, fmt.Errorf("pulling filler image %q: %w", t.FillerImage, err) + } + + mounts := []docker.Mount{ + {Source: dataMount, Target: spec.DataDir(), Type: "bind"}, + {Source: jwtPath, Target: spec.JWTPath(), Type: "bind", ReadOnly: true}, + } + + if files := spec.DefaultConfigFiles(); len(files) > 0 { + configMounts, cfgCleanup, cfgErr := writeTempConfigFiles(files) + if cfgErr != nil { + err = cfgErr + + return "", "", nil, cfgErr + } + + mounts = append(mounts, configMounts...) + configCleanup = cfgCleanup + } + + if t.Genesis != "" { + mounts = append(mounts, docker.Mount{ + Source: t.Genesis, Target: spec.GenesisPath(), Type: "bind", ReadOnly: true, + }) + } + + suffix, err := randSuffix() + if err != nil { + return "", "", nil, fmt.Errorf("generating container name suffix: %w", err) + } + + cmd := fillerCommand(t, spec) + + containerSpec := &docker.ContainerSpec{ + Name: fmt.Sprintf("benchmarkoor-build-eest-filler-%s-%s", t.FillerClient, suffix), + Image: t.FillerImage, + Command: cmd, + Mounts: mounts, + NetworkName: EESTBuildNetwork, + SecurityOpt: []string{"seccomp=unconfined"}, + // Run as the invoking host user so the state the filler writes into the + // copied datadir is owned by that user and can be cleaned up afterwards + // (the copy is made by the host user, not root). + User: currentUserSpec(), + Labels: b.labels(t), + } + + log.WithField("argv", cmd).Info("Starting filler client") + + id, err = b.mgr.CreateContainer(ctx, containerSpec) + if err != nil { + return "", "", nil, fmt.Errorf("creating filler container: %w", err) + } + + if err = b.mgr.StartContainer(ctx, id); err != nil { + _ = b.mgr.RemoveContainer(context.Background(), id) + + return "", "", nil, fmt.Errorf("starting filler container: %w", err) + } + + go func() { + // Stream filler-client output in the same "🟣 … CLIE | |" + // format `benchmarkoor run` uses for client logs. + w := containerStream("CLIE", t.FillerClient) + if streamErr := b.mgr.StreamLogs(streamCtx, id, w, w); streamErr != nil { + log.WithError(streamErr).Debug("Filler log streaming stopped") + } + }() + + ip, err = b.mgr.GetContainerIP(ctx, id, EESTBuildNetwork) + if err != nil { + _ = b.mgr.RemoveContainer(context.Background(), id) + + return "", "", nil, fmt.Errorf("getting filler container IP: %w", err) + } + + return id, ip, configCleanup, nil +} + +// stopFiller stops and removes the filler container. It uses a background +// context so cleanup still runs when the build context is cancelled. +func (b *EESTPayloadsBuilder) stopFiller(log logrus.FieldLogger, id string) { + timeout := fillerStopTimeoutSec + if err := b.mgr.StopContainer(context.Background(), id, &timeout); err != nil { + log.WithError(err).Warn("Failed to stop filler container") + } + + if err := b.mgr.RemoveContainer(context.Background(), id); err != nil { + log.WithError(err).Warn("Failed to remove filler container") + } +} + +// runFill runs the fill-stateful container against the live filler client. +// ensureFillImage returns the fill image reference, building it from +// fill_dockerfile (once, regardless of target count) when configured, otherwise +// pulling the configured fill_image. +func (b *EESTPayloadsBuilder) ensureFillImage(ctx context.Context, log logrus.FieldLogger) (string, error) { + if !b.cfg.BuildsFillImage() { + if err := b.mgr.PullImage(ctx, b.cfg.FillImage, b.cfg.PullPolicy); err != nil { + return "", fmt.Errorf("pulling fill image %q: %w", b.cfg.FillImage, err) + } + + return b.cfg.FillImage, nil + } + + b.fillImageOnce.Do(func() { + tag := b.cfg.ResolveFillImageTag() + if err := b.buildFillImage(ctx, log, b.cfg.FillDockerfile, tag); err != nil { + b.fillImageErr = err + + return + } + + b.fillImage = tag + }) + + return b.fillImage, b.fillImageErr +} + +// buildFillImage builds the fill image from dockerfile, tagging it tag, by +// shelling out to the configured container runtime's `build`. The build +// context is the Dockerfile's directory. +func (b *EESTPayloadsBuilder) buildFillImage(ctx context.Context, log logrus.FieldLogger, dockerfile, tag string) error { + contextDir := filepath.Dir(dockerfile) + + log.WithFields(logrus.Fields{ + "dockerfile": dockerfile, + "tag": tag, + "runtime": b.runtime, + }).Info("Building fill image") + + cmd := exec.CommandContext(ctx, b.runtime, "build", "-f", dockerfile, "-t", tag, contextDir) + + w := containerStream("BULD", "fill-image-build") + cmd.Stdout = w + cmd.Stderr = w + + if err := cmd.Run(); err != nil { + return fmt.Errorf("building fill image from %q: %w", dockerfile, err) + } + + return nil +} + +func (b *EESTPayloadsBuilder) runFill( + ctx context.Context, + log logrus.FieldLogger, + t *config.EESTPayloadTarget, + fillerIP string, + spec client.Spec, + jwtPath, snapshotHash, eestRepoPath string, +) error { + fillImage, err := b.ensureFillImage(ctx, log) + if err != nil { + return err + } + + args := buildFillArgs(b.cfg.ResolveFillCommand(), t, fillerIP, spec, snapshotHash) + + mounts := []docker.Mount{ + {Source: jwtPath, Target: fillJWTPath, Type: "bind", ReadOnly: true}, + {Source: t.OutputDir, Target: fillOutputPath, Type: "bind"}, + } + + if t.AddressStubsFile != "" { + mounts = append(mounts, docker.Mount{ + Source: t.AddressStubsFile, Target: fillStubsPath, Type: "bind", ReadOnly: true, + }) + } + + // Run as the invoking host user so fixtures written to /out are owned by + // that user. The uv/pytest fill image keeps writable state under /tmp. + env := map[string]string{ + "HOME": "/tmp", + "UV_CACHE_DIR": "/tmp/uv-cache", + "PYTEST_ADDOPTS": "-o cache_dir=/tmp/.pytest_cache", + // fill-stateful reads its commit hash from the /eest git checkout; as a + // non-root user git can refuse with "dubious ownership". Inject + // safe.directory=* via git's env-based config so it trusts the repo. + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "safe.directory", + "GIT_CONFIG_VALUE_0": "*", + } + + // Mount the host-cloned, user-owned EEST checkout at /eest. uv builds the + // venv into this writable dir on first use (cached across runs), so we do + // NOT skip the sync — the toolchain image carries no prebuilt venv. + mounts = append(mounts, docker.Mount{ + Source: eestRepoPath, Target: fillRepoPath, Type: "bind", + }) + + suffix, err := randSuffix() + if err != nil { + return fmt.Errorf("generating container name suffix: %w", err) + } + + containerSpec := &docker.ContainerSpec{ + Name: fmt.Sprintf("benchmarkoor-build-eest-fill-%s-%s", t.FillerClient, suffix), + Image: fillImage, + Command: args, + Mounts: mounts, + NetworkName: EESTBuildNetwork, + User: currentUserSpec(), + Env: env, + Labels: b.labels(t), + } + + tail := newTailBuffer(64 * 1024) + out := io.MultiWriter(containerStream("BULD", "fill-stateful"), tail) + + log.WithField("argv", args).Info("Running fill-stateful") + + if err := b.mgr.RunInitContainer(ctx, containerSpec, out, out); err != nil { + return fmt.Errorf("running fill-stateful: %w (output tail: %s)", err, tail.String()) + } + + log.Info("Build completed") + + return nil +} + +// labels returns the standard label set for a builder container. +func (b *EESTPayloadsBuilder) labels(t *config.EESTPayloadTarget) map[string]string { + return map[string]string{ + "benchmarkoor.managed-by": "benchmarkoor", + "benchmarkoor.builder": EESTPayloadsBuilderName, + "benchmarkoor.client": t.FillerClient, + "benchmarkoor.target": t.EffectiveName(), + "benchmarkoor.output-dir": t.OutputDir, + } +} + +// fillerCommand builds the filler EL client's argv, dispatching on the +// configured filler_client. fill-stateful drives whichever client over the +// standard Engine API flow (eth_sendRawTransaction + engine_forkchoiceUpdated / +// getPayload / newPayload), so every client just needs HTTP RPC (with txpool), +// the Engine API (JWT), peering disabled, and the snapshot-boot workarounds the +// runner uses for the same client. config validation guarantees the client is +// one of the cases below. +func fillerCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + switch t.FillerClient { + case "besu": + return fillerBesuCommand(t, spec) + case "nethermind": + return fillerNethermindCommand(t, spec) + default: + return fillerGethCommand(t, spec) + } +} + +// fillerGethCommand builds the geth argv for the filler client: the http API +// exposes the eth/net/web3/txpool/engine namespaces fill-stateful needs, archive +// gcmode keeps full state, and peering is disabled. spec supplies the +// in-container paths and ports. +func fillerGethCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + args := []string{ + "--config=/tmp/config.toml", + "--datadir=" + spec.DataDir(), + "--port=0", + "--nodiscover", + "--maxpeers=0", + "--bootnodes=", + "--nat=none", + "--syncmode=full", + "--gcmode=archive", + "--snapshot=false", + "--http", + "--http.addr=0.0.0.0", + "--http.vhosts=*", + "--http.corsdomain=*", + "--http.api=admin,debug,eth,miner,net,txpool,web3,testing,engine", + "--http.port=" + strconv.Itoa(spec.RPCPort()), + "--authrpc.jwtsecret=" + spec.JWTPath(), + "--authrpc.addr=0.0.0.0", + "--authrpc.port=" + strconv.Itoa(spec.EnginePort()), + "--authrpc.vhosts=*", + "--miner.gaslimit=" + minerGasLimit, + } + + if t.Genesis != "" { + args = append(args, spec.GenesisFlag()+spec.GenesisPath()) + } + + return append(args, t.FillerExtraArgs...) +} + +// fillerBesuCommand builds the besu argv for the filler client. fill-stateful +// drives every filler over testing_buildBlockV1, which besu exposes behind the +// TESTING JSON-RPC namespace (added in besu-eth/besu#9838, in besu >= 26.6; +// without TESTING in --rpc-http-api besu answers -32604 "Method not enabled"). +// Mirrors the runner besu command (pkg/client/besu.go) otherwise: the ETH/TXPOOL +// namespaces back eth_sendRawTransaction, DEBUG backs the per-test debug_setHead +// rewind, and --p2p-enabled=true lets besu's synchronizer register the snapshot +// head as in-sync — without it besu answers SYNCING to the fill tool's initial +// forkchoice_updated (--max-peers=0 + --discovery-enabled=false keep it +// isolated). The genesis file is required: besu reads chainId from +// --genesis-file at boot, not from the datadir. +func fillerBesuCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + args := []string{ + "--data-path=" + spec.DataDir(), + "--data-storage-format=BONSAI", + // Trust the genesis state hash baked into the state-actor snapshot + // instead of recomputing it from the empty chainspec alloc. + "--genesis-state-hash-cache-enabled=true", + "--sync-mode=FULL", + "--p2p-enabled=true", + "--max-peers=0", + "--discovery-enabled=false", + "--rpc-http-enabled=true", + "--rpc-http-host=0.0.0.0", + "--rpc-http-port=" + strconv.Itoa(spec.RPCPort()), + // TESTING exposes testing_buildBlockV1 (fill-stateful's block builder). + "--rpc-http-api=ETH,NET,WEB3,TXPOOL,DEBUG,ADMIN,MINER,TESTING", + "--rpc-http-cors-origins=*", + "--host-allowlist=*", + "--Xhttp-timeout-seconds=660", + "--engine-rpc-enabled=true", + "--engine-jwt-secret=" + spec.JWTPath(), + "--engine-rpc-port=" + strconv.Itoa(spec.EnginePort()), + "--engine-host-allowlist=*", + "--target-gas-limit=" + minerGasLimit, + } + + if t.Genesis != "" { + args = append(args, spec.GenesisFlag()+spec.GenesisPath()) + } + + return append(args, t.FillerExtraArgs...) +} + +// fillerNethermindCommand builds the nethermind argv for the filler client. +// fill-stateful drives every filler over testing_buildBlockV1, so the HTTP RPC +// must expose the Testing module (without it nethermind answers -32604 "Method +// not enabled"); this needs a nethermind build that ships testing_buildBlockV1 +// (e.g. nethermindeth/nethermind:testing_build_block_with_opcode_tracing — set +// as the target's filler_image). Module list, target gas limit and BaseDbPath +// mirror NethermindEth/gas-benchmarks' stateful generator. --Init.BaseDbPath +// points at the datadir so nethermind reads the state-actor snapshot written +// there (its BaseDbPath otherwise defaults to a nested per-network dir holding +// an empty state db). The genesis file is required: nethermind reads the chain +// config from --Init.ChainSpecPath. +func fillerNethermindCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + args := []string{ + "--datadir=" + spec.DataDir(), + "--Init.BaseDbPath=" + spec.DataDir(), + "--config=none", + "--Network.DiscoveryPort=0", + "--Network.MaxActivePeers=0", + "--Init.DiscoveryEnabled=false", + "--Sync.MaxAttemptsToUpdatePivot=0", + "--Network.ExternalIp=127.0.0.1", + "--JsonRpc.Enabled=true", + "--JsonRpc.Host=0.0.0.0", + "--JsonRpc.Port=" + strconv.Itoa(spec.RPCPort()), + // Testing exposes testing_buildBlockV1 (fill-stateful's block builder). + "--JsonRpc.EnabledModules=Eth,Net,Web3,Admin,Debug,Trace,TxPool,Subscribe,Testing", + "--JsonRpc.EngineEnabledModules=Net,Eth,Subscribe,Web3,Testing,Engine", + "--JsonRpc.Timeout=600000", + "--JsonRpc.JwtSecretFile=" + spec.JWTPath(), + "--JsonRpc.EngineHost=0.0.0.0", + "--JsonRpc.EnginePort=" + strconv.Itoa(spec.EnginePort()), + "--Merge.TerminalTotalDifficulty=0", + "--Blocks.TargetBlockGasLimit=" + minerGasLimit, + } + + if t.Genesis != "" { + args = append(args, spec.GenesisFlag()+spec.GenesisPath()) + } + + return append(args, t.FillerExtraArgs...) +} + +// buildFillArgs assembles the fill-stateful argv: the configured command +// prefix, the live-client endpoints, the run knobs, and the test selection. +func buildFillArgs( + prefix []string, + t *config.EESTPayloadTarget, + fillerIP string, + spec client.Spec, + snapshotHash string, +) []string { + // NB: we deliberately do NOT pass --clean. fill-stateful's --clean does + // shutil.rmtree(output), which fails with EBUSY when output is a bind + // mount (it can't remove the mountpoint). benchmarkoor already owns the + // output_dir lifecycle — Build leaves it empty (skip-if-populated; --force + // wipes it) before we get here, so fill-stateful just mkdirs into it. + args := append([]string{}, prefix...) + args = append(args, + // -v makes the underlying pytest print each test node id and its + // outcome as it is built, so the fill progress is visible instead of + // bare progress dots. + "-v", + fmt.Sprintf("--rpc-endpoint=http://%s:%d", fillerIP, spec.RPCPort()), + fmt.Sprintf("--engine-endpoint=http://%s:%d", fillerIP, spec.EnginePort()), + "--engine-jwt-secret-file="+fillJWTPath, + "--fork="+t.Fork, + "--snapshot-block="+snapshotHash, + "--output="+fillOutputPath, + ) + + if len(t.GasBenchmarkValues) > 0 { + vals := make([]string, len(t.GasBenchmarkValues)) + for i, v := range t.GasBenchmarkValues { + vals[i] = strconv.Itoa(v) + } + + // fill-stateful's --gas-benchmark-values takes one comma-separated + // argument, e.g. "10,30" for 10M and 30M gas. + args = append(args, "--gas-benchmark-values="+strings.Join(vals, ",")) + } + + // --fixed-opcode-count is mutually exclusive with --gas-benchmark-values + // (config validation enforces this). A non-nil but empty list passes the + // flag bare, which makes fill-stateful use its .fixed_opcode_counts.json. + if t.FixedOpcodeCount != nil { + if counts := *t.FixedOpcodeCount; len(counts) > 0 { + vals := make([]string, len(counts)) + for i, v := range counts { + vals[i] = strconv.FormatFloat(v, 'g', -1, 64) + } + + args = append(args, "--fixed-opcode-count="+strings.Join(vals, ",")) + } else { + args = append(args, "--fixed-opcode-count") + } + } + + if t.MaxGasPerTest != nil { + args = append(args, fmt.Sprintf("--max-gas-per-test=%d", *t.MaxGasPerTest)) + } + + if t.RPCSeedKey != "" { + args = append(args, "--rpc-seed-key="+t.RPCSeedKey) + } + + // besu suggests a zero priority fee on a freshly-booted snapshot; pin a + // non-zero tip so fill-stateful's session-fee check passes (see + // besuDefaultPriorityFeeWei). geth derives a non-zero tip itself. + if t.FillerClient == "besu" { + args = append(args, "--default-max-priority-fee-per-gas="+besuDefaultPriorityFeeWei) + } + + if t.AddressStubsFile != "" { + args = append(args, "--address-stubs="+fillStubsPath) + } + + args = append(args, t.Tests...) + + if t.Filter != "" { + args = append(args, "-k", t.Filter) + } + + if t.Marker != "" { + args = append(args, "-m", t.Marker) + } + + return args +} + +// patchFillerGenesis reads the target's boot genesis, applies its +// genesis_fork_override / genesis_eip_override (mutually exclusive; validated), +// and writes the result to a temp file the filler container can bind-mount. +// Returns the temp path and a cleanup callback. It is the builder-side analogue +// of the runner's per-instance genesis patching. +func patchFillerGenesis( + log logrus.FieldLogger, t *config.EESTPayloadTarget, +) (string, func(), error) { + raw, err := os.ReadFile(t.Genesis) + if err != nil { + return "", nil, fmt.Errorf("reading genesis %q: %w", t.Genesis, err) + } + + var patched []byte + + switch { + case len(t.GenesisForkOverride) > 0: + patched, err = genesis.ApplyForkOverrides(raw, t.GenesisForkOverride) + if err != nil { + return "", nil, fmt.Errorf("applying genesis_fork_override: %w", err) + } + + log.WithField("forks", t.GenesisForkOverride). + Info("Applied genesis fork-time overrides to filler genesis") + case t.GenesisEIPOverride != nil: + patched, err = genesis.ApplyEIPOverrides( + raw, t.GenesisEIPOverride.Timestamp, t.GenesisEIPOverride.EIPs, + ) + if err != nil { + return "", nil, fmt.Errorf("applying genesis_eip_override: %w", err) + } + + log.WithFields(logrus.Fields{ + "eips": t.GenesisEIPOverride.EIPs, "timestamp": t.GenesisEIPOverride.Timestamp, + }).Info("Applied genesis EIP-time overrides to filler genesis") + } + + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-eest-genesis-*") + if err != nil { + return "", nil, fmt.Errorf("creating temp genesis file: %w", err) + } + + path := f.Name() + + cleanup := func() { _ = os.Remove(path) } + + if _, err := f.Write(patched); err != nil { + _ = f.Close() + cleanup() + + return "", nil, fmt.Errorf("writing temp genesis file: %w", err) + } + + if err := f.Close(); err != nil { + cleanup() + + return "", nil, fmt.Errorf("closing temp genesis file: %w", err) + } + + if err := os.Chmod(path, 0o644); err != nil { + cleanup() + + return "", nil, fmt.Errorf("chmod temp genesis file: %w", err) + } + + return path, cleanup, nil +} + +// materializeAddressStubs serializes a target's inline address_stubs map to a +// temp JSON file readable by the container UID (0644) and returns its path plus +// a cleanup callback. The file mirrors the on-disk address_stubs_file format so +// downstream mount + --address-stubs handling is identical. +func materializeAddressStubs( + log logrus.FieldLogger, t *config.EESTPayloadTarget, +) (string, func(), error) { + data, err := json.MarshalIndent(t.AddressStubs, "", " ") + if err != nil { + return "", nil, fmt.Errorf("marshaling address_stubs: %w", err) + } + + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-eest-stubs-*") + if err != nil { + return "", nil, fmt.Errorf("creating temp stubs file: %w", err) + } + + path := f.Name() + + cleanup := func() { _ = os.Remove(path) } + + if _, err := f.Write(data); err != nil { + _ = f.Close() + cleanup() + + return "", nil, fmt.Errorf("writing temp stubs file: %w", err) + } + + if err := f.Close(); err != nil { + cleanup() + + return "", nil, fmt.Errorf("closing temp stubs file: %w", err) + } + + if err := os.Chmod(path, 0o644); err != nil { + cleanup() + + return "", nil, fmt.Errorf("chmod temp stubs file: %w", err) + } + + log.WithField("stubs", len(t.AddressStubs)).Info("Materialized inline address_stubs to temp file") + + return path, cleanup, nil +} + +// writeTempJWT writes the JWT secret to a temp file readable by the +// container UID (0644) and returns its path plus a cleanup callback. +func writeTempJWT(secret string) (string, func(), error) { + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-eest-jwt-*") + if err != nil { + return "", nil, fmt.Errorf("creating temp jwt file: %w", err) + } + + path := f.Name() + + cleanup := func() { _ = os.Remove(path) } + + if _, err := f.WriteString(secret); err != nil { + _ = f.Close() + cleanup() + + return "", nil, fmt.Errorf("writing temp jwt file: %w", err) + } + + if err := f.Close(); err != nil { + cleanup() + + return "", nil, fmt.Errorf("closing temp jwt file: %w", err) + } + + if err := os.Chmod(path, 0o644); err != nil { + cleanup() + + return "", nil, fmt.Errorf("chmod temp jwt file: %w", err) + } + + return path, cleanup, nil +} + +// writeTempConfigFiles materialises a client's in-container config files +// (path → content) to host temp files and returns the corresponding +// read-only bind mounts plus a cleanup callback. +func writeTempConfigFiles(files map[string]string) ([]docker.Mount, func(), error) { + mounts := make([]docker.Mount, 0, len(files)) + paths := make([]string, 0, len(files)) + + cleanup := func() { + for _, p := range paths { + _ = os.Remove(p) + } + } + + for target, content := range files { + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-eest-config-*") + if err != nil { + cleanup() + + return nil, nil, fmt.Errorf("creating temp config file: %w", err) + } + + path := f.Name() + paths = append(paths, path) + + if _, err := f.WriteString(content); err != nil { + _ = f.Close() + cleanup() + + return nil, nil, fmt.Errorf("writing temp config file: %w", err) + } + + if err := f.Close(); err != nil { + cleanup() + + return nil, nil, fmt.Errorf("closing temp config file: %w", err) + } + + if err := os.Chmod(path, 0o644); err != nil { + cleanup() + + return nil, nil, fmt.Errorf("chmod temp config file: %w", err) + } + + mounts = append(mounts, docker.Mount{ + Source: path, Target: target, Type: "bind", ReadOnly: true, + }) + } + + return mounts, cleanup, nil +} diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go new file mode 100644 index 000000000..79983718c --- /dev/null +++ b/pkg/builder/eest_payloads_test.go @@ -0,0 +1,289 @@ +package builder + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/ethpandaops/benchmarkoor/pkg/client" + "github.com/ethpandaops/benchmarkoor/pkg/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildFillArgs(t *testing.T) { + spec := client.NewGethSpec() + prefix := []string{"uv", "run", "fill-stateful"} + + tests := []struct { + name string + target *config.EESTPayloadTarget + wantContain []string + wantAbsent []string + }{ + { + name: "minimal", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + Tests: []string{"tests/benchmark/compute"}, + }, + wantContain: []string{ + "uv", "run", "fill-stateful", "-v", + "--rpc-endpoint=http://10.0.0.5:8545", + "--engine-endpoint=http://10.0.0.5:8551", + "--engine-jwt-secret-file=" + fillJWTPath, + "--fork=Osaka", + "--snapshot-block=0xabc", + "--output=" + fillOutputPath, + "tests/benchmark/compute", + }, + wantAbsent: []string{ + "--clean", "--gas-benchmark-values", "--fixed-opcode-count", "--max-gas-per-test", + "--rpc-seed-key", "--address-stubs", "-k", + }, + }, + { + name: "fixed opcode count values", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + FixedOpcodeCount: &[]float64{0.5, 1, 2}, + Tests: []string{"tests/benchmark/compute"}, + }, + wantContain: []string{"--fixed-opcode-count=0.5,1,2"}, + wantAbsent: []string{"--gas-benchmark-values"}, + }, + { + name: "fixed opcode count bare (default json)", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + FixedOpcodeCount: &[]float64{}, + Tests: []string{"tests/benchmark/compute"}, + }, + wantContain: []string{"--fixed-opcode-count"}, + wantAbsent: []string{"--fixed-opcode-count=", "--gas-benchmark-values"}, + }, + { + name: "full", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + GasBenchmarkValues: []int{10, 30}, + MaxGasPerTest: u64(45000000), + RPCSeedKey: "0xdead", + AddressStubsFile: "/host/stubs.json", + Tests: []string{"tests/benchmark/compute", "tests/benchmark/stateful"}, + Filter: "bn128", + }, + wantContain: []string{ + "--gas-benchmark-values=10,30", + "--max-gas-per-test=45000000", + "--rpc-seed-key=0xdead", + "--address-stubs=" + fillStubsPath, + "tests/benchmark/compute", + "tests/benchmark/stateful", + "-k", "bn128", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildFillArgs(prefix, tt.target, "10.0.0.5", spec, "0xabc") + + for _, want := range tt.wantContain { + assert.Contains(t, got, want) + } + + for _, absent := range tt.wantAbsent { + for _, arg := range got { + assert.NotContains(t, arg, absent) + } + } + }) + } + + // The -k filter value must immediately follow the -k flag. + got := buildFillArgs(prefix, &config.EESTPayloadTarget{ + FillerClient: "geth", Fork: "Osaka", Tests: []string{"t"}, Filter: "expr", + }, "1.2.3.4", spec, "0x1") + idx := slices.Index(got, "-k") + require.GreaterOrEqual(t, idx, 0, "-k flag must be present") + require.Less(t, idx+1, len(got)) + assert.Equal(t, "expr", got[idx+1]) + assert.NotContains(t, got, "-m", "-m absent when marker is unset") + + // The -m marker value must immediately follow the -m flag. + gotMarker := buildFillArgs(prefix, &config.EESTPayloadTarget{ + FillerClient: "geth", Fork: "Osaka", Tests: []string{"t"}, Marker: "repricing", + }, "1.2.3.4", spec, "0x1") + mIdx := slices.Index(gotMarker, "-m") + require.GreaterOrEqual(t, mIdx, 0, "-m flag must be present") + require.Less(t, mIdx+1, len(gotMarker)) + assert.Equal(t, "repricing", gotMarker[mIdx+1]) +} + +func TestFillerGethCommand(t *testing.T) { + spec := client.NewGethSpec() + + t.Run("without genesis", func(t *testing.T) { + cmd := fillerGethCommand(&config.EESTPayloadTarget{FillerClient: "geth"}, spec) + + assert.Contains(t, cmd, "--datadir=/data") + assert.Contains(t, cmd, "--http.api=admin,debug,eth,miner,net,txpool,web3,testing,engine") + assert.Contains(t, cmd, "--authrpc.jwtsecret=/tmp/jwtsecret") + assert.Contains(t, cmd, "--authrpc.port=8551") + assert.Contains(t, cmd, "--http.port=8545") + assert.Contains(t, cmd, "--gcmode=archive") + + for _, arg := range cmd { + assert.NotContains(t, arg, "--override.genesis", "no genesis flag when genesis_file unset") + } + }) + + t.Run("with genesis and extra args", func(t *testing.T) { + cmd := fillerGethCommand(&config.EESTPayloadTarget{ + FillerClient: "geth", + Genesis: "/host/genesis.json", + FillerExtraArgs: []string{"--verbosity=5"}, + }, spec) + + assert.Contains(t, cmd, "--override.genesis=/tmp/genesis.json") + assert.Contains(t, cmd, "--verbosity=5") + assert.Equal(t, "--verbosity=5", cmd[len(cmd)-1], "extra args are appended last") + }) +} + +func TestFillerCommand_Besu(t *testing.T) { + spec := client.NewBesuSpec() + + cmd := fillerCommand(&config.EESTPayloadTarget{ + FillerClient: "besu", + Genesis: "/host/besu-chainspec.json", + FillerExtraArgs: []string{"--logging=DEBUG"}, + }, spec) + + assert.Contains(t, cmd, "--data-path=/data") + // TESTING exposes testing_buildBlockV1 (required by fill-stateful). + assert.Contains(t, cmd, "--rpc-http-api=ETH,NET,WEB3,TXPOOL,DEBUG,ADMIN,MINER,TESTING") + assert.Contains(t, cmd, "--engine-jwt-secret=/tmp/jwtsecret") + assert.Contains(t, cmd, "--engine-rpc-port=8551") + assert.Contains(t, cmd, "--rpc-http-port=8545") + // besu boots a snapshot only when its synchronizer can register the head. + assert.Contains(t, cmd, "--p2p-enabled=true") + // besu reads chainId from the genesis file, so it must be passed. + assert.Contains(t, cmd, "--genesis-file=/tmp/genesis.json") + assert.Equal(t, "--logging=DEBUG", cmd[len(cmd)-1], "extra args are appended last") +} + +func TestFillerCommand_Nethermind(t *testing.T) { + spec := client.NewNethermindSpec() + + cmd := fillerCommand(&config.EESTPayloadTarget{ + FillerClient: "nethermind", + Genesis: "/host/parity-chainspec.json", + }, spec) + + assert.Contains(t, cmd, "--datadir=/data") + // Point BaseDbPath at the datadir so nethermind reads the state-actor snapshot. + assert.Contains(t, cmd, "--Init.BaseDbPath=/data") + // Testing module exposes testing_buildBlockV1 (required by fill-stateful). + assert.Contains(t, cmd, "--JsonRpc.EnabledModules=Eth,Net,Web3,Admin,Debug,Trace,TxPool,Subscribe,Testing") + assert.Contains(t, cmd, "--JsonRpc.JwtSecretFile=/tmp/jwtsecret") + assert.Contains(t, cmd, "--JsonRpc.EnginePort=8551") + assert.Contains(t, cmd, "--JsonRpc.Port=8545") + assert.Contains(t, cmd, "--Init.ChainSpecPath=/tmp/genesis.json") +} + +func TestEESTPayloadsBuilder_Targets(t *testing.T) { + cfg := &config.EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []config.EESTPayloadTarget{ + {Name: "compute", FillerClient: "geth", OutputDir: "/srv/c"}, + {FillerClient: "geth", OutputDir: "/srv/g"}, + }, + } + + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}, t.TempDir()) + + got := b.Targets() + require.Len(t, got, 2) + assert.Equal(t, TargetInfo{Name: "compute", Client: "geth", OutputDir: "/srv/c"}, got[0]) + assert.Equal(t, TargetInfo{Name: "geth", Client: "geth", OutputDir: "/srv/g"}, got[1]) +} + +func TestEESTPayloadsBuilder_BuildUnknownTarget(t *testing.T) { + cfg := &config.EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []config.EESTPayloadTarget{{FillerClient: "geth", OutputDir: "/srv/g"}}, + } + + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}, t.TempDir()) + + _, err := b.Build(context.Background(), "nope", BuildOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no target named") +} + +func TestEESTPayloadsBuilder_BuildSkipsPopulatedDir(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "leftover"), []byte("x"), 0o644)) + + // fakeMgr panics on the orchestration methods; reaching them would fail + // the test, proving the skip path returns before any container work. + cfg := &config.EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []config.EESTPayloadTarget{{ + Name: "compute", FillerClient: "geth", SourceDir: "/snap", OutputDir: dir, + Fork: "Osaka", FillerImage: "geth:master", Tests: []string{"tests/benchmark/compute"}, + }}, + } + + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}, t.TempDir()) + + skipped, err := b.Build(context.Background(), "compute", BuildOptions{}) + require.NoError(t, err) + assert.True(t, skipped, "Build should report skipped=true when output_dir is non-empty") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "leftover", entries[0].Name()) +} + +func TestMaterializeAddressStubs(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + + stubs := map[string]map[string]string{ + "bloated_eoa_10GB": { + "addr": "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04", + "pkey": "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e", + }, + } + tgt := &config.EESTPayloadTarget{AddressStubs: stubs} + + path, cleanup, err := materializeAddressStubs(noopLogger(), tgt) + require.NoError(t, err) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o644), info.Mode().Perm(), "stubs file must be container-readable") + + data, err := os.ReadFile(path) + require.NoError(t, err) + + var got map[string]map[string]string + require.NoError(t, json.Unmarshal(data, &got)) + assert.Equal(t, stubs, got, "materialized JSON must round-trip the inline map") + + // cleanup removes the temp file. + cleanup() + + _, err = os.Stat(path) + assert.True(t, os.IsNotExist(err), "cleanup should remove the temp stubs file") +} diff --git a/pkg/builder/rpc.go b/pkg/builder/rpc.go new file mode 100644 index 000000000..f63f69960 --- /dev/null +++ b/pkg/builder/rpc.go @@ -0,0 +1,127 @@ +package builder + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// healthCheckInterval is how often waitForRPC polls the endpoint. +const healthCheckInterval = time.Second + +// waitForRPC blocks until the EL client at host:port answers +// web3_clientVersion or ctx is cancelled, returning the client version. +// These helpers mirror pkg/runner/rpc.go but are standalone so the builder +// does not depend on the runner. +func waitForRPC(ctx context.Context, host string, port int) (string, error) { + url := fmt.Sprintf("http://%s:%d", host, port) + + ticker := time.NewTicker(healthCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return "", fmt.Errorf("timeout waiting for RPC at %s: %w", url, ctx.Err()) + case <-ticker.C: + if version, ok := checkRPCHealth(ctx, url); ok { + return version, nil + } + } + } +} + +// getLatestBlockHash returns the hash of the latest block at host:port. +func getLatestBlockHash(ctx context.Context, host string, port int) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + url := fmt.Sprintf("http://%s:%d", host, port) + body := `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}` + + resp, err := postJSON(ctx, url, body) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("reading response: %w", err) + } + + var rpcResp struct { + Result struct { + Hash string `json:"hash"` + } `json:"result"` + } + + if err := json.Unmarshal(respBody, &rpcResp); err != nil { + return "", fmt.Errorf("parsing response: %w", err) + } + + if rpcResp.Result.Hash == "" { + return "", fmt.Errorf("latest block has no hash (client not ready?)") + } + + return rpcResp.Result.Hash, nil +} + +// checkRPCHealth performs a single web3_clientVersion call, returning the +// version on success. +func checkRPCHealth(ctx context.Context, url string) (string, bool) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + body := `{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":1}` + + resp, err := postJSON(ctx, url, body) + if err != nil { + return "", false + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", false + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", false + } + + var rpcResp struct { + Result string `json:"result"` + } + + if err := json.Unmarshal(respBody, &rpcResp); err != nil { + return "", false + } + + return rpcResp.Result, true +} + +// postJSON issues a JSON-RPC POST and returns the response. +func postJSON(ctx context.Context, url, body string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("executing request: %w", err) + } + + return resp, nil +} diff --git a/pkg/builder/state_actor.go b/pkg/builder/state_actor.go index 3c8fed08a..c83885b9b 100644 --- a/pkg/builder/state_actor.go +++ b/pkg/builder/state_actor.go @@ -1,10 +1,7 @@ package builder import ( - "bytes" "context" - "crypto/rand" - "encoding/hex" "fmt" "io" "os" @@ -89,12 +86,10 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp return false, fmt.Errorf("no image configured for client %q", target.Client) } - log := b.log.WithFields(logrus.Fields{ - "target": target.EffectiveName(), - "client": target.Client, - "output_dir": target.OutputDir, - "image": image, - }) + // Keep only the target on the per-line logger; client/output_dir are already + // in build.go's "Building target" header and image is logged on the + // "Running state-actor" line, so they don't repeat on every streamed line. + log := b.log.WithField("target", target.EffectiveName()) // The CLI `--force` flag and per-target `force: true` both bypass the // skip-on-populated check, wipe the existing output_dir, and forward @@ -115,7 +110,7 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp } } - if err := b.prepareOutputDir(target.OutputDir, force); err != nil { + if err := prepareOutputDir(target.OutputDir, force); err != nil { return false, err } @@ -153,6 +148,9 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp Image: image, Command: args, Mounts: mounts, + // Run as the invoking host user so the output datadir is owned by that + // user (not root) and is readable when a later step copies it. + User: currentUserSpec(), Labels: map[string]string{ "benchmarkoor.managed-by": "benchmarkoor", "benchmarkoor.builder": StateActorBuilderName, @@ -166,10 +164,10 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp // exit yields a useful error without spamming the whole log. tail := newTailBuffer(64 * 1024) - stdout := io.MultiWriter(logWriter(log, logrus.InfoLevel), tail) - stderr := io.MultiWriter(logWriter(log, logrus.InfoLevel), tail) + stdout := io.MultiWriter(containerStream("BULD", "state-actor"), tail) + stderr := io.MultiWriter(containerStream("BULD", "state-actor"), tail) - log.WithField("argv", args).Info("Running state-actor") + log.WithFields(logrus.Fields{"image": image, "argv": args}).Info("Running state-actor") if err := b.mgr.RunInitContainer(ctx, spec, stdout, stderr); err != nil { return false, fmt.Errorf("running state-actor: %w (output tail: %s)", @@ -193,38 +191,6 @@ func (b *StateActorBuilder) findTargetIndex(name string) int { return -1 } -// isPopulated reports whether dir exists and contains at least one -// entry. A missing dir returns (false, nil). -func isPopulated(dir string) (bool, error) { - entries, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - - return false, fmt.Errorf("reading output_dir %q: %w", dir, err) - } - - return len(entries) > 0, nil -} - -// prepareOutputDir ensures the target's output_dir exists. When force is -// true the directory is removed first; the populated check that gates -// skip-vs-build lives in Build, not here. -func (b *StateActorBuilder) prepareOutputDir(dir string, force bool) error { - if force { - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("removing existing output_dir %q: %w", dir, err) - } - } - - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("creating output_dir %q: %w", dir, err) - } - - return nil -} - // buildMounts assembles the bind mounts state-actor needs: an identity // mount of the output_dir (RW) and, when a spec path is resolved, a // read-only mount of that file at the same path inside the container so @@ -272,7 +238,7 @@ func (b *StateActorBuilder) resolveSpecPath() (string, func(), error) { return abs, nil, nil case config.StateActorSpecInline: - f, err := os.CreateTemp("", "benchmarkoor-state-actor-spec-*.yaml") + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-state-actor-spec-*.yaml") if err != nil { return "", nil, fmt.Errorf("creating temp spec file: %w", err) } @@ -388,81 +354,5 @@ func dbPath(target *config.StateActorTarget) string { return target.OutputDir } -// randSuffix returns a 6-hex-character random string used to keep -// concurrent build container names unique. -func randSuffix() (string, error) { - var b [3]byte - - if _, err := rand.Read(b[:]); err != nil { - return "", err - } - - return hex.EncodeToString(b[:]), nil -} - -// logWriter returns an io.Writer that forwards each line written to it -// to the supplied logger at the given level. Used to stream container -// stdout/stderr without writing the bytes directly to the global -// stdout/stderr. -func logWriter(log logrus.FieldLogger, level logrus.Level) io.Writer { - return &lineLogger{log: log, level: level} -} - -type lineLogger struct { - log logrus.FieldLogger - level logrus.Level - buf bytes.Buffer -} - -func (w *lineLogger) Write(p []byte) (int, error) { - w.buf.Write(p) - - for { - i := bytes.IndexByte(w.buf.Bytes(), '\n') - if i < 0 { - break - } - - line := w.buf.Next(i + 1) - msg := string(bytes.TrimRight(line, "\r\n")) - - switch w.level { - case logrus.ErrorLevel: - w.log.Error(msg) - case logrus.WarnLevel: - w.log.Warn(msg) - case logrus.DebugLevel: - w.log.Debug(msg) - default: - w.log.Info(msg) - } - } - - return len(p), nil -} - -// tailBuffer is an io.Writer that retains at most `max` bytes of the -// most recent input. Useful for surfacing the trailing output of a -// failing container in the resulting error. -type tailBuffer struct { - buf bytes.Buffer - max int -} - -func newTailBuffer(maxBytes int) *tailBuffer { - return &tailBuffer{max: maxBytes} -} - -func (t *tailBuffer) Write(p []byte) (int, error) { - t.buf.Write(p) - - if excess := t.buf.Len() - t.max; excess > 0 { - t.buf.Next(excess) - } - - return len(p), nil -} - -func (t *tailBuffer) String() string { - return t.buf.String() -} +// randSuffix, logWriter/lineLogger and tailBuffer live in util.go — shared +// across builders. diff --git a/pkg/builder/state_actor_test.go b/pkg/builder/state_actor_test.go index 29499d1f3..3567fcf5d 100644 --- a/pkg/builder/state_actor_test.go +++ b/pkg/builder/state_actor_test.go @@ -539,6 +539,9 @@ func (m *fakeMgr) Stop() error { panic("Stop not used in build func (m *fakeMgr) EnsureNetwork(_ context.Context, _ string) error { panic("EnsureNetwork not used in builder tests") } +func (m *fakeMgr) NetworkExists(_ context.Context, _ string) (bool, error) { + panic("NetworkExists not used in builder tests") +} func (m *fakeMgr) RemoveNetwork(_ context.Context, _ string) error { panic("RemoveNetwork not used in builder tests") } diff --git a/pkg/builder/util.go b/pkg/builder/util.go new file mode 100644 index 000000000..a6a632685 --- /dev/null +++ b/pkg/builder/util.go @@ -0,0 +1,173 @@ +package builder + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/ethpandaops/benchmarkoor/pkg/config" +) + +// mountTempDir returns the system temp dir with symlinks resolved. It is the +// base for host files bind-mounted into builder containers. On macOS $TMPDIR is +// /var/folders/… where /var is a symlink to /private/var; Docker Desktop shares +// /private but not the /var alias, so bind-mounting a file at the unresolved +// path silently fails to appear in the container ("no such file or directory"). +// Resolving to /private/var/… makes the mount work; on Linux it's a no-op. +func mountTempDir() string { + if resolved, err := filepath.EvalSymlinks(os.TempDir()); err == nil { + return resolved + } + + return os.TempDir() +} + +// isPopulated reports whether dir exists and contains at least one +// entry. A missing dir returns (false, nil). +func isPopulated(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + + return false, fmt.Errorf("reading output_dir %q: %w", dir, err) + } + + return len(entries) > 0, nil +} + +// prepareOutputDir ensures dir exists. When force is true the directory is +// removed first; the populated check that gates skip-vs-build lives in each +// builder's Build, not here. +func prepareOutputDir(dir string, force bool) error { + if force { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("removing existing output_dir %q: %w", dir, err) + } + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating output_dir %q: %w", dir, err) + } + + return nil +} + +// currentUserSpec returns the invoking process's user as a docker "uid:gid" +// string. Builder containers run as this user so the datadirs and fixtures +// they write are owned by the host user instead of root — avoiding +// permission-denied failures when a later non-root step (e.g. the datadir +// copy) reads that output. +func currentUserSpec() string { + return fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) +} + +// randSuffix returns a 6-hex-character random string used to keep +// concurrent build container names unique. +func randSuffix() (string, error) { + var b [3]byte + + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + + return hex.EncodeToString(b[:]), nil +} + +// containerStream returns an io.Writer that prefixes each line of streamed +// container output with "$emoji $TS $label | $name | " and writes it directly +// to stdout. EL-client output ("CLIE") uses 🟣 to match the client-log format +// `benchmarkoor run` uses (see pkg/runner clientLogPrefix); build-tool +// containers ("BULD", e.g. state-actor / fill-stateful) use 🟠 so they are easy +// to distinguish from the filler client streaming alongside them. +func containerStream(label, name string) io.Writer { + return &containerStreamWriter{emoji: streamEmoji(label), label: label, name: name, w: os.Stdout} +} + +// streamEmoji picks the leading emoji for a streamed-container line based on its +// label. Build-tool output gets 🟠; everything else (EL clients) keeps 🟣. +func streamEmoji(label string) string { + if label == "BULD" { + return "🟠" + } + + return "🟣" +} + +// ansiReset clears any ANSI color/style left set by a streamed line. Tools like +// pytest emit a bold/color sequence (e.g. the test-session header) without a +// trailing reset, which would otherwise bleed into the next line — including our +// prefix — making everything after it bold. +const ansiReset = "\x1b[0m" + +type containerStreamWriter struct { + emoji string + label string + name string + w io.Writer + buf bytes.Buffer +} + +func (w *containerStreamWriter) Write(p []byte) (int, error) { + w.buf.Write(p) + + for { + i := bytes.IndexByte(w.buf.Bytes(), '\n') + if i < 0 { + break + } + + line := w.buf.Next(i + 1) + ts := time.Now().UTC().Format(config.LogTimestampFormat) + msg := bytes.TrimRight(line, "\r\n") + + if _, err := fmt.Fprintf(w.w, "%s %s %s | %s | %s%s\n", w.emoji, ts, w.label, w.name, msg, ansiReset); err != nil { + return len(p), err + } + } + + return len(p), nil +} + +// tailBuffer is an io.Writer that retains at most `max` bytes of the most +// recent input. Useful for surfacing the trailing output of a failing +// container in the resulting error. It is safe for concurrent use: the +// container log-streaming goroutine can still be draining final lines into it +// while the caller reads String() on the error path (the streaming goroutine is +// not joined before the container-wait returns). +type tailBuffer struct { + mu sync.Mutex + buf bytes.Buffer + max int +} + +func newTailBuffer(maxBytes int) *tailBuffer { + return &tailBuffer{max: maxBytes} +} + +func (t *tailBuffer) Write(p []byte) (int, error) { + t.mu.Lock() + defer t.mu.Unlock() + + t.buf.Write(p) + + if excess := t.buf.Len() - t.max; excess > 0 { + t.buf.Next(excess) + } + + return len(p), nil +} + +func (t *tailBuffer) String() string { + t.mu.Lock() + defer t.mu.Unlock() + + return t.buf.String() +} diff --git a/pkg/builder/util_test.go b/pkg/builder/util_test.go new file mode 100644 index 000000000..5f7201d74 --- /dev/null +++ b/pkg/builder/util_test.go @@ -0,0 +1,39 @@ +package builder + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestTailBufferConcurrent exercises the case the mutex guards: the container +// log-streaming goroutine keeps Write()-ing while the caller reads String() on +// the error path (RunInitContainer doesn't join the streaming goroutine). Run +// with -race to catch a regression. +func TestTailBufferConcurrent(t *testing.T) { + const maxBytes = 64 + + tb := newTailBuffer(maxBytes) + + var wg sync.WaitGroup + + wg.Add(1) + + go func() { + defer wg.Done() + + for i := 0; i < 2000; i++ { + _, _ = tb.Write([]byte("a streamed container log line\n")) + } + }() + + for i := 0; i < 2000; i++ { + _ = tb.String() + } + + wg.Wait() + + // The retained tail is bounded by max regardless of how much was written. + assert.LessOrEqual(t, len(tb.String()), maxBytes) +} diff --git a/pkg/client/besu.go b/pkg/client/besu.go index 4f8dd2b08..366c3242e 100644 --- a/pkg/client/besu.go +++ b/pkg/client/besu.go @@ -23,6 +23,14 @@ func (s *besuSpec) DefaultCommand() []string { // Data directory - should always point to /data "--data-path=/data", "--data-storage-format=BONSAI", + // Trust the genesis state hash stored in the datadir instead of + // recomputing it from the chainspec alloc. Required to boot a + // state-actor snapshot: state-actor writes synthetic state (and the + // genesis state root) directly into RocksDB and emits an empty + // chainspec alloc, so the default recompute path would otherwise fail + // with "Supplied genesis block does not match chain data stored". Has + // no effect on a normal genesis-initialised datadir. + "--genesis-state-hash-cache-enabled=true", // Peering / Syncing / TXPool "--p2p-enabled=false", "--sync-mode=FULL", diff --git a/pkg/client/reth.go b/pkg/client/reth.go index 38b3f7a19..4ce92fd15 100644 --- a/pkg/client/reth.go +++ b/pkg/client/reth.go @@ -38,6 +38,13 @@ func (s *rethSpec) DefaultCommand() []string { "--authrpc.addr=0.0.0.0", "--authrpc.port=8551", "--engine.disable-precompile-cache", + // Trust the genesis stored in the datadir instead of recomputing it from + // the chainspec alloc. Required to boot a state-actor snapshot: state-actor + // direct-writes synthetic state (and the genesis) into reth's DB and emits a + // chainspec whose alloc can't reproduce it, so reth would otherwise abort with + // "genesis hash in the storage does not match the specified chainspec". + // (Besu's equivalent is --genesis-state-hash-cache-enabled.) + "--debug.skip-genesis-validation", // Others "--full", } diff --git a/pkg/config/config.go b/pkg/config/config.go index a67410365..d1738abfb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -73,11 +73,13 @@ type Config struct { Builder *BuilderConfig `yaml:"builder,omitempty" mapstructure:"builder"` } -// BuilderConfig is the top-level builder block. Today it only houses -// state-actor; future builders (e.g. geth-import, snap-sync) plug in -// alongside. +// BuilderConfig is the top-level builder block. It houses state-actor +// (materialises pre-populated datadirs) and eest_payloads (generates +// stateful EEST benchmark fixtures against such a datadir). Future +// builders plug in alongside. type BuilderConfig struct { - StateActor *StateActorConfig `yaml:"state_actor,omitempty" mapstructure:"state_actor"` + StateActor *StateActorConfig `yaml:"state_actor,omitempty" mapstructure:"state_actor"` + EESTPayloads *EESTPayloadsConfig `yaml:"eest_payloads,omitempty" mapstructure:"eest_payloads"` } // StateActorConfig configures how the state-actor binary is invoked via @@ -90,6 +92,12 @@ type BuilderConfig struct { // complementary — when both are set state-actor uses the spec and // treats target_size as a headroom budget for any further auto-fill. // +// Spec may be written either as a structured YAML mapping (so editors give it +// syntax highlighting) or as a "|" block scalar; both normalize to the YAML +// body state-actor consumes. The field is excluded from Viper decoding and +// populated by normalizeStateActorSpec from the raw YAML (Viper can't decode a +// mapping into a string, and re-parsing preserves numbers/casing/comments). +// // Config holds the per-target build parameters that can be hoisted up // to avoid repeating them. Any field set on a target overrides the // corresponding field in Config. @@ -97,7 +105,7 @@ type StateActorConfig struct { ContainerRuntime string `yaml:"container_runtime,omitempty" mapstructure:"container_runtime"` Images map[string]string `yaml:"images,omitempty" mapstructure:"images"` PullPolicy string `yaml:"pull_policy,omitempty" mapstructure:"pull_policy"` - Spec string `yaml:"spec,omitempty" mapstructure:"spec"` + Spec string `yaml:"spec,omitempty" mapstructure:"-"` SpecFile string `yaml:"spec_file,omitempty" mapstructure:"spec_file"` Config *StateActorClientDefaults `yaml:"config,omitempty" mapstructure:"config"` Targets []StateActorTarget `yaml:"targets,omitempty" mapstructure:"targets"` @@ -258,13 +266,15 @@ func (s *StateActorConfig) ImageFor(client string) string { } // stateActorSupportedClients lists the clients state-actor itself can -// materialise datadirs for. Erigon and Nimbus are intentionally absent -// (state-actor does not implement writers for them). +// materialise datadirs for. Nimbus is intentionally absent (state-actor +// does not implement a writer for it). var stateActorSupportedClients = map[string]struct{}{ "geth": {}, "reth": {}, "besu": {}, "nethermind": {}, + "ethrex": {}, + "erigon": {}, } // stateActorValidPullPolicies mirrors the pull-policy vocabulary used by @@ -276,6 +286,280 @@ var stateActorValidPullPolicies = map[string]bool{ "never": true, } +// EESTPayloadsConfig configures generation of stateful EEST benchmark +// fixtures via the `fill-stateful` command (execution-specs). See +// https://github.com/ethereum/execution-specs/pull/2637. +// +// Unlike state-actor, this builder is an orchestrator: per target it +// boots a filler EL client on a copy of a pre-populated snapshot datadir +// (e.g. produced by builder.state_actor), runs fill-stateful against the +// live client, and writes the resulting fixtures to the target's +// output_dir. The fixtures are later replayed by `benchmarkoor run`. +// +// FillImage is the container image carrying the `fill-stateful` command +// (uv + execution-specs). Config holds per-target defaults that can be +// hoisted to avoid repetition; any field set on a target wins. +type EESTPayloadsConfig struct { + ContainerRuntime string `yaml:"container_runtime,omitempty" mapstructure:"container_runtime"` + FillImage string `yaml:"fill_image,omitempty" mapstructure:"fill_image"` + // FillDockerfile, when set, makes benchmarkoor build the fill image from the + // given Dockerfile at build time (using the container runtime), instead of + // pulling a pre-built image. The built image is tagged FillImage when set, + // otherwise DefaultFillImageTag. Mutually informative with FillImage: at + // least one of the two must be provided. + FillDockerfile string `yaml:"fill_dockerfile,omitempty" mapstructure:"fill_dockerfile"` + PullPolicy string `yaml:"pull_policy,omitempty" mapstructure:"pull_policy"` + JWT string `yaml:"jwt,omitempty" mapstructure:"jwt"` + // FillCommand is the argv prefix invoked inside FillImage before the + // fill-stateful flags. Defaults to ["uv", "run", "fill-stateful"]. + FillCommand []string `yaml:"fill_command,omitempty" mapstructure:"fill_command"` + // EESTRepo / EESTRef select the execution-specs checkout used for filling. + // benchmarkoor always clones the repo at this ref into an on-disk cache at + // build time and mounts it into the fill container at /eest (the fill image + // carries only the uv/python toolchain, not the repo). This lets the EEST + // version live in config and change without rebuilding the image. Both + // default when unset: EESTRepo to the execution-specs URL, EESTRef to + // DefaultEESTRef. + EESTRepo string `yaml:"eest_repo,omitempty" mapstructure:"eest_repo"` + EESTRef string `yaml:"eest_ref,omitempty" mapstructure:"eest_ref"` + Config *EESTPayloadDefaults `yaml:"config,omitempty" mapstructure:"config"` + Targets []EESTPayloadTarget `yaml:"targets,omitempty" mapstructure:"targets"` +} + +const ( + // DefaultEESTRepo is the execution-specs repository cloned for fill-stateful + // when builder.eest_payloads.eest_repo is unset. + DefaultEESTRepo = "https://github.com/ethereum/execution-specs.git" + // DefaultEESTRef is the execution-specs ref cloned for fill-stateful when + // builder.eest_payloads.eest_ref is unset (where fill-stateful currently lives). + DefaultEESTRef = "forks/amsterdam" + // DefaultFillImageTag is the tag applied to a fill image built from + // fill_dockerfile when no explicit fill_image tag is given. + DefaultFillImageTag = "benchmarkoor-eest-fill:local" +) + +// BuildsFillImage reports whether benchmarkoor should build the fill image from +// FillDockerfile (rather than pulling a pre-built FillImage). +func (e *EESTPayloadsConfig) BuildsFillImage() bool { + return e.FillDockerfile != "" +} + +// ResolveFillImageTag returns the image reference for the fill container: the +// configured FillImage, or DefaultFillImageTag when only a Dockerfile is set. +func (e *EESTPayloadsConfig) ResolveFillImageTag() string { + if e.FillImage != "" { + return e.FillImage + } + + return DefaultFillImageTag +} + +// ResolveEESTRepo returns the configured EEST repo URL, defaulting to +// DefaultEESTRepo. +func (e *EESTPayloadsConfig) ResolveEESTRepo() string { + if e.EESTRepo != "" { + return e.EESTRepo + } + + return DefaultEESTRepo +} + +// ResolveEESTRef returns the configured EEST ref, defaulting to DefaultEESTRef. +func (e *EESTPayloadsConfig) ResolveEESTRef() string { + if e.EESTRef != "" { + return e.EESTRef + } + + return DefaultEESTRef +} + +// DefaultFillCommand is the argv prefix used to invoke fill-stateful inside +// the fill image when EESTPayloadsConfig.FillCommand is unset. +var DefaultFillCommand = []string{"uv", "run", "fill-stateful"} + +// ResolveFillCommand returns the configured fill-stateful argv prefix, or +// DefaultFillCommand when unset. +func (e *EESTPayloadsConfig) ResolveFillCommand() []string { + if len(e.FillCommand) > 0 { + return e.FillCommand + } + + return DefaultFillCommand +} + +// EESTPayloadDefaults are the per-target build parameters that may be +// hoisted to the top level under `builder.eest_payloads.config`. Every +// field is also present on EESTPayloadTarget; a non-nil/non-empty value +// on the target wins over the corresponding default. See ResolveTarget. +type EESTPayloadDefaults struct { + FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` + Fork string `yaml:"fork,omitempty" mapstructure:"fork"` + Tests []string `yaml:"tests,omitempty" mapstructure:"tests"` + Filter string `yaml:"filter,omitempty" mapstructure:"filter"` + Marker string `yaml:"marker,omitempty" mapstructure:"marker"` + AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` + AddressStubs map[string]map[string]string `yaml:"address_stubs,omitempty" mapstructure:"address_stubs"` + GasBenchmarkValues []int `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` + FixedOpcodeCount *[]float64 `yaml:"fixed_opcode_count,omitempty" mapstructure:"fixed_opcode_count"` + DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` + MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` + RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` + FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` +} + +// EESTPayloadTarget is one fixture-generation run. Identity/locator fields +// (Name, FillerClient, SourceDir, OutputDir, Genesis, GenesisForkOverride, +// GenesisEIPOverride) live exclusively on the target; the remaining fields +// mirror EESTPayloadDefaults and are resolved via ResolveTarget. +type EESTPayloadTarget struct { + Name string `yaml:"name,omitempty" mapstructure:"name"` + FillerClient string `yaml:"filler_client" mapstructure:"filler_client"` + SourceDir string `yaml:"source_dir" mapstructure:"source_dir"` + OutputDir string `yaml:"output_dir" mapstructure:"output_dir"` + // Genesis is the genesis/chainspec the filler boots from (besu/nethermind + // read their fork schedule from it). geth/erigon boot from the snapshot + // datadir instead and activate forks via --override. in + // FillerExtraArgs, so they need no Genesis. Mirrors runner client `genesis`. + Genesis string `yaml:"genesis,omitempty" mapstructure:"genesis"` + // GenesisForkOverride / GenesisEIPOverride patch the Genesis at filler boot + // to activate a fork the file doesn't schedule (e.g. amsterdam on an osaka + // snapshot), identically to the runner. GenesisForkOverride sets + // config.Time in a geth-format genesis (besu/reth/ethrex); + // GenesisEIPOverride sets params.eipTransitionTimestamp in a parity + // chainspec (nethermind). + GenesisForkOverride map[string]uint64 `yaml:"genesis_fork_override,omitempty" mapstructure:"genesis_fork_override"` + GenesisEIPOverride *GenesisEIPOverride `yaml:"genesis_eip_override,omitempty" mapstructure:"genesis_eip_override"` + Force bool `yaml:"force,omitempty" mapstructure:"force"` + + // Hoistable fields (mirror EESTPayloadDefaults): a non-empty/non-nil value + // here wins over the corresponding builder.eest_payloads.config default. + FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` + Fork string `yaml:"fork,omitempty" mapstructure:"fork"` + // Tests are pytest paths inside the fill image, e.g. tests/benchmark/compute. + Tests []string `yaml:"tests,omitempty" mapstructure:"tests"` + // Filter is a pytest -k expression (substring/node-id selection). + Filter string `yaml:"filter,omitempty" mapstructure:"filter"` + // Marker is a pytest -m marker expression, orthogonal to Filter's -k. e.g. + // "repricing" to select the gas-repricing reference benchmarks, or + // "not repricing" to exclude them. + Marker string `yaml:"marker,omitempty" mapstructure:"marker"` + // AddressStubsFile points at a JSON file of named address stubs; AddressStubs + // defines the same mapping inline (the builder materializes it to a temp JSON + // file). They are mutually exclusive. Each stub maps a symbolic name to an + // arbitrary set of string fields (e.g. addr, pkey) that fill-stateful resolves + // against the snapshot's pre-deployed state via --address-stubs. When a target + // sets neither, both are hoisted as a unit from the config defaults. + AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` + AddressStubs map[string]map[string]string `yaml:"address_stubs,omitempty" mapstructure:"address_stubs"` + GasBenchmarkValues []int `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` + FixedOpcodeCount *[]float64 `yaml:"fixed_opcode_count,omitempty" mapstructure:"fixed_opcode_count"` + DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` + MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` + RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` + FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` +} + +// ResolveTarget returns a copy of the i-th target with any unset hoistable +// fields filled in from EESTPayloadsConfig.Config. Identity/locator fields +// are never touched. Per-target value wins when set (non-nil for pointer +// types, non-empty for strings/slices); otherwise the value from Config is +// used. When Config is nil, the target is returned unchanged. +func (e *EESTPayloadsConfig) ResolveTarget(i int) EESTPayloadTarget { + t := e.Targets[i] + if e.Config == nil { + return t + } + + g := e.Config + + if t.FillerImage == "" { + t.FillerImage = g.FillerImage + } + + if t.Fork == "" { + t.Fork = g.Fork + } + + if len(t.Tests) == 0 { + t.Tests = g.Tests + } + + if t.Filter == "" { + t.Filter = g.Filter + } + + if t.Marker == "" { + t.Marker = g.Marker + } + + // Hoist the address-stubs pair as a unit: a target that sets either form + // keeps its own and inherits neither, preserving their mutual exclusion. + if len(t.AddressStubs) == 0 && t.AddressStubsFile == "" { + t.AddressStubs = g.AddressStubs + t.AddressStubsFile = g.AddressStubsFile + } + + if len(t.GasBenchmarkValues) == 0 { + t.GasBenchmarkValues = g.GasBenchmarkValues + } + + if t.FixedOpcodeCount == nil { + t.FixedOpcodeCount = g.FixedOpcodeCount + } + + if t.DataDirMethod == "" { + t.DataDirMethod = g.DataDirMethod + } + + if t.MaxGasPerTest == nil { + t.MaxGasPerTest = g.MaxGasPerTest + } + + if t.RPCSeedKey == "" { + t.RPCSeedKey = g.RPCSeedKey + } + + if len(t.FillerExtraArgs) == 0 { + t.FillerExtraArgs = g.FillerExtraArgs + } + + return t +} + +// EffectiveName returns the target's user-facing name, defaulting to the +// filler client when Name was not set — matching the `--target` filter +// behaviour in the build command. +func (t *EESTPayloadTarget) EffectiveName() string { + if t.Name != "" { + return t.Name + } + + return t.FillerClient +} + +// eestFillerSupportedClients lists the clients benchmarkoor knows how to boot as +// the fill-stateful filler (see fillerCommand in pkg/builder). fill-stateful +// forces use_testing_build_block=True, so a filler MUST implement the +// testing_buildBlockV1 RPC and debug_setHead (the per-test chain rewind). +// +// Status as of this writing: +// - geth: fully works (ethpandaops/geth implements both). +// - besu: plumbed but blocked upstream — testing_buildBlockV1 (TESTING +// namespace, besu-eth/besu#9838) + debug_setHead both respond, but +// besu's self-built block fails its own engine_newPayloadV4 with a +// World-State-Root mismatch. Kept as scaffolding. +// - nethermind: plumbed but blocked upstream — testing_buildBlockV1 works +// (Testing module, special image) but debug_setHead is unimplemented and +// its receipt trips EEST's strict model. Kept as scaffolding. +// +// besu/nethermind stay listed so configs can experiment with them once the +// upstream gaps close; their example targets are commented out. +var eestFillerSupportedClients = map[string]struct{}{ + "geth": {}, + "besu": {}, + "nethermind": {}, +} + // RunnerConfig contains all run-specific configuration settings. type RunnerConfig struct { ContainerRuntime string `yaml:"container_runtime,omitempty" mapstructure:"container_runtime"` @@ -403,16 +687,44 @@ type MetadataConfig struct { // GlobalConfig contains global application settings. type GlobalConfig struct { LogLevel string `yaml:"log_level" mapstructure:"log_level"` + // Env declares config-local variables available to ${VAR} / ${VAR:-default} + // substitution throughout the file, as a per-config default for an env var of + // the same name. A real shell env var of that name still wins, so configs stay + // overridable. Consumed at load time (see envExpander); the parsed map is not + // otherwise used and — unlike the substitution source — is Viper-lowercased. + Env map[string]string `yaml:"env,omitempty" mapstructure:"env"` + Directories GlobalDirectoriesConfig `yaml:"directories,omitempty" mapstructure:"directories"` } -// DirectoriesConfig contains directory path configurations. +// GlobalDirectoriesConfig contains directory paths shared across the build and +// run commands. +type GlobalDirectoriesConfig struct { + // CacheDir is the on-disk cache shared by both commands: executor git/archive + // clones (run) and the EEST repo clone (build). If empty, defaults to + // ~/.cache/benchmarkoor. + CacheDir string `yaml:"cachedir,omitempty" mapstructure:"cachedir"` +} + +// DirectoriesConfig contains runner-specific directory path configurations. type DirectoriesConfig struct { // TmpDataDir is the directory for temporary datadir copies. // If empty, uses the system default temp directory. TmpDataDir string `yaml:"tmp_datadir,omitempty" mapstructure:"tmp_datadir"` - // TmpCacheDir is the directory for executor cache (git clones, etc). - // If empty, uses ~/.cache/benchmarkoor. - TmpCacheDir string `yaml:"tmp_cachedir,omitempty" mapstructure:"tmp_cachedir"` +} + +// ResolveCacheDir returns the configured global cache directory, defaulting to +// ~/.cache/benchmarkoor when unset. +func (c *Config) ResolveCacheDir() (string, error) { + if c.Global.Directories.CacheDir != "" { + return c.Global.Directories.CacheDir, nil + } + + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("getting home directory for cache dir: %w", err) + } + + return filepath.Join(home, ".cache", "benchmarkoor"), nil } // BenchmarkConfig contains benchmark-specific settings. @@ -551,19 +863,20 @@ func (e *EESTFixturesSource) validate() error { // Validate local dir mode. if hasLocalDir { if e.LocalFixturesDir == "" { - return fmt.Errorf("eest_fixtures: local_fixtures_dir is required when local_genesis_dir is set") - } - - if e.LocalGenesisDir == "" { - return fmt.Errorf("eest_fixtures: local_genesis_dir is required when local_fixtures_dir is set") + return fmt.Errorf("eest_fixtures: local_fixtures_dir is required for local directory mode") } if err := validateDirExists(e.LocalFixturesDir, "eest_fixtures.local_fixtures_dir"); err != nil { return err } - if err := validateDirExists(e.LocalGenesisDir, "eest_fixtures.local_genesis_dir"); err != nil { - return err + // local_genesis_dir is optional: stateful-engine fixtures boot from a + // pre-populated snapshot datadir (configured via runner.client.datadirs) + // and carry no genesis. Validate it only when provided. + if e.LocalGenesisDir != "" { + if err := validateDirExists(e.LocalGenesisDir, "eest_fixtures.local_genesis_dir"); err != nil { + return err + } } } @@ -684,6 +997,19 @@ func (s *SourceConfig) IsConfigured() bool { // DefaultContainerDir is the default container mount path for data directories. const DefaultContainerDir = "/data" +// GenesisEIPOverride activates a set of EIPs at a given timestamp in a +// parity/nethermind-format chainspec (which schedules forks per-EIP rather than +// by fork name). It is the parity-format counterpart of GenesisForkOverride: the +// devnet-specific EIP list lives in config and benchmarkoor patches the +// chainspec at boot. +type GenesisEIPOverride struct { + // Timestamp is the activation time (unix seconds) applied to every listed EIP. + Timestamp uint64 `yaml:"timestamp" mapstructure:"timestamp"` + // EIPs are the EIP numbers to activate, e.g. [7928, 8037]. Each becomes a + // params.eipTransitionTimestamp entry. + EIPs []uint64 `yaml:"eips" mapstructure:"eips"` +} + // DataDirConfig configures a pre-populated data directory for a client. type DataDirConfig struct { SourceDir string `yaml:"source_dir" json:"source_dir" mapstructure:"source_dir"` @@ -1007,6 +1333,8 @@ type ClientInstance struct { Restart string `yaml:"restart,omitempty" mapstructure:"restart"` Environment map[string]string `yaml:"environment,omitempty" mapstructure:"environment"` Genesis string `yaml:"genesis,omitempty" mapstructure:"genesis"` + GenesisForkOverride map[string]uint64 `yaml:"genesis_fork_override,omitempty" mapstructure:"genesis_fork_override"` + GenesisEIPOverride *GenesisEIPOverride `yaml:"genesis_eip_override,omitempty" mapstructure:"genesis_eip_override"` DataDir *DataDirConfig `yaml:"datadir,omitempty" mapstructure:"datadir"` DropMemoryCaches string `yaml:"drop_memory_caches,omitempty" mapstructure:"drop_memory_caches"` RollbackStrategy string `yaml:"rollback_strategy,omitempty" mapstructure:"rollback_strategy"` @@ -1040,6 +1368,64 @@ func expandEnvWithDefaults(s string) string { return os.Getenv(s) } +// rawGlobalEnv is a minimal struct used to read global.env from the raw +// (pre-expansion) YAML. Read this way rather than from the parsed Config so the +// keys keep their original casing (Viper lowercases all map keys), since they +// are used as case-sensitive ${VAR} substitution names. +type rawGlobalEnv struct { + Global struct { + Env map[string]string `yaml:"env"` + } `yaml:"global"` +} + +// collectGlobalEnv parses global.env from each raw config and merges them +// (later files win). A value may itself reference the shell environment (e.g. +// "${BASE:-/tmp}/state-actor"), which is expanded here; values do not see one +// another. +func collectGlobalEnv(contents []string) map[string]string { + env := make(map[string]string) + + for _, content := range contents { + var rg rawGlobalEnv + if err := yaml.Unmarshal([]byte(content), &rg); err != nil { + continue + } + + for k, val := range rg.Global.Env { + env[k] = os.Expand(val, expandEnvWithDefaults) + } + } + + return env +} + +// envExpander returns the os.Expand mapping used for ${VAR} / ${VAR:-default} +// substitution across config files. Resolution order is: the shell environment, +// then global.env from the config, then the inline default. Keeping the shell +// first means global.env acts as a per-config default that an env var can still +// override (e.g. in CI). +func envExpander(contents []string) func(string) string { + globalEnv := collectGlobalEnv(contents) + + return func(s string) string { + name, defaultVal, hasDefault := strings.Cut(s, ":-") + + if v := os.Getenv(name); v != "" { + return v + } + + if v, ok := globalEnv[name]; ok && v != "" { + return v + } + + if hasDefault { + return defaultVal + } + + return "" + } +} + // Load reads and parses configuration files from the given paths. // When multiple paths are provided, configs are merged in order (later values override earlier). // Environment variables can be substituted in config values using ${VAR}, $VAR, or @@ -1060,27 +1446,37 @@ func Load(paths ...string) (*Config, error) { v.SetConfigType("yaml") - // Load and merge configs in order, collecting expanded YAML for - // post-processing (Viper lowercases map keys, so we re-parse to - // restore original casing for environment variables). - rawYAMLs := make([]string, 0, len(paths)) + // Read every file up front so global.env (which may live in any of them) is + // known before we expand ${VAR} references. + contents := make([]string, 0, len(paths)) - for i, path := range paths { + for _, path := range paths { content, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("reading config file %q: %w", path, err) } - expanded := os.Expand(string(content), expandEnvWithDefaults) + contents = append(contents, string(content)) + } + + expand := envExpander(contents) + + // Load and merge configs in order, collecting expanded YAML for + // post-processing (Viper lowercases map keys, so we re-parse to + // restore original casing for environment variables). + rawYAMLs := make([]string, 0, len(paths)) + + for i, content := range contents { + expanded := os.Expand(content, expand) rawYAMLs = append(rawYAMLs, expanded) if i == 0 { if err := v.ReadConfig(strings.NewReader(expanded)); err != nil { - return nil, fmt.Errorf("parsing config %q: %w", path, err) + return nil, fmt.Errorf("parsing config %q: %w", paths[i], err) } } else { if err := v.MergeConfig(strings.NewReader(expanded)); err != nil { - return nil, fmt.Errorf("merging config %q: %w", path, err) + return nil, fmt.Errorf("merging config %q: %w", paths[i], err) } } } @@ -1101,6 +1497,8 @@ func Load(paths ...string) (*Config, error) { } restoreEnvironmentKeyCasing(&cfg, rawYAMLs) + restoreAddressStubsKeyCasing(&cfg, rawYAMLs) + normalizeStateActorSpec(&cfg, rawYAMLs) cfg.applyDefaults() @@ -1113,6 +1511,7 @@ func bindEnvKeys(v *viper.Viper) { keys := []string{ // Global settings "global.log_level", + "global.directories.cachedir", // Runner settings "runner.container_runtime", "runner.client_logs_to_stdout", @@ -1182,6 +1581,8 @@ func bindEnvKeys(v *viper.Viper) { // Builder settings "builder.state_actor.container_runtime", "builder.state_actor.pull_policy", + "builder.eest_payloads.container_runtime", + "builder.eest_payloads.pull_policy", } for _, key := range keys { @@ -1310,6 +1711,19 @@ func (c *Config) applyDefaults() { c.Builder.StateActor.PullPolicy = DefaultPullPolicy } } + + // Apply builder.eest_payloads defaults. ContainerRuntime is left empty + // so GetEESTPayloadsContainerRuntime can fall back at call time; JWT + // defaults to DefaultJWT so the filler client and fill-stateful share it. + if c.Builder != nil && c.Builder.EESTPayloads != nil { + if c.Builder.EESTPayloads.PullPolicy == "" { + c.Builder.EESTPayloads.PullPolicy = DefaultPullPolicy + } + + if c.Builder.EESTPayloads.JWT == "" { + c.Builder.EESTPayloads.JWT = DefaultJWT + } + } } // GetStateActorContainerRuntime returns the container runtime to use for @@ -1323,6 +1737,17 @@ func (c *Config) GetStateActorContainerRuntime() string { return c.GetContainerRuntime() } +// GetEESTPayloadsContainerRuntime returns the container runtime to use for +// eest_payloads builds. Falls back to the runner's runtime when the builder +// block does not override it. +func (c *Config) GetEESTPayloadsContainerRuntime() string { + if c.Builder != nil && c.Builder.EESTPayloads != nil && c.Builder.EESTPayloads.ContainerRuntime != "" { + return c.Builder.EESTPayloads.ContainerRuntime + } + + return c.GetContainerRuntime() +} + // ValidateOpts controls optional validation behavior. type ValidateOpts struct { // ActiveInstanceIDs limits validation to instances with these IDs. @@ -1378,6 +1803,18 @@ func (c *Config) Validate(opts ...ValidateOpts) error { return fmt.Errorf("instance %q: unknown client type %q", instance.ID, instance.Client) } + // genesis_fork_override (geth-format Time) and genesis_eip_override + // (parity eipTransitionTimestamp) patch different genesis formats, so a + // single instance can set at most one — mirrors the eest_payloads target + // check. (Without this, the contradiction only surfaces at boot when the + // override apply rejects the genesis format.) + if len(instance.GenesisForkOverride) > 0 && instance.GenesisEIPOverride != nil { + return fmt.Errorf( + "instance %q: genesis_fork_override and genesis_eip_override are mutually exclusive", + instance.ID, + ) + } + // Validate instance-level datadir (skip if not in active set). if instance.DataDir != nil { if len(opt.ActiveInstanceIDs) == 0 { @@ -1565,11 +2002,24 @@ func (c *Config) ValidateBuilder() error { return c.validateBuilder() } -// validateBuilder enforces the builder.state_actor rules: supported +// validateBuilder validates each configured builder block. +func (c *Config) validateBuilder() error { + if c.Builder == nil { + return nil + } + + if err := c.validateStateActor(); err != nil { + return err + } + + return c.validateEESTPayloads() +} + +// validateStateActor enforces the builder.state_actor rules: supported // clients, single-source-of-truth target_size XOR spec, archive/binary-trie // applicability, group_depth range, image resolvability, and uniqueness of // target names and output_dirs. -func (c *Config) validateBuilder() error { +func (c *Config) validateStateActor() error { if c.Builder == nil || c.Builder.StateActor == nil { return nil } @@ -1615,7 +2065,7 @@ func (c *Config) validateBuilder() error { if _, ok := stateActorSupportedClients[t.Client]; !ok { return fmt.Errorf( "%s.client: %q is not supported by state-actor "+ - "(must be geth, reth, besu, or nethermind)", + "(must be geth, reth, besu, nethermind, ethrex, or erigon)", prefix, t.Client, ) } @@ -1701,6 +2151,241 @@ func (c *Config) validateBuilder() error { return nil } +// validateEESTPayloads enforces the builder.eest_payloads rules: a +// configured fill_image, supported filler clients, required locator fields +// (source_dir/output_dir/tests/fork), valid datadir method and +// gas-benchmark values, absolute paths, and uniqueness of target names and +// output_dirs. Existence of source_dir/genesis_file/address_stubs_file is +// checked at build time, not here — a state-actor target earlier in the +// same config may still need to produce them. +func (c *Config) validateEESTPayloads() error { + if c.Builder == nil || c.Builder.EESTPayloads == nil { + return nil + } + + ep := c.Builder.EESTPayloads + + if !validContainerRuntimes[ep.ContainerRuntime] { + return fmt.Errorf( + "builder.eest_payloads.container_runtime: invalid value %q "+ + "(must be \"docker\" or \"podman\")", ep.ContainerRuntime, + ) + } + + if !stateActorValidPullPolicies[ep.PullPolicy] { + return fmt.Errorf( + "builder.eest_payloads.pull_policy: invalid value %q "+ + "(must be \"always\", \"if-not-present\", or \"never\")", + ep.PullPolicy, + ) + } + + if ep.FillImage == "" && ep.FillDockerfile == "" { + return fmt.Errorf( + "builder.eest_payloads: one of fill_image (a pre-built image) or " + + "fill_dockerfile (built by benchmarkoor) is required", + ) + } + + // Reject a missing Dockerfile at config time so typos surface early. + // Relative paths are resolved against the working directory. + if ep.FillDockerfile != "" { + if _, err := os.Stat(ep.FillDockerfile); err != nil { + return fmt.Errorf("builder.eest_payloads.fill_dockerfile: %w", err) + } + } + + seenOutputs := make(map[string]int, len(ep.Targets)) + seenNames := make(map[string]int, len(ep.Targets)) + + for i := range ep.Targets { + t := ep.ResolveTarget(i) + prefix := fmt.Sprintf("builder.eest_payloads.targets[%d]", i) + + if _, ok := eestFillerSupportedClients[t.FillerClient]; !ok { + return fmt.Errorf( + "%s.filler_client: %q cannot act as the fill-stateful filler "+ + "(supported: geth, besu, nethermind)", + prefix, t.FillerClient, + ) + } + + name := t.EffectiveName() + if prev, dup := seenNames[name]; dup { + return fmt.Errorf( + "%s: name %q duplicates targets[%d] (set an explicit name to disambiguate)", + prefix, name, prev, + ) + } + + seenNames[name] = i + + if err := validateEESTPayloadPaths(&t, prefix, seenOutputs, i); err != nil { + return err + } + + if len(t.Tests) == 0 { + return fmt.Errorf( + "%s.tests is required (at least one pytest path, e.g. tests/benchmark/compute)", + prefix, + ) + } + + if t.Fork == "" { + return fmt.Errorf( + "%s.fork is required (set it on the target or builder.eest_payloads.config.fork)", + prefix, + ) + } + + if t.FillerImage == "" { + return fmt.Errorf( + "%s.filler_image is required (e.g. ethpandaops/geth:master)", prefix, + ) + } + + if !validDataDirMethods[t.DataDirMethod] { + return fmt.Errorf( + "%s.datadir_method: invalid value %q "+ + "(must be copy, overlayfs, fuse-overlayfs, zfs, direct, or schelk)", + prefix, t.DataDirMethod, + ) + } + + if err := validateGasBenchmarkValues(t.GasBenchmarkValues, prefix); err != nil { + return err + } + + if err := validateFixedOpcodeCount(t.FixedOpcodeCount, prefix); err != nil { + return err + } + + if len(t.GasBenchmarkValues) > 0 && t.FixedOpcodeCount != nil { + return fmt.Errorf( + "%s: gas_benchmark_values and fixed_opcode_count are mutually exclusive "+ + "(fill-stateful rejects both)", prefix, + ) + } + } + + return nil +} + +// validateEESTPayloadPaths checks output_dir / genesis / +// address_stubs_file are absolute and output_dir is unique. +func validateEESTPayloadPaths(t *EESTPayloadTarget, prefix string, seenOutputs map[string]int, i int) error { + if t.SourceDir == "" { + return fmt.Errorf("%s.source_dir is required", prefix) + } + + if !filepath.IsAbs(t.SourceDir) { + return fmt.Errorf("%s.source_dir must be an absolute path, got %q", prefix, t.SourceDir) + } + + if t.OutputDir == "" { + return fmt.Errorf("%s.output_dir is required", prefix) + } + + if !filepath.IsAbs(t.OutputDir) { + return fmt.Errorf("%s.output_dir must be an absolute path, got %q", prefix, t.OutputDir) + } + + if prev, dup := seenOutputs[t.OutputDir]; dup { + return fmt.Errorf( + "%s.output_dir %q duplicates targets[%d].output_dir", prefix, t.OutputDir, prev, + ) + } + + seenOutputs[t.OutputDir] = i + + if t.Genesis != "" && !filepath.IsAbs(t.Genesis) { + return fmt.Errorf("%s.genesis must be an absolute path, got %q", prefix, t.Genesis) + } + + // genesis_fork_override / genesis_eip_override patch the boot genesis, so + // they require one. (geth/erigon fillers boot from the datadir and use + // --override. in filler_extra_args instead.) + if len(t.GenesisForkOverride) > 0 && t.Genesis == "" { + return fmt.Errorf("%s.genesis_fork_override requires genesis", prefix) + } + + if t.GenesisEIPOverride != nil && len(t.GenesisEIPOverride.EIPs) > 0 && t.Genesis == "" { + return fmt.Errorf("%s.genesis_eip_override requires genesis", prefix) + } + + if len(t.GenesisForkOverride) > 0 && t.GenesisEIPOverride != nil { + return fmt.Errorf( + "%s: genesis_fork_override and genesis_eip_override are mutually exclusive", prefix, + ) + } + + if t.AddressStubsFile != "" && !filepath.IsAbs(t.AddressStubsFile) { + return fmt.Errorf( + "%s.address_stubs_file must be an absolute path, got %q", prefix, t.AddressStubsFile, + ) + } + + if t.AddressStubsFile != "" && len(t.AddressStubs) > 0 { + return fmt.Errorf( + "%s: address_stubs_file and address_stubs are mutually exclusive", prefix, + ) + } + + for name, stub := range t.AddressStubs { + if stub["addr"] == "" { + return fmt.Errorf("%s.address_stubs[%q].addr is required", prefix, name) + } + } + + return nil +} + +// validateGasBenchmarkValues checks a list of positive integers (millions of +// gas), e.g. [10, 30]. An empty list is allowed. +func validateGasBenchmarkValues(values []int, prefix string) error { + for _, v := range values { + if v < 1 { + return fmt.Errorf( + "%s.gas_benchmark_values: %d is not a positive integer (millions of gas)", prefix, v, + ) + } + } + + return nil +} + +// validateFixedOpcodeCount checks a list of positive numbers (thousands of +// opcodes), e.g. [0.5, 1, 2]. A nil pointer (unset) is allowed, as is a +// non-nil empty list (passes the bare --fixed-opcode-count flag, which uses +// the fill image's .fixed_opcode_counts.json default). +func validateFixedOpcodeCount(values *[]float64, prefix string) error { + if values == nil { + return nil + } + + for _, v := range *values { + if v <= 0 { + return fmt.Errorf( + "%s.fixed_opcode_count: %v is not a positive number (thousands of opcodes)", prefix, v, + ) + } + } + + return nil +} + +// validDataDirMethods mirrors the datadir.method vocabulary accepted by +// pkg/datadir.NewProvider and DataDirConfig validation. +var validDataDirMethods = map[string]bool{ + "": true, // unset → copy + "copy": true, + "overlayfs": true, + "fuse-overlayfs": true, + "zfs": true, + "direct": true, + "schelk": true, +} + // validateLiveReporting checks the runner.live_reporting config when enabled. func (c *Config) validateLiveReporting() error { lr := c.Runner.LiveReporting @@ -3241,3 +3926,144 @@ func restoreEnvironmentKeyCasing(cfg *Config, rawYAMLs []string) { } } } + +// rawEESTStubs holds just the inline address_stubs map for one level of the +// eest_payloads config (the global config block or a single target). +type rawEESTStubs struct { + AddressStubs map[string]map[string]string `yaml:"address_stubs"` +} + +// rawEESTBuilderConfig is a minimal struct used to re-parse inline +// address_stubs maps, whose stub-name keys Viper lowercases (it is +// case-insensitive). EEST resolves stub names by exact match, so the +// original casing must be restored — both at the global config level +// (hoisted into targets via ResolveTarget) and per target. +type rawEESTBuilderConfig struct { + Builder struct { + EESTPayloads struct { + Config *rawEESTStubs `yaml:"config"` + Targets []rawEESTStubs `yaml:"targets"` + } `yaml:"eest_payloads"` + } `yaml:"builder"` +} + +// restoreAddressStubsKeyCasing re-parses the raw YAML to recover the original +// casing of inline address_stubs stub-name keys that Viper lowercased. Viper +// replaces (rather than appends) list values on merge, so the last config file +// that defines targets (or the config block) wins — mirror that with a +// last-wins positional match. +func restoreAddressStubsKeyCasing(cfg *Config, rawYAMLs []string) { + if cfg.Builder == nil || cfg.Builder.EESTPayloads == nil { + return + } + + ep := cfg.Builder.EESTPayloads + + // configStubs accumulates the config-block stubs across all files (later + // files win per key), mirroring how Viper deep-merges the config map — so a + // config.address_stubs set in an earlier file isn't dropped when a later + // file only touches some other config field. Targets, by contrast, are + // replaced wholesale on merge, so the last file's list wins (see below). + configStubs := make(map[string]map[string]string) + + var rawTargets []rawEESTStubs + + for _, raw := range rawYAMLs { + var parsed rawEESTBuilderConfig + if err := yaml.Unmarshal([]byte(raw), &parsed); err != nil { + continue + } + + if c := parsed.Builder.EESTPayloads.Config; c != nil { + for name, stub := range c.AddressStubs { + configStubs[name] = stub + } + } + + if len(parsed.Builder.EESTPayloads.Targets) > 0 { + rawTargets = parsed.Builder.EESTPayloads.Targets + } + } + + // Global config defaults (hoisted into targets at resolve time). + if ep.Config != nil && len(configStubs) > 0 { + ep.Config.AddressStubs = configStubs + } + + // Per-target stubs. Only restore when the winning file's target list aligns + // 1:1 with the resolved config; otherwise leave the (lowercased) keys + // untouched rather than risk mismatching stubs onto the wrong target. + if len(rawTargets) != len(ep.Targets) { + return + } + + for i := range ep.Targets { + if len(rawTargets[i].AddressStubs) > 0 { + ep.Targets[i].AddressStubs = rawTargets[i].AddressStubs + } + } +} + +// normalizeStateActorSpec resolves builder.state_actor.spec from the raw YAML +// into a YAML string body. The field is excluded from Viper decoding so it can +// be authored either as a structured mapping (for editor syntax highlighting) +// or as a "|" block scalar — both normalize to the body state-actor consumes. +// Re-parsing the raw YAML preserves number formatting, value casing and +// comments that a Viper round-trip would lose. Last file with a spec wins (it +// is a scalar override, not a merged map). +func normalizeStateActorSpec(cfg *Config, rawYAMLs []string) { + if cfg.Builder == nil || cfg.Builder.StateActor == nil { + return + } + + for _, raw := range rawYAMLs { + var doc yaml.Node + if err := yaml.Unmarshal([]byte(raw), &doc); err != nil || len(doc.Content) == 0 { + continue + } + + spec := yamlMapValue(yamlMapValue(yamlMapValue(doc.Content[0], "builder"), "state_actor"), "spec") + if spec == nil { + continue + } + + body, err := stateActorSpecBody(spec) + if err != nil { + continue + } + + cfg.Builder.StateActor.Spec = body + } +} + +// stateActorSpecBody serializes a spec node to the YAML body state-actor reads: +// a scalar (a "|" block) yields its string content verbatim; a mapping is +// re-marshaled to YAML. +func stateActorSpecBody(node *yaml.Node) (string, error) { + if node.Kind == yaml.ScalarNode { + return node.Value, nil + } + + out, err := yaml.Marshal(node) + if err != nil { + return "", err + } + + return string(out), nil +} + +// yamlMapValue returns the value node for key in a YAML mapping node, or nil +// when the node is not a mapping or the key is absent. +func yamlMapValue(m *yaml.Node, key string) *yaml.Node { + if m == nil || m.Kind != yaml.MappingNode { + return nil + } + + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1] + } + } + + return nil +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 934cb1452..125d35d7d 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -13,6 +13,260 @@ import ( "github.com/stretchr/testify/require" ) +func TestLoad_InlineAddressStubsKeyCasing(t *testing.T) { + // Viper is case-insensitive and lowercases all map keys; EEST resolves stub + // names by exact match, so Load must restore the original casing. + configContent := ` +builder: + eest_payloads: + fill_image: fill:latest + targets: + - name: geth-stateful + filler_client: geth + filler_image: ethpandaops/geth:master + source_dir: /snap + output_dir: /out + fork: Amsterdam + tests: + - tests/benchmark/stateful/bloatnet + address_stubs: + bloated_EOA_10GB: + addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" + pkey: "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" +` + + configPath := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0o644)) + + cfg, err := Load(configPath) + require.NoError(t, err) + + stubs := cfg.Builder.EESTPayloads.Targets[0].AddressStubs + require.Contains(t, stubs, "bloated_EOA_10GB", "stub-name casing must survive Viper lowercasing") + assert.Equal(t, "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04", stubs["bloated_EOA_10GB"]["addr"]) + assert.Equal(t, "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e", stubs["bloated_EOA_10GB"]["pkey"]) +} + +func TestLoad_GlobalAddressStubsHoistAndCasing(t *testing.T) { + // address_stubs defined once under config: must keep its key casing and be + // hoisted into a target that sets none of its own. + configContent := ` +builder: + eest_payloads: + fill_image: fill:latest + config: + fork: Amsterdam + datadir_method: copy + tests: + - tests/benchmark/stateful/bloatnet + filter: "not erc20" + address_stubs: + bloated_EOA_10GB: + addr: "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04" + targets: + - name: geth-stateful + filler_client: geth + filler_image: ethpandaops/geth:master + source_dir: /snap + output_dir: /out +` + + configPath := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0o644)) + + cfg, err := Load(configPath) + require.NoError(t, err) + + // Global config block keeps the original stub-name casing. + require.Contains(t, cfg.Builder.EESTPayloads.Config.AddressStubs, "bloated_EOA_10GB") + + // And it (plus tests/filter) is hoisted into the bare target. + resolved := cfg.Builder.EESTPayloads.ResolveTarget(0) + assert.Equal(t, []string{"tests/benchmark/stateful/bloatnet"}, resolved.Tests) + assert.Equal(t, "not erc20", resolved.Filter) + require.Contains(t, resolved.AddressStubs, "bloated_EOA_10GB") + assert.Equal(t, "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04", resolved.AddressStubs["bloated_EOA_10GB"]["addr"]) +} + +func TestValidate_InstanceGenesisOverrideMutualExclusion(t *testing.T) { + forkOverride := map[string]uint64{"amsterdam": 1} + eipOverride := &GenesisEIPOverride{Timestamp: 1, EIPs: []uint64{7928}} + + mkCfg := func(inst ClientInstance) *Config { + return &Config{Runner: RunnerConfig{Instances: []ClientInstance{inst}}} + } + + t.Run("both set is rejected", func(t *testing.T) { + err := mkCfg(ClientInstance{ + ID: "geth", Client: "geth", + GenesisForkOverride: forkOverride, + GenesisEIPOverride: eipOverride, + }).Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "genesis_fork_override and genesis_eip_override are mutually exclusive") + }) + + t.Run("only fork override does not trip the check", func(t *testing.T) { + // Validate may still fail for unrelated reasons (no test source, etc.) — + // just assert it isn't the mutual-exclusion error. + err := mkCfg(ClientInstance{ + ID: "geth", Client: "geth", GenesisForkOverride: forkOverride, + }).Validate() + if err != nil { + assert.NotContains(t, err.Error(), "mutually exclusive") + } + }) + + t.Run("only eip override does not trip the check", func(t *testing.T) { + err := mkCfg(ClientInstance{ + ID: "nethermind", Client: "nethermind", GenesisEIPOverride: eipOverride, + }).Validate() + if err != nil { + assert.NotContains(t, err.Error(), "mutually exclusive") + } + }) +} + +func TestLoad_StateActorSpec(t *testing.T) { + dir := t.TempDir() + + write := func(name, body string) string { + p := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(p, []byte(body), 0o644)) + + return p + } + + t.Run("structured mapping materializes to the YAML body with number fidelity", func(t *testing.T) { + cfg, err := Load(write("structured.yaml", ` +builder: + state_actor: + images: { geth: img } + config: { target_size: 1GB } + spec: + entities: + - kind: eoa + name: bloated + approximate_size_bytes: 2_000_000_000 + - kind: contract + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + targets: + - { client: geth, output_dir: /o } +`)) + require.NoError(t, err) + require.NoError(t, cfg.validateBuilder()) + + kind, body := cfg.Builder.StateActor.ResolveSpec() + assert.Equal(t, StateActorSpecInline, kind) + assert.Contains(t, body, "entities:") + // Bare integers and mixed-case hex round-trip exactly (no float coercion, + // no lowercasing) because the spec is re-parsed from the raw YAML. + assert.Contains(t, body, "2_000_000_000") + assert.Contains(t, body, "0x4e59b44847b379578588920cA78FbF26c0B4956C") + }) + + t.Run("block-scalar spec still works (back-compat)", func(t *testing.T) { + cfg, err := Load(write("scalar.yaml", "builder:\n"+ + " state_actor:\n"+ + " images: { geth: img }\n"+ + " config: { target_size: 1GB }\n"+ + " spec: |\n"+ + " entities:\n"+ + " - kind: eoa\n"+ + " name: legacy\n"+ + " targets:\n"+ + " - { client: geth, output_dir: /o }\n")) + require.NoError(t, err) + + kind, body := cfg.Builder.StateActor.ResolveSpec() + assert.Equal(t, StateActorSpecInline, kind) + assert.Contains(t, body, "name: legacy") + }) +} + +func TestLoad_MultiFileConfigStubCasing(t *testing.T) { + // Viper deep-merges the eest_payloads.config map across --config files. The + // stub-name casing restore must accumulate config.address_stubs from every + // file, not just the last one — otherwise a config.address_stubs defined in + // an earlier file keeps its Viper-lowercased keys when a later file only + // touches a different config field. + dir := t.TempDir() + + base := filepath.Join(dir, "base.yaml") + require.NoError(t, os.WriteFile(base, []byte(` +builder: + eest_payloads: + fill_image: fill:latest + config: + address_stubs: + bloated_EOA_10GB: { addr: "0xabc" } + targets: + - name: t + filler_client: geth + filler_image: g + source_dir: /s + output_dir: /o + fork: Osaka + tests: [x] +`), 0o644)) + + // Merged last; sets a different config field, no address_stubs. + override := filepath.Join(dir, "override.yaml") + require.NoError(t, os.WriteFile(override, []byte(` +builder: + eest_payloads: + config: + fork: Prague +`), 0o644)) + + cfg, err := Load(base, override) + require.NoError(t, err) + + stubs := cfg.Builder.EESTPayloads.Config.AddressStubs + require.Contains(t, stubs, "bloated_EOA_10GB", + "config.address_stubs casing must survive a multi-file merge that touches other config fields") + assert.Equal(t, "Prague", cfg.Builder.EESTPayloads.Config.Fork, "later file's fork still wins") +} + +func TestLoad_GlobalEnv(t *testing.T) { + configContent := ` +global: + log_level: info + env: + STATE_DIR: /tmp/bench/state-actor/simple-amsterdam-compute + NESTED: ${BASE_DIR:-/srv}/fixtures +runner: + container_network: ${STATE_DIR} + benchmark: + results_dir: ${NESTED} + tests: + filter: ${MISSING:-fallback} +` + configPath := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(configPath, []byte(configContent), 0o644)) + + t.Run("global.env supplies ${VAR}; defaults and nesting work", func(t *testing.T) { + cfg, err := Load(configPath) + require.NoError(t, err) + + // global.env value substituted where no inline default is given. + assert.Equal(t, "/tmp/bench/state-actor/simple-amsterdam-compute", cfg.Runner.ContainerNetwork) + // global.env value that itself references the shell env (BASE_DIR unset → its default). + assert.Equal(t, "/srv/fixtures", cfg.Runner.Benchmark.ResultsDir) + // inline default still applies when neither shell nor global.env has the var. + assert.Equal(t, "fallback", cfg.Runner.Benchmark.Tests.Filter) + // keys keep their original casing for substitution despite Viper lowercasing. + require.Contains(t, cfg.Global.Env, "state_dir") // parsed map is lowercased (documented) + }) + + t.Run("shell env overrides global.env", func(t *testing.T) { + t.Setenv("STATE_DIR", "/mnt/big") + cfg, err := Load(configPath) + require.NoError(t, err) + assert.Equal(t, "/mnt/big", cfg.Runner.ContainerNetwork, "shell env must win over global.env") + }) +} + func TestLoad_EnvVarOverrides(t *testing.T) { // Create a minimal config file for testing. configContent := ` @@ -106,12 +360,12 @@ runner: }, }, { - name: "nested field override - directories.tmp_cachedir", + name: "nested field override - global.directories.cachedir", envVars: map[string]string{ - "BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_CACHEDIR": "/cache/custom", + "BENCHMARKOOR_GLOBAL_DIRECTORIES_CACHEDIR": "/cache/custom", }, validate: func(t *testing.T, cfg *Config) { - assert.Equal(t, "/cache/custom", cfg.Runner.Directories.TmpCacheDir) + assert.Equal(t, "/cache/custom", cfg.Global.Directories.CacheDir) }, }, { @@ -452,14 +706,13 @@ func TestSourceConfig_Validate(t *testing.T) { wantErr: false, }, { - name: "eest_fixtures local dir missing local_genesis_dir", + name: "eest_fixtures local dir without local_genesis_dir (stateful)", source: SourceConfig{ EESTFixtures: &EESTFixturesSource{ LocalFixturesDir: tmpDir, }, }, - wantErr: true, - errSubstr: "local_genesis_dir is required", + wantErr: false, }, { name: "eest_fixtures local dir missing local_fixtures_dir", @@ -3233,6 +3486,8 @@ func TestValidateBuilder(t *testing.T) { "reth": "ghcr.io/ethereum/state-actor-reth:latest", "besu": "ghcr.io/ethereum/state-actor-besu:latest", "nethermind": "ghcr.io/ethereum/state-actor-nethermind:latest", + "ethrex": "ghcr.io/ethereum/state-actor-ethrex:latest", + "erigon": "ghcr.io/ethereum/state-actor-erigon:latest", } // mkCfg builds a Config with just the builder block populated so the @@ -3287,13 +3542,11 @@ func TestValidateBuilder(t *testing.T) { }, }, { - name: "unsupported client erigon", + name: "supported client erigon", sa: &StateActorConfig{ Images: allImages, Targets: []StateActorTarget{{Client: "erigon", OutputDir: dirA, TargetSize: "5GB"}}, }, - wantErr: true, - errSubstr: "erigon", }, { name: "unsupported client nimbus", @@ -3665,3 +3918,423 @@ func TestGetStateActorContainerRuntime(t *testing.T) { assert.Equal(t, "docker", cfg.GetStateActorContainerRuntime()) }) } + +func TestValidateEESTPayloads(t *testing.T) { + dirA := t.TempDir() + dirB := t.TempDir() + + // mkCfg builds a Config with only the eest_payloads builder block so the + // builder validation runs in isolation. + mkCfg := func(ep *EESTPayloadsConfig) *Config { + return &Config{Builder: &BuilderConfig{EESTPayloads: ep}} + } + + // base returns a minimal valid target rooted at dir. + base := func(dir string) EESTPayloadTarget { + return EESTPayloadTarget{ + FillerClient: "geth", + FillerImage: "ethpandaops/geth:master", + SourceDir: "/snap", + OutputDir: dir, + Fork: "Osaka", + Tests: []string{"tests/benchmark/compute"}, + } + } + + dockerfile := filepath.Join(dirA, "Dockerfile.eest-filler") + if err := os.WriteFile(dockerfile, []byte("FROM scratch\n"), 0o644); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + ep *EESTPayloadsConfig + wantErr bool + errSubstr string + }{ + { + name: "nil builder is fine", + ep: nil, + }, + { + name: "valid minimal", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, + { + name: "neither fill_image nor fill_dockerfile", + ep: &EESTPayloadsConfig{ + Targets: []EESTPayloadTarget{base(dirA)}, + }, + wantErr: true, + errSubstr: "fill_image", + }, + { + name: "fill_dockerfile only is valid", + ep: &EESTPayloadsConfig{ + FillDockerfile: dockerfile, + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, + { + name: "fill_dockerfile not found", + ep: &EESTPayloadsConfig{ + FillDockerfile: filepath.Join(dirB, "missing", "Dockerfile"), + Targets: []EESTPayloadTarget{base(dirA)}, + }, + wantErr: true, + errSubstr: "fill_dockerfile", + }, + { + name: "eest_repo without eest_ref is valid (ref defaults)", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + EESTRepo: "https://github.com/ethereum/execution-specs.git", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, + { + name: "eest_ref alone is valid (repo defaults)", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + EESTRef: "v1.2.3", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, + { + name: "invalid container_runtime", + ep: &EESTPayloadsConfig{ + ContainerRuntime: "lima", + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + wantErr: true, + errSubstr: "container_runtime", + }, + { + name: "unsupported filler client", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FillerClient = "reth" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "cannot act as the fill-stateful filler", + }, + { + name: "besu filler client is supported", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FillerClient = "besu" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + }, + { + name: "nethermind filler client is supported", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FillerClient = "nethermind" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + }, + { + name: "missing tests", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.Tests = nil + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "tests is required", + }, + { + name: "missing fork", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.Fork = "" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "fork is required", + }, + { + name: "fork hoisted from config defaults", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.Fork = "" + + return &EESTPayloadsConfig{ + FillImage: "fill:latest", + Config: &EESTPayloadDefaults{Fork: "Osaka"}, + Targets: []EESTPayloadTarget{tgt}, + } + }(), + }, + { + name: "relative source_dir", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.SourceDir = "relative/path" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "source_dir must be an absolute path", + }, + { + name: "relative output_dir", + ep: func() *EESTPayloadsConfig { + tgt := base("relative/out") + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "output_dir must be an absolute path", + }, + { + name: "duplicate output_dir", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{ + func() EESTPayloadTarget { t := base(dirA); t.Name = "a"; return t }(), + func() EESTPayloadTarget { t := base(dirA); t.Name = "b"; return t }(), + }, + }, + wantErr: true, + errSubstr: "duplicates", + }, + { + name: "duplicate name", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{ + func() EESTPayloadTarget { t := base(dirA); t.Name = "dup"; return t }(), + func() EESTPayloadTarget { t := base(dirB); t.Name = "dup"; return t }(), + }, + }, + wantErr: true, + errSubstr: "duplicates", + }, + { + name: "invalid datadir_method", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.DataDirMethod = "btrfs" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "datadir_method", + }, + { + name: "invalid gas_benchmark_values", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.GasBenchmarkValues = []int{10, 0} + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "gas_benchmark_values", + }, + { + name: "invalid fixed_opcode_count", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FixedOpcodeCount = &[]float64{0.5, -1} + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "fixed_opcode_count", + }, + { + name: "gas_benchmark_values and fixed_opcode_count are mutually exclusive", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.GasBenchmarkValues = []int{10} + tgt.FixedOpcodeCount = &[]float64{0.5} + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "mutually exclusive", + }, + { + name: "fixed_opcode_count bare (empty list) is valid", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FixedOpcodeCount = &[]float64{} + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: false, + }, + { + name: "inline address_stubs is valid", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.AddressStubs = map[string]map[string]string{ + "bloated_eoa_10GB": {"addr": "0x87a6", "pkey": "0x4da3"}, + } + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: false, + }, + { + name: "address_stubs_file and address_stubs are mutually exclusive", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.AddressStubsFile = "/host/stubs.json" + tgt.AddressStubs = map[string]map[string]string{ + "bloated_eoa_10GB": {"addr": "0x87a6"}, + } + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "mutually exclusive", + }, + { + name: "inline address_stubs entry without addr fails", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.AddressStubs = map[string]map[string]string{ + "bloated_eoa_10GB": {"pkey": "0x4da3"}, + } + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "addr is required", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := mkCfg(tt.ep).validateBuilder() + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + + return + } + + require.NoError(t, err) + }) + } +} + +func TestEESTPayloadsResolveTarget(t *testing.T) { + ep := &EESTPayloadsConfig{ + Config: &EESTPayloadDefaults{ + FillerImage: "ethpandaops/geth:master", + Fork: "Osaka", + Tests: []string{"tests/benchmark/stateful/bloatnet"}, + Filter: "not erc20", + Marker: "not repricing", + AddressStubs: map[string]map[string]string{"bloated_eoa_10GB": {"addr": "0x87a6"}}, + GasBenchmarkValues: []int{10, 30}, + DataDirMethod: "zfs", + MaxGasPerTest: u64Cfg(45000000), + RPCSeedKey: "0xseed", + FillerExtraArgs: []string{"--verbosity=3"}, + }, + Targets: []EESTPayloadTarget{ + // Inherits everything from config. + {Name: "inherit", FillerClient: "geth", SourceDir: "/s", OutputDir: "/o"}, + // Overrides fork and gas values. + {Name: "override", FillerClient: "geth", SourceDir: "/s2", OutputDir: "/o2", + Fork: "Prague", GasBenchmarkValues: []int{60}}, + }, + } + + inherit := ep.ResolveTarget(0) + assert.Equal(t, "ethpandaops/geth:master", inherit.FillerImage) + assert.Equal(t, "Osaka", inherit.Fork) + assert.Equal(t, []string{"tests/benchmark/stateful/bloatnet"}, inherit.Tests) + assert.Equal(t, "not erc20", inherit.Filter) + assert.Equal(t, "not repricing", inherit.Marker) + assert.Equal(t, map[string]map[string]string{"bloated_eoa_10GB": {"addr": "0x87a6"}}, inherit.AddressStubs) + assert.Equal(t, []int{10, 30}, inherit.GasBenchmarkValues) + assert.Equal(t, "zfs", inherit.DataDirMethod) + require.NotNil(t, inherit.MaxGasPerTest) + assert.Equal(t, uint64(45000000), *inherit.MaxGasPerTest) + assert.Equal(t, []string{"--verbosity=3"}, inherit.FillerExtraArgs) + + override := ep.ResolveTarget(1) + assert.Equal(t, "Prague", override.Fork, "per-target fork wins") + assert.Equal(t, []int{60}, override.GasBenchmarkValues, "per-target gas values win") + assert.Equal(t, "ethpandaops/geth:master", override.FillerImage, "still inherits unset fields") + assert.Equal(t, "not erc20", override.Filter, "inherits filter when unset") +} + +// TestEESTPayloadsResolveTarget_AddressStubsUnit verifies the address-stubs +// pair is hoisted as a unit: a target setting either form inherits neither, +// preserving their mutual exclusion. +func TestEESTPayloadsResolveTarget_AddressStubsUnit(t *testing.T) { + ep := &EESTPayloadsConfig{ + Config: &EESTPayloadDefaults{ + AddressStubs: map[string]map[string]string{"global": {"addr": "0xglobal"}}, + }, + Targets: []EESTPayloadTarget{ + // Sets only the file form: must NOT inherit the global inline map. + {Name: "file-only", FillerClient: "geth", SourceDir: "/s", OutputDir: "/o", + AddressStubsFile: "/host/stubs.json"}, + // Sets neither: inherits the global inline map. + {Name: "inherit", FillerClient: "geth", SourceDir: "/s2", OutputDir: "/o2"}, + }, + } + + fileOnly := ep.ResolveTarget(0) + assert.Equal(t, "/host/stubs.json", fileOnly.AddressStubsFile) + assert.Empty(t, fileOnly.AddressStubs, "target with file form must not inherit global inline stubs") + + inherit := ep.ResolveTarget(1) + assert.Empty(t, inherit.AddressStubsFile) + assert.Equal(t, map[string]map[string]string{"global": {"addr": "0xglobal"}}, inherit.AddressStubs) +} + +func TestEESTPayloadsEffectiveName(t *testing.T) { + withName := EESTPayloadTarget{Name: "compute", FillerClient: "geth"} + assert.Equal(t, "compute", withName.EffectiveName()) + + noName := EESTPayloadTarget{FillerClient: "geth"} + assert.Equal(t, "geth", noName.EffectiveName()) +} + +func TestEESTPayloadsResolveFillCommand(t *testing.T) { + def := (&EESTPayloadsConfig{}).ResolveFillCommand() + assert.Equal(t, []string{"uv", "run", "fill-stateful"}, def) + + custom := (&EESTPayloadsConfig{FillCommand: []string{"fill-stateful"}}).ResolveFillCommand() + assert.Equal(t, []string{"fill-stateful"}, custom) +} + +func TestGetEESTPayloadsContainerRuntime(t *testing.T) { + t.Run("builder override wins", func(t *testing.T) { + cfg := &Config{ + Runner: RunnerConfig{ContainerRuntime: "docker"}, + Builder: &BuilderConfig{EESTPayloads: &EESTPayloadsConfig{ContainerRuntime: "podman"}}, + } + assert.Equal(t, "podman", cfg.GetEESTPayloadsContainerRuntime()) + }) + + t.Run("falls back to runner runtime", func(t *testing.T) { + cfg := &Config{ + Runner: RunnerConfig{ContainerRuntime: "podman"}, + Builder: &BuilderConfig{EESTPayloads: &EESTPayloadsConfig{}}, + } + assert.Equal(t, "podman", cfg.GetEESTPayloadsContainerRuntime()) + }) +} + +func u64Cfg(v uint64) *uint64 { return &v } diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index ed65ae23c..7ae900c3e 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -28,6 +28,7 @@ type ContainerManager interface { // Network operations. EnsureNetwork(ctx context.Context, name string) error + NetworkExists(ctx context.Context, name string) (bool, error) RemoveNetwork(ctx context.Context, name string) error // Container operations. @@ -106,6 +107,7 @@ type ContainerSpec struct { ResourceLimits *ResourceLimits CapAdd []string // Additional Linux capabilities (e.g., "SYS_PTRACE" for CRIU). SecurityOpt []string // Security options (e.g., "seccomp=unconfined"). + User string // Container user (e.g., "1000:1000"); defaults to "root" when empty. } // Mount defines a volume mount. @@ -216,6 +218,24 @@ func (m *manager) EnsureNetwork(ctx context.Context, name string) error { return nil } +// NetworkExists reports whether a network with the given name exists. +func (m *manager) NetworkExists(ctx context.Context, name string) (bool, error) { + networks, err := m.client.NetworkList(ctx, network.ListOptions{ + Filters: filters.NewArgs(filters.Arg("name", name)), + }) + if err != nil { + return false, fmt.Errorf("listing networks: %w", err) + } + + for _, net := range networks { + if net.Name == name { + return true, nil + } + } + + return false, nil +} + // RemoveNetwork removes a Docker network. func (m *manager) RemoveNetwork(ctx context.Context, name string) error { if err := m.client.NetworkRemove(ctx, name); err != nil { @@ -247,9 +267,17 @@ func (m *manager) CreateContainer(ctx context.Context, spec *ContainerSpec) (str }) } + // Containers run as root unless the spec requests a specific user (e.g. the + // state-actor builder runs as the invoking host user so its output datadir + // is owned by that user rather than root). + user := spec.User + if user == "" { + user = "root" + } + containerCfg := &container.Config{ Image: spec.Image, - User: "root", + User: user, Env: env, Labels: spec.Labels, Entrypoint: spec.Entrypoint, diff --git a/pkg/eest/converter.go b/pkg/eest/converter.go index 063671aca..0ee72f4a5 100644 --- a/pkg/eest/converter.go +++ b/pkg/eest/converter.go @@ -63,6 +63,73 @@ func ConvertFixture(name string, fixture *Fixture) (*ConvertedTest, error) { return result, nil } +// ConvertStatefulFixture converts a stateful-engine fixture to JSON-RPC calls. +// Replay boots from a snapshot datadir rather than a genesis, so the setup +// phase is the shared pre_run payloads (snapshot → start block, preRun may be +// nil) followed by the fixture's own setupEngineNewPayloads (start block → +// per-test pre-state). The fixture's engineNewPayloads (the benchmark block) +// become the measured test step. Each payload still emits an +// engine_newPayload + engine_forkchoiceUpdated pair, so the chain head +// advances naturally and no separate forkchoice injection is needed. +func ConvertStatefulFixture(name string, fixture *Fixture, preRun *StatefulPreRun) (*ConvertedTest, error) { + if fixture == nil { + return nil, fmt.Errorf("fixture is nil") + } + + if len(fixture.EngineNewPayloads) == 0 { + return nil, fmt.Errorf("fixture has no benchmark payloads") + } + + // Setup = shared pre_run payloads, then the fixture's own setup payloads. + setupPayloads := make([]*EngineNewPayload, 0, + len(fixture.SetupEngineNewPayloads)+preRunPayloadCount(preRun)) + + if preRun != nil { + setupPayloads = append(setupPayloads, preRun.EngineNewPayloads...) + } + + setupPayloads = append(setupPayloads, fixture.SetupEngineNewPayloads...) + + result := &ConvertedTest{ + Name: name, + SetupLines: make([]string, 0, len(setupPayloads)*2), + TestLines: make([]string, 0, len(fixture.EngineNewPayloads)*2), + GenesisHash: fixture.SnapshotBlockHash, + PayloadCount: len(setupPayloads) + len(fixture.EngineNewPayloads), + } + + for i, payload := range setupPayloads { + lines, err := convertPayload(payload, i+1) + if err != nil { + return nil, fmt.Errorf("converting setup payload %d: %w", i, err) + } + + result.SetupLines = append(result.SetupLines, lines...) + } + + for i, payload := range fixture.EngineNewPayloads { + lines, err := convertPayload(payload, len(setupPayloads)+i+1) + if err != nil { + return nil, fmt.Errorf("converting benchmark payload %d: %w", i, err) + } + + result.TestLines = append(result.TestLines, lines...) + result.FinalHash = payload.ExecutionPayload.BlockHash + } + + return result, nil +} + +// preRunPayloadCount returns the number of pre_run payloads, tolerating a nil +// pre_run (capacity hint only). +func preRunPayloadCount(preRun *StatefulPreRun) int { + if preRun == nil { + return 0 + } + + return len(preRun.EngineNewPayloads) +} + // convertPayload generates JSON-RPC lines for a single payload. func convertPayload(payload *EngineNewPayload, id int) ([]string, error) { if payload.ExecutionPayload == nil { diff --git a/pkg/eest/converter_test.go b/pkg/eest/converter_test.go index fa4b7a513..ab85e7879 100644 --- a/pkg/eest/converter_test.go +++ b/pkg/eest/converter_test.go @@ -220,6 +220,198 @@ func TestConvertFixture_PayloadVersions(t *testing.T) { } } +// statefulPayload builds a minimal EngineNewPayload for stateful conversion +// tests, with the given block number/hash/parent. +func statefulPayload(number, hash, parent string) *EngineNewPayload { + return &EngineNewPayload{ + ExecutionPayload: &ExecutionPayload{ + ParentHash: parent, + FeeRecipient: "0xfee", + StateRoot: "0xstate", + ReceiptsRoot: "0xreceipts", + LogsBloom: "0xbloom", + PrevRandao: "0xrandao", + BlockNumber: number, + GasLimit: "0x1000000", + GasUsed: "0x0", + Timestamp: "0x100", + ExtraData: "0x", + BaseFeePerGas: "0x7", + BlockHash: hash, + Transactions: []string{}, + }, + NewPayloadVersion: 4, + ForkchoiceUpdatedVersion: 3, + BlobVersionedHashes: []string{}, + ParentBeaconBlockRoot: "0xbeacon", + ExecutionRequests: []string{}, + } +} + +func TestConvertStatefulFixture(t *testing.T) { + preRun := &StatefulPreRun{ + SnapshotBlockHash: "0xsnapshot", + StartBlockHash: "0xstart", + // snapshot (0x0) -> start (0x3): three pre_run blocks. + EngineNewPayloads: []*EngineNewPayload{ + statefulPayload("0x1", "0xb1", "0xsnapshot"), + statefulPayload("0x2", "0xb2", "0xb1"), + statefulPayload("0x3", "0xstart", "0xb2"), + }, + } + + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + Network: "Osaka", + SnapshotBlockHash: "0xsnapshot", + StartBlockHash: "0xstart", + LastBlockHash: "0xbench", + // start (0x3) -> setup (0x4). + SetupEngineNewPayloads: []*EngineNewPayload{ + statefulPayload("0x4", "0xsetup", "0xstart"), + }, + // setup (0x4) -> benchmark (0x5): the measured block. + EngineNewPayloads: []*EngineNewPayload{ + statefulPayload("0x5", "0xbench", "0xsetup"), + }, + } + + result, err := ConvertStatefulFixture("test_stateful", fixture, preRun) + require.NoError(t, err) + + assert.Equal(t, "test_stateful", result.Name) + // GenesisHash carries the snapshot hash for reporting. + assert.Equal(t, "0xsnapshot", result.GenesisHash) + assert.Equal(t, "0xbench", result.FinalHash) + // 3 pre_run + 1 setup + 1 benchmark = 5 payloads. + assert.Equal(t, 5, result.PayloadCount) + // Setup = (3 pre_run + 1 setup) * 2 lines (newPayload + fcU). + assert.Len(t, result.SetupLines, 8) + // Test = 1 benchmark * 2 lines. + assert.Len(t, result.TestLines, 2) + + // Ordering by CONTENT (not just method name): the shared pre_run blocks must + // precede the fixture's own setup block. SetupLines are (newPayload, fcU) + // pairs, so newPayload lines sit at even indices: [0]=pre_run#1, [2]=pre_run#2, + // [4]=pre_run#3 (start), [6]=setup. + assert.Equal(t, "engine_newPayloadV4", rpcMethod(t, result.SetupLines[0])) + assert.Equal(t, "0xb1", newPayloadBlockHash(t, result.SetupLines[0]), + "first setup line must replay the first pre_run block, not the fixture's setup") + assert.Equal(t, "0xstart", newPayloadBlockHash(t, result.SetupLines[4]), + "third newPayload is the last pre_run block (start)") + assert.Equal(t, "0xsetup", newPayloadBlockHash(t, result.SetupLines[6]), + "the fixture's own setup block comes AFTER the pre_run blocks") + + // The benchmark newPayload is the test step. + assert.Equal(t, "engine_newPayloadV4", rpcMethod(t, result.TestLines[0])) + assert.Equal(t, "0xbench", newPayloadBlockHash(t, result.TestLines[0])) +} + +// rpcMethod decodes a JSON-RPC line and returns its "method". +func rpcMethod(t *testing.T, line string) string { + t.Helper() + + var call map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &call)) + + method, _ := call["method"].(string) + + return method +} + +// newPayloadBlockHash decodes an engine_newPayloadVX line and returns the +// execution payload's blockHash (params[0].blockHash). +func newPayloadBlockHash(t *testing.T, line string) string { + t.Helper() + + var call map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &call)) + + params, ok := call["params"].([]any) + require.True(t, ok && len(params) > 0, "newPayload line must carry params") + + payload, ok := params[0].(map[string]any) + require.True(t, ok, "first param must be the execution payload object") + + hash, _ := payload["blockHash"].(string) + + return hash +} + +func TestConvertStatefulFixture_NilPreRun(t *testing.T) { + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + SnapshotBlockHash: "0xsnapshot", + SetupEngineNewPayloads: []*EngineNewPayload{statefulPayload("0x4", "0xsetup", "0xstart")}, + EngineNewPayloads: []*EngineNewPayload{statefulPayload("0x5", "0xbench", "0xsetup")}, + } + + result, err := ConvertStatefulFixture("test_stateful", fixture, nil) + require.NoError(t, err) + + // Without pre_run, only the fixture's own setup payload is replayed. + assert.Len(t, result.SetupLines, 2) + assert.Len(t, result.TestLines, 2) + assert.Equal(t, 2, result.PayloadCount) +} + +func TestConvertStatefulFixture_NoBenchmarkPayloads(t *testing.T) { + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + SetupEngineNewPayloads: []*EngineNewPayload{statefulPayload("0x4", "0xsetup", "0xstart")}, + EngineNewPayloads: []*EngineNewPayload{}, + } + + _, err := ConvertStatefulFixture("test", fixture, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no benchmark payloads") +} + +func TestConvertStatefulFixture_NilFixture(t *testing.T) { + _, err := ConvertStatefulFixture("test", nil, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "fixture is nil") +} + +func TestParsePreRunFile(t *testing.T) { + jsonData := `{ + "network": "Osaka", + "snapshotBlockHash": "0xsnapshot", + "startBlockHash": "0xstart", + "engineNewPayloads": [ + { + "newPayloadVersion": "4", + "forkchoiceUpdatedVersion": "3", + "params": [ + {"parentHash":"0xsnapshot","feeRecipient":"0xfee","stateRoot":"0xs", + "receiptsRoot":"0xr","logsBloom":"0xb","prevRandao":"0xrd", + "blockNumber":"0x1","gasLimit":"0x1000000","gasUsed":"0x0", + "timestamp":"0x100","extraData":"0x","baseFeePerGas":"0x7", + "blockHash":"0xstart","transactions":[]}, + [], "0xbeacon", [] + ] + } + ] + }` + + preRun, err := ParsePreRunFile([]byte(jsonData)) + require.NoError(t, err) + assert.Equal(t, "0xstart", preRun.StartBlockHash) + assert.Equal(t, "0xsnapshot", preRun.SnapshotBlockHash) + require.Len(t, preRun.EngineNewPayloads, 1) + assert.Equal(t, 4, preRun.EngineNewPayloads[0].NewPayloadVersion) +} + +func TestFixture_IsStateful(t *testing.T) { + stateful := &Fixture{Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}} + assert.True(t, stateful.IsStateful()) + assert.True(t, stateful.IsSupportedFormat()) + + genesisBased := &Fixture{Info: &FixtureInfo{FixtureFormat: SupportedFixtureFormat}} + assert.False(t, genesisBased.IsStateful()) + assert.True(t, genesisBased.IsSupportedFormat()) +} + func TestParseFixtureFile(t *testing.T) { jsonData := `{ "test_one": { diff --git a/pkg/eest/fixture.go b/pkg/eest/fixture.go index 37cd563e5..9b2877de4 100644 --- a/pkg/eest/fixture.go +++ b/pkg/eest/fixture.go @@ -6,15 +6,42 @@ import ( "strconv" ) -// SupportedFixtureFormat is the fixture format we support. +// SupportedFixtureFormat is the genesis-based fixture format we support. const SupportedFixtureFormat = "blockchain_test_engine_x" +// SupportedStatefulFixtureFormat is the stateful-engine fixture format. Unlike +// the genesis-based format, these fixtures boot from a pre-populated snapshot +// datadir (no genesis): shared pre_run payloads advance the snapshot to a +// common start block, SetupEngineNewPayloads bring the per-test pre-state into +// place, and EngineNewPayloads carry the benchmark block. +const SupportedStatefulFixtureFormat = "blockchain_test_stateful_engine" + // Fixture represents a single EEST test fixture. type Fixture struct { Info *FixtureInfo `json:"_info"` Network string `json:"network"` GenesisBlockHeader *BlockHeader `json:"genesisBlockHeader"` EngineNewPayloads []*EngineNewPayload `json:"engineNewPayloads"` + + // Stateful-engine fields (only set for SupportedStatefulFixtureFormat). + // SetupEngineNewPayloads run after the shared pre_run payloads to build the + // per-test pre-state; StartBlockHash links the fixture to its pre_run file. + SetupEngineNewPayloads []*EngineNewPayload `json:"setupEngineNewPayloads"` + SnapshotBlockHash string `json:"snapshotBlockHash"` + StartBlockHash string `json:"startBlockHash"` + LastBlockHash string `json:"lastblockhash"` +} + +// StatefulPreRun is a shared pre_run file referenced by stateful fixtures via +// their startBlockHash. Its EngineNewPayloads advance the snapshot datadir to +// the common start block before each fixture's per-test setup runs. The same +// pre_run is reused by every fixture that starts from the same block, which is +// why EEST writes it once under pre_run/.json. +type StatefulPreRun struct { + Network string `json:"network"` + SnapshotBlockHash string `json:"snapshotBlockHash"` + StartBlockHash string `json:"startBlockHash"` + EngineNewPayloads []*EngineNewPayload `json:"engineNewPayloads"` } // FixtureInfo contains metadata about the fixture. @@ -30,7 +57,18 @@ type FixtureInfo struct { // IsSupportedFormat returns true if the fixture has a supported format. func (f *Fixture) IsSupportedFormat() bool { - return f.Info != nil && f.Info.FixtureFormat == SupportedFixtureFormat + if f.Info == nil { + return false + } + + return f.Info.FixtureFormat == SupportedFixtureFormat || + f.Info.FixtureFormat == SupportedStatefulFixtureFormat +} + +// IsStateful reports whether the fixture uses the stateful-engine format, which +// replays against a snapshot datadir and carries no genesis. +func (f *Fixture) IsStateful() bool { + return f.Info != nil && f.Info.FixtureFormat == SupportedStatefulFixtureFormat } // BlockHeader represents an Ethereum block header. @@ -216,3 +254,14 @@ func ParseFixtureFile(data []byte) (map[string]*Fixture, error) { return fixtures, nil } + +// ParsePreRunFile parses a stateful pre_run JSON file. Unlike a fixture file, +// it holds a single object (not a map keyed by test name). +func ParsePreRunFile(data []byte) (*StatefulPreRun, error) { + var preRun StatefulPreRun + if err := json.Unmarshal(data, &preRun); err != nil { + return nil, err + } + + return &preRun, nil +} diff --git a/pkg/executor/eest_source.go b/pkg/executor/eest_source.go index a4b8f8dd3..9bfea87c2 100644 --- a/pkg/executor/eest_source.go +++ b/pkg/executor/eest_source.go @@ -560,6 +560,72 @@ func (s *EESTSource) downloadAndExtractTarball(ctx context.Context, url, targetD return nil } +// statefulPreRunMissing reports whether a stateful fixture's absent pre_run +// file is worth warning about. It is only a concern when the start block is +// ahead of the snapshot: then the snapshot→start advance is skipped and the +// test would replay against the wrong state. When start == snapshot (e.g. a +// pre-funded seed lets fill-stateful skip the funding block) there are no +// pre_run blocks to replay, so the absence is expected and silent. +func statefulPreRunMissing(f *eest.Fixture) bool { + return f.StartBlockHash != "" && f.StartBlockHash != f.SnapshotBlockHash +} + +// loadPreRuns reads the shared pre_run files for stateful-engine fixtures from +// /pre_run/*.json, keyed by start block hash. A missing pre_run +// directory is not an error (the genesis-based format has none) and yields an +// empty map. +func (s *EESTSource) loadPreRuns(searchDir string) (map[string]*eest.StatefulPreRun, error) { + preRunDir := filepath.Join(searchDir, "pre_run") + + entries, err := os.ReadDir(preRunDir) + if err != nil { + if os.IsNotExist(err) { + return map[string]*eest.StatefulPreRun{}, nil + } + + return nil, fmt.Errorf("reading pre_run directory: %w", err) + } + + preRuns := make(map[string]*eest.StatefulPreRun, len(entries)) + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + + path := filepath.Join(preRunDir, entry.Name()) + + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading pre_run file %s: %w", path, err) + } + + preRun, err := eest.ParsePreRunFile(data) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": path, + "error": err, + }).Warn("Failed to parse pre_run file, skipping") + + continue + } + + // Key by the start block hash the fixtures reference. Fall back to the + // file name (which EEST names .json) if the field is + // absent. + key := preRun.StartBlockHash + if key == "" { + key = strings.TrimSuffix(entry.Name(), ".json") + } + + preRuns[key] = preRun + } + + s.log.WithField("count", len(preRuns)).Debug("Loaded stateful pre_run files") + + return preRuns, nil +} + // discoverTests parses fixture files and creates test entries. func (s *EESTSource) discoverTests() (*PreparedSource, error) { // Determine the fixtures search directory. @@ -581,19 +647,38 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { Tests: make([]*TestWithSteps, 0), } + // EEST writes a .meta directory (fixtures.ini with the fill command + python/ + // tool versions, index.json, report_fill.html) at the root of the fixtures + // dir. Attach it when present so each suite's output can carry the provenance. + metaDir := filepath.Join(s.fixturesDir, ".meta") + if fi, err := os.Stat(metaDir); err == nil && fi.IsDir() { + result.MetaDir = metaDir + + s.log.WithField("meta_dir", metaDir).Debug("Found EEST .meta directory") + } + s.log.WithField("path", searchDir).Info("Searching for fixtures") + // Load shared pre_run files (stateful-engine format). Keyed by start block + // hash; empty for the genesis-based format, which has no pre_run dir. + preRuns, err := s.loadPreRuns(searchDir) + if err != nil { + return nil, err + } + // Map fixture keys (testIds) to their TestWithSteps for pre_alloc matching. testsByFixtureKey := make(map[string]*TestWithSteps, 256) // Walk fixture directory for JSON files. - err := filepath.Walk(searchDir, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(searchDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { - if info.Name() == "pre_alloc" { + // pre_alloc holds genesis groups; pre_run holds stateful setup + // payloads loaded separately above. Neither contains fixtures. + if info.Name() == "pre_alloc" || info.Name() == "pre_run" { return filepath.SkipDir } @@ -643,7 +728,25 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { continue } - converted, err := eest.ConvertFixture(name, fixture) + var converted *eest.ConvertedTest + + if fixture.IsStateful() { + preRun := preRuns[fixture.StartBlockHash] + if preRun == nil && statefulPreRunMissing(fixture) { + s.log.WithFields(logrus.Fields{ + "file": path, + "fixture": name, + "start_block": fixture.StartBlockHash, + "snapshot_block": fixture.SnapshotBlockHash, + }).Warn("No pre_run file for stateful fixture's start block; " + + "replaying setup payloads only") + } + + converted, err = eest.ConvertStatefulFixture(name, fixture, preRun) + } else { + converted, err = eest.ConvertFixture(name, fixture) + } + if err != nil { s.log.WithFields(logrus.Fields{ "file": path, diff --git a/pkg/executor/eest_source_test.go b/pkg/executor/eest_source_test.go new file mode 100644 index 000000000..2bce66aa6 --- /dev/null +++ b/pkg/executor/eest_source_test.go @@ -0,0 +1,43 @@ +package executor + +import ( + "testing" + + "github.com/ethpandaops/benchmarkoor/pkg/eest" + "github.com/stretchr/testify/assert" +) + +func TestStatefulPreRunMissing(t *testing.T) { + tests := []struct { + name string + startHash string + snapHash string + want bool + }{ + { + name: "start ahead of snapshot warns", + startHash: "0xstart", + snapHash: "0xsnapshot", + want: true, + }, + { + name: "start equals snapshot is silent", + startHash: "0xsnapshot", + snapHash: "0xsnapshot", + want: false, + }, + { + name: "empty start block is silent", + startHash: "", + snapHash: "0xsnapshot", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &eest.Fixture{StartBlockHash: tt.startHash, SnapshotBlockHash: tt.snapHash} + assert.Equal(t, tt.want, statefulPreRunMissing(f)) + }) + } +} diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index f6a253b47..1126858e0 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -1296,29 +1296,12 @@ func (e *executor) executeRPC( start := time.Now() resp, err := http.DefaultClient.Do(req) - // Read stats AFTER the request completes and compute delta. - // This captures resource usage during server processing, not during body read. - var delta *ResourceDelta - if e.statsReader != nil && beforeStats != nil { - if afterStats, readErr := e.statsReader.ReadStats(); readErr == nil { - statsDelta := stats.ComputeDelta(beforeStats, afterStats) - if statsDelta != nil { - delta = &ResourceDelta{ - MemoryDelta: statsDelta.MemoryDelta, - MemoryAbsBytes: afterStats.Memory, - CPUDeltaUsec: statsDelta.CPUDeltaUsec, - DiskReadBytes: statsDelta.DiskReadBytes, - DiskWriteBytes: statsDelta.DiskWriteBytes, - DiskReadOps: statsDelta.DiskReadOps, - DiskWriteOps: statsDelta.DiskWriteOps, - } - } - } - } - if err != nil { fullDuration := time.Since(start).Nanoseconds() + // Collect the delta after timing is captured (see collectResourceDelta). + delta := e.collectResourceDelta(beforeStats) + return "", 0, fullDuration, delta, fmt.Errorf("executing request: %w", err) } @@ -1335,6 +1318,13 @@ func (e *executor) executeRPC( duration = bodyReadComplete.Sub(wroteRequest).Nanoseconds() } + // Read the AFTER stats only now that the timing window is closed. The stats + // backend can block — the Docker Stats API on macOS takes ~1-2s per read — + // and any time spent there must NOT count toward the measured RPC duration, + // which feeds MGas/s. Reading it earlier (between Do() and the body read) + // inflated every newPayload to ~2s and crushed MGas/s. + delta := e.collectResourceDelta(beforeStats) + if err != nil { return "", duration, fullDuration, delta, fmt.Errorf("reading response: %w", err) } @@ -1342,6 +1332,38 @@ func (e *executor) executeRPC( return strings.TrimSpace(string(body)), duration, fullDuration, delta, nil } +// collectResourceDelta reads the post-request stats snapshot and diffs it +// against before, returning nil when stats collection is disabled/unavailable. +// +// It MUST be called OUTSIDE the request-timing window: the stats backend can +// block (notably the Docker Stats API on macOS), and that latency must never be +// attributed to the RPC under measurement. +func (e *executor) collectResourceDelta(before *stats.Stats) *ResourceDelta { + if e.statsReader == nil || before == nil { + return nil + } + + afterStats, err := e.statsReader.ReadStats() + if err != nil { + return nil + } + + statsDelta := stats.ComputeDelta(before, afterStats) + if statsDelta == nil { + return nil + } + + return &ResourceDelta{ + MemoryDelta: statsDelta.MemoryDelta, + MemoryAbsBytes: afterStats.Memory, + CPUDeltaUsec: statsDelta.CPUDeltaUsec, + DiskReadBytes: statsDelta.DiskReadBytes, + DiskWriteBytes: statsDelta.DiskWriteBytes, + DiskReadOps: statsDelta.DiskReadOps, + DiskWriteOps: statsDelta.DiskWriteOps, + } +} + // rpcRequest is used to parse the method from a JSON-RPC request. type rpcRequest struct { Method string `json:"method"` @@ -1739,7 +1761,7 @@ func executeSimpleRPC(ctx context.Context, endpoint, payload string) (string, er func (e *executor) dumpPostTestResponse( resultsDir, testName, filename, response string, ) error { - postTestDir := filepath.Join(resultsDir, testName, "post_test_rpc_calls") + postTestDir := filepath.Join(resultsDir, sanitizeResultPath(testName), "post_test_rpc_calls") if err := fsutil.MkdirAll(postTestDir, 0755, e.cfg.ResultsOwner); err != nil { return fmt.Errorf("creating post_test_rpc_calls directory: %w", err) } diff --git a/pkg/executor/results.go b/pkg/executor/results.go index caea97026..8ad2c0848 100644 --- a/pkg/executor/results.go +++ b/pkg/executor/results.go @@ -1,6 +1,8 @@ package executor import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "maps" @@ -13,6 +15,67 @@ import ( "github.com/ethpandaops/benchmarkoor/pkg/fsutil" ) +// testNameMarker is the file written into each per-test result directory holding +// the full, un-sanitized test name. Result aggregation reads it to recover the +// real name when the directory was shortened by sanitizeResultPath. +const testNameMarker = ".test-name" + +// resolveTestName returns the full test name for a result directory relative to +// resultsDir, reading the testNameMarker written by WriteStepResults. It falls +// back to the directory path when the marker is absent (older results), and +// caches per directory. +func resolveTestName(resultsDir, dir string, cache map[string]string) string { + if v, ok := cache[dir]; ok { + return v + } + + name := dir + + if data, err := os.ReadFile(filepath.Join(resultsDir, dir, testNameMarker)); err == nil { + if s := strings.TrimSpace(string(data)); s != "" { + name = s + } + } + + cache[dir] = name + + return name +} + +// maxResultPathComponent caps each path component of a test result directory +// below the common 255-byte filename limit, leaving headroom for suffixes like +// ".request". EEST benchmark node ids — especially the verbose stateful/bloatnet +// params — can exceed the limit and make MkdirAll fail with "file name too long". +const maxResultPathComponent = 200 + +// sanitizeResultPath caps each "/"-separated component of a test result path to +// maxResultPathComponent bytes. Over-long components are truncated and suffixed +// with a short content hash so they stay unique and stable; components within +// the limit are returned unchanged (so existing layouts don't move). Apply it +// consistently wherever a test name becomes a filesystem path (suite output, +// step results, post-test dir) so the directories match. The result.json inside +// still records the full test name, and result aggregation walks the tree, so +// truncating the directory name is safe. +func sanitizeResultPath(name string) string { + parts := strings.Split(name, "/") + + changed := false + + for i, p := range parts { + if len(p) > maxResultPathComponent { + sum := sha256.Sum256([]byte(p)) + parts[i] = p[:maxResultPathComponent-17] + "-" + hex.EncodeToString(sum[:8]) + changed = true + } + } + + if !changed { + return name + } + + return strings.Join(parts, "/") +} + // MethodStats contains aggregated statistics for a single method (int64 values). type MethodStats struct { Count int64 `json:"count"` @@ -570,11 +633,22 @@ func WriteStepResults( owner *fsutil.OwnerConfig, ) error { // Ensure the test directory exists. - testDir := filepath.Join(resultDir, testName) + testDir := filepath.Join(resultDir, sanitizeResultPath(testName)) if err := fsutil.MkdirAll(testDir, 0755, owner); err != nil { return fmt.Errorf("creating test result directory: %w", err) } + // Persist the full, un-sanitized test name alongside the results so result + // aggregation can recover it even when the directory name was shortened to + // fit the filesystem's filename limit (see sanitizeResultPath). Without this, + // downstream consumers (stats.json, the UI's EEST node-id parsing) would see + // the truncated+hashed directory name instead of the real test name. + if err := fsutil.WriteFile( + filepath.Join(testDir, testNameMarker), []byte(testName), 0644, owner, + ); err != nil { + return fmt.Errorf("writing test name marker: %w", err) + } + // Base path is the step type (e.g., "setup", "test", "cleanup"). basePath := filepath.Join(testDir, string(stepType)) @@ -627,6 +701,9 @@ func GenerateRunResult(resultsDir string) (*RunResult, error) { Tests: make(map[string]*TestEntry), } + // Caches the directory → full test name lookup (see testNameMarker). + nameCache := make(map[string]string) + // Walk the results directory looking for .result-aggregated.json files. err := filepath.Walk(resultsDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -683,10 +760,13 @@ func GenerateRunResult(resultsDir string) (*RunResult, error) { return nil } - // The test name is the directory containing the step files. + // The test name is the directory containing the step files; recover the + // full name from the marker when the directory was shortened. testName := dir if testName == "." { testName = "" + } else { + testName = resolveTestName(resultsDir, dir, nameCache) } // Set the step result. diff --git a/pkg/executor/sanitize_path_test.go b/pkg/executor/sanitize_path_test.go new file mode 100644 index 000000000..61c896b3a --- /dev/null +++ b/pkg/executor/sanitize_path_test.go @@ -0,0 +1,52 @@ +package executor + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSanitizeResultPath(t *testing.T) { + // Short components are returned unchanged. + short := "benchmark/stateful/bloatnet/test_x.py::test_y[fork_Amsterdam]" + assert.Equal(t, short, sanitizeResultPath(short)) + + // An over-long leaf component is truncated to the cap and suffixed with a hash. + longLeaf := strings.Repeat("a", 400) + name := "benchmark/bloatnet/" + longLeaf + out := sanitizeResultPath(name) + parts := strings.Split(out, "/") + leaf := parts[len(parts)-1] + assert.Len(t, leaf, maxResultPathComponent) + assert.Equal(t, "benchmark/bloatnet/", strings.Join(parts[:len(parts)-1], "/")+"/") + + // Distinct long names map to distinct sanitized paths (hash uniqueness). + a := sanitizeResultPath("p/" + strings.Repeat("a", 300) + "X") + b := sanitizeResultPath("p/" + strings.Repeat("a", 300) + "Y") + assert.NotEqual(t, a, b) + + // Deterministic. + assert.Equal(t, out, sanitizeResultPath(name)) +} + +func TestResolveTestName(t *testing.T) { + dir := t.TempDir() + full := "benchmark/stateful/bloatnet/test_x.py::test_y[" + strings.Repeat("p", 400) + "]" + sanitized := sanitizeResultPath(full) + + // WriteStepResults writes the marker into the (sanitized) test dir. + testDir := dir + "/" + sanitized + require.NoError(t, os.MkdirAll(testDir, 0o755)) + require.NoError(t, os.WriteFile(testDir+"/"+testNameMarker, []byte(full), 0o644)) + + cache := map[string]string{} + // The marker restores the full name for a shortened directory. + assert.Equal(t, full, resolveTestName(dir, sanitized, cache)) + // Cached on second call. + assert.Equal(t, full, resolveTestName(dir, sanitized, cache)) + // No marker → falls back to the directory path. + assert.Equal(t, "some/dir", resolveTestName(dir, "some/dir", cache)) +} diff --git a/pkg/executor/source.go b/pkg/executor/source.go index 781ebfbea..6eb8d8e73 100644 --- a/pkg/executor/source.go +++ b/pkg/executor/source.go @@ -6,13 +6,13 @@ import ( "encoding/hex" "fmt" "os" - "os/exec" "path/filepath" "sort" "strings" "github.com/ethpandaops/benchmarkoor/pkg/config" "github.com/ethpandaops/benchmarkoor/pkg/eest" + "github.com/ethpandaops/benchmarkoor/pkg/gitrepo" "github.com/sirupsen/logrus" ) @@ -57,6 +57,10 @@ type PreparedSource struct { BasePath string PreRunSteps []*StepFile Tests []*TestWithSteps + // MetaDir is the path to an auxiliary metadata directory to attach to each + // suite's output (e.g. an EEST fill's .meta dir with fixtures.ini, the fill + // report and index). Empty when the source has no such directory. + MetaDir string } // Source provides test files from local or git sources. @@ -199,77 +203,9 @@ func (s *GitSource) Prepare(ctx context.Context) (*PreparedSource, error) { return s.discoverTests() } -// prepareRepo clones or updates the git repository. +// prepareRepo clones or updates the git repository into the on-disk cache. func (s *GitSource) prepareRepo(ctx context.Context) (string, error) { - repoHash := hashRepoURL(s.cfg.Repo) - localPath := filepath.Join(s.cacheDir, repoHash) - - log := s.log.WithFields(logrus.Fields{ - "repo": s.cfg.Repo, - "version": s.cfg.Version, - "path": localPath, - }) - - if _, err := os.Stat(localPath); os.IsNotExist(err) { - log.Info("Cloning repository") - - if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil { - return "", fmt.Errorf("creating cache directory: %w", err) - } - - if looksLikeCommitHash(s.cfg.Version) { - // Commit hashes can't be used with --branch, so we init + fetch instead. - if err := s.cloneByCommitHash(ctx, localPath); err != nil { - return "", err - } - } else { - // Shallow clone with specific branch/tag. - cmd := exec.CommandContext(ctx, "git", "clone", - "--depth=1", - "--branch", s.cfg.Version, - "--single-branch", - s.cfg.Repo, localPath) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("cloning repository: %w", err) - } - } - } else { - // For commit hashes, skip fetch if HEAD already matches. - if looksLikeCommitHash(s.cfg.Version) { - headHash, err := s.getHeadHash(ctx, localPath) - if err == nil && strings.HasPrefix(headHash, s.cfg.Version) { - log.Info("Cached repository already at requested version") - - return localPath, nil - } - } - - log.Info("Updating cached repository") - - // Fetch the specific version. - cmd := exec.CommandContext(ctx, "git", "-C", localPath, "fetch", - "--depth=1", "origin", s.cfg.Version) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("fetching version: %w", err) - } - - // Checkout FETCH_HEAD. - cmd = exec.CommandContext(ctx, "git", "-C", localPath, "checkout", "FETCH_HEAD") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("checking out version: %w", err) - } - } - - return localPath, nil + return gitrepo.CloneOrUpdate(ctx, s.log, s.cfg.Repo, s.cfg.Version, s.cacheDir) } // discoverTests discovers all tests from the git source. @@ -307,78 +243,6 @@ func (s *GitSource) GetSourceInfo() (*SuiteSource, error) { return &SuiteSource{Git: git}, nil } -// cloneByCommitHash initializes a repo and fetches a specific commit hash. -func (s *GitSource) cloneByCommitHash(ctx context.Context, localPath string) error { - // git init - cmd := exec.CommandContext(ctx, "git", "init", localPath) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("initializing repository: %w", err) - } - - // git remote add origin - cmd = exec.CommandContext(ctx, "git", "-C", localPath, - "remote", "add", "origin", s.cfg.Repo) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("adding remote: %w", err) - } - - // git fetch --depth=1 origin - cmd = exec.CommandContext(ctx, "git", "-C", localPath, - "fetch", "--depth=1", "origin", s.cfg.Version) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("fetching commit %s: %w", s.cfg.Version, err) - } - - // git checkout FETCH_HEAD - cmd = exec.CommandContext(ctx, "git", "-C", localPath, - "checkout", "FETCH_HEAD") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("checking out commit %s: %w", s.cfg.Version, err) - } - - return nil -} - -// getHeadHash returns the current HEAD commit hash for the repository at repoPath. -func (s *GitSource) getHeadHash(ctx context.Context, repoPath string) (string, error) { - cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", "HEAD") - - out, err := cmd.Output() - if err != nil { - return "", err - } - - return strings.TrimSpace(string(out)), nil -} - -// looksLikeCommitHash returns true if s looks like a git commit hash -// (7-40 lowercase/uppercase hex characters). -func looksLikeCommitHash(s string) bool { - if len(s) < 7 || len(s) > 40 { - return false - } - - for _, c := range s { - if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { - return false - } - } - - return true -} - // hashRepoURL creates a hash of the repository URL for caching. func hashRepoURL(url string) string { hash := sha256.Sum256([]byte(url)) diff --git a/pkg/executor/source_test.go b/pkg/executor/source_test.go index 30ab54a83..783b46610 100644 --- a/pkg/executor/source_test.go +++ b/pkg/executor/source_test.go @@ -59,31 +59,3 @@ func TestDiscoverTestsFromConfig_PreRunStepsNotFiltered(t *testing.T) { assert.Len(t, result.Tests, 1, "only bn128 test should match filter") assert.Contains(t, result.Tests[0].Name, "bn128") } - -func TestLooksLikeCommitHash(t *testing.T) { - tests := []struct { - name string - input string - expected bool - }{ - {name: "full sha1", input: "e5011aa5f75d7a1722481f25408347fadfb7fd3c", expected: true}, - {name: "short hash 7 chars", input: "e5011aa", expected: true}, - {name: "short hash 8 chars", input: "e5011aa5", expected: true}, - {name: "uppercase hex", input: "E5011AA5F75D7A17", expected: true}, - {name: "mixed case hex", input: "e5011AA5f75d", expected: true}, - {name: "branch name", input: "main", expected: false}, - {name: "branch with slash", input: "feature/foo", expected: false}, - {name: "tag semver", input: "v1.0.0", expected: false}, - {name: "too short 6 chars", input: "e5011a", expected: false}, - {name: "too long 41 chars", input: "e5011aa5f75d7a1722481f25408347fadfb7fd3c0", expected: false}, - {name: "empty string", input: "", expected: false}, - {name: "hex with non-hex char", input: "e5011gg", expected: false}, - {name: "7 char all digits", input: "1234567", expected: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, looksLikeCommitHash(tt.input)) - }) - } -} diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 53a07445b..c4d703f99 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -24,6 +24,10 @@ type SuiteInfo struct { Metadata *config.MetadataConfig `json:"metadata,omitempty"` PreRunSteps []SuiteFile `json:"pre_run_steps,omitempty"` Tests []SuiteTest `json:"tests"` + // EESTMetadata is true when the suite output contains an .eest-meta + // directory copied from the EEST fixtures' .meta (fill provenance). Lets the + // UI surface an "EEST Metadata" view without statting the suite directory. + EESTMetadata bool `json:"eest_metadata,omitempty"` } // SuiteSource contains source information for the suite. @@ -166,9 +170,15 @@ func CreateSuiteOutput( suiteExists := false - // Check if suite already exists. - if _, err := os.Stat(suiteDir); err == nil { - suiteExists = true + // Treat the suite as already built only when a complete summary.json with + // tests is present. A bare or partial directory — e.g. left behind by a run + // that aborted mid-creation — must be rebuilt; otherwise info.Tests stays + // nil and summary.json gets (re)written with "tests": null. + if data, err := os.ReadFile(filepath.Join(suiteDir, "summary.json")); err == nil { + var existing SuiteInfo + if json.Unmarshal(data, &existing) == nil && len(existing.Tests) > 0 { + suiteExists = true + } } if !suiteExists { @@ -177,6 +187,19 @@ func CreateSuiteOutput( return fmt.Errorf("creating suite dir: %w", err) } + // Attach the source's metadata directory (EEST .meta) as .eest-meta. + // Best-effort: the metadata is auxiliary provenance, so a copy failure + // must not fail the whole suite. + if prepared.MetaDir != "" { + metaDst := filepath.Join(suiteDir, ".eest-meta") + if err := fsutil.CopyDir(prepared.MetaDir, metaDst, owner); err != nil { + log.WithError(err).WithField("src", prepared.MetaDir). + Warn("Failed to copy EEST .meta into suite output") + } else { + log.WithField("dst", metaDst).Debug("Copied EEST .meta into suite output") + } + } + // Copy pre-run steps. // Structure: //pre_run.request (same pattern as tests). for _, f := range prepared.PreRunSteps { @@ -207,7 +230,7 @@ func CreateSuiteOutput( } // Create test directory. - testDir := filepath.Join(suiteDir, test.Name) + testDir := filepath.Join(suiteDir, sanitizeResultPath(test.Name)) if err := fsutil.MkdirAll(testDir, 0755, owner); err != nil { return fmt.Errorf("creating test dir for %s: %w", test.Name, err) } @@ -301,6 +324,13 @@ func CreateSuiteOutput( } } + // Reflect whether the suite carries an .eest-meta directory. Derived from + // the on-disk dir (not just this run's copy) so it stays correct on re-runs + // where the suite already existed and the copy step above was skipped. + if fi, err := os.Stat(filepath.Join(suiteDir, ".eest-meta")); err == nil && fi.IsDir() { + info.EESTMetadata = true + } + // Always write summary.json — metadata (e.g. labels) can change between // runs without affecting the suite hash, so we update it every time. summaryPath := filepath.Join(suiteDir, "summary.json") @@ -318,7 +348,7 @@ func CreateSuiteOutput( mergeOpcodeData(existing.Tests, prepared) lineProvider := func(testName string, step StepKind) []string { - reqPath := filepath.Join(suiteDir, testName, string(step)+".request") + reqPath := filepath.Join(suiteDir, sanitizeResultPath(testName), string(step)+".request") data, err := os.ReadFile(reqPath) if err != nil { // Missing files are normal — most tests don't have setup/cleanup. diff --git a/pkg/executor/suite_test.go b/pkg/executor/suite_test.go index 8bc3cf170..cec0339da 100644 --- a/pkg/executor/suite_test.go +++ b/pkg/executor/suite_test.go @@ -76,6 +76,81 @@ func TestSuiteInfo_BackwardCompat_LoadsOldSummary(t *testing.T) { assert.Nil(t, parsed.Tests[0].PayloadSizes) } +func TestCreateSuiteOutput_CopiesEESTMeta(t *testing.T) { + tmp := t.TempDir() + + // Build a source .meta dir with a top-level file and a nested file. + metaDir := filepath.Join(tmp, "fixtures", ".meta") + require.NoError(t, os.MkdirAll(filepath.Join(metaDir, "assets"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(metaDir, "fixtures.ini"), + []byte("[environment]\npython = 3.12.13\n"), 0o644)) + require.NoError(t, os.WriteFile( + filepath.Join(metaDir, "assets", "style.css"), []byte("body{}"), 0o644)) + + prepared := &PreparedSource{ + MetaDir: metaDir, + Tests: []*TestWithSteps{ + { + Name: "test_meta", + Test: &StepFile{ + Name: "test_meta", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + info := &SuiteInfo{Hash: "abc123"} + + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "abc123", info, prepared, nil)) + + suiteMeta := filepath.Join(tmp, "suites", "abc123", ".eest-meta") + + gotIni, err := os.ReadFile(filepath.Join(suiteMeta, "fixtures.ini")) + require.NoError(t, err) + assert.Contains(t, string(gotIni), "python = 3.12.13") + + gotCSS, err := os.ReadFile(filepath.Join(suiteMeta, "assets", "style.css")) + require.NoError(t, err) + assert.Equal(t, "body{}", string(gotCSS)) + + // summary.json flags the metadata so the UI can surface it. + data, err := os.ReadFile(filepath.Join(tmp, "suites", "abc123", "summary.json")) + require.NoError(t, err) + + var parsed SuiteInfo + require.NoError(t, json.Unmarshal(data, &parsed)) + assert.True(t, parsed.EESTMetadata) +} + +func TestCreateSuiteOutput_NoEESTMetaWhenAbsent(t *testing.T) { + tmp := t.TempDir() + prepared := &PreparedSource{ + Tests: []*TestWithSteps{ + { + Name: "test_nometa", + Test: &StepFile{ + Name: "test_nometa", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + info := &SuiteInfo{Hash: "nometa01"} + + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "nometa01", info, prepared, nil)) + + _, err := os.Stat(filepath.Join(tmp, "suites", "nometa01", ".eest-meta")) + assert.True(t, os.IsNotExist(err)) + + data, err := os.ReadFile(filepath.Join(tmp, "suites", "nometa01", "summary.json")) + require.NoError(t, err) + + var parsed SuiteInfo + require.NoError(t, json.Unmarshal(data, &parsed)) + assert.False(t, parsed.EESTMetadata) +} + func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { tmp := t.TempDir() testLine := minimalDenebRequest(t) diff --git a/pkg/fsutil/fsutil.go b/pkg/fsutil/fsutil.go index 96aa4d5b9..9cc7a1bc9 100644 --- a/pkg/fsutil/fsutil.go +++ b/pkg/fsutil/fsutil.go @@ -97,3 +97,41 @@ func Create(path string, owner *OwnerConfig) (*os.File, error) { return f, nil } + +// CopyDir recursively copies the directory tree rooted at src into dst, +// creating dst (and any missing parents) and applying owner to every directory +// and file it writes. Only regular files and directories are copied; symlinks +// and other special files are skipped. Intended for small auxiliary trees. +func CopyDir(src, dst string, owner *OwnerConfig) error { + entries, err := os.ReadDir(src) + if err != nil { + return fmt.Errorf("reading source dir %q: %w", src, err) + } + + if err := MkdirAll(dst, 0755, owner); err != nil { + return fmt.Errorf("creating dest dir %q: %w", dst, err) + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + switch { + case entry.IsDir(): + if err := CopyDir(srcPath, dstPath, owner); err != nil { + return err + } + case entry.Type().IsRegular(): + data, err := os.ReadFile(srcPath) + if err != nil { + return fmt.Errorf("reading %q: %w", srcPath, err) + } + + if err := WriteFile(dstPath, data, 0644, owner); err != nil { + return fmt.Errorf("writing %q: %w", dstPath, err) + } + } + } + + return nil +} diff --git a/pkg/fsutil/fsutil_test.go b/pkg/fsutil/fsutil_test.go new file mode 100644 index 000000000..505582459 --- /dev/null +++ b/pkg/fsutil/fsutil_test.go @@ -0,0 +1,50 @@ +package fsutil + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyDir(t *testing.T) { + src := t.TempDir() + + // Lay out a small tree: top-level file + a nested dir with a file. + require.NoError(t, os.MkdirAll(filepath.Join(src, "assets"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "fixtures.ini"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "assets", "style.css"), []byte("body{}"), 0o644)) + + dst := filepath.Join(t.TempDir(), "copy") + + require.NoError(t, CopyDir(src, dst, nil)) + + top, err := os.ReadFile(filepath.Join(dst, "fixtures.ini")) + require.NoError(t, err) + assert.Equal(t, "hello", string(top)) + + nested, err := os.ReadFile(filepath.Join(dst, "assets", "style.css")) + require.NoError(t, err) + assert.Equal(t, "body{}", string(nested)) +} + +func TestCopyDir_CreatesMissingParents(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "f.txt"), []byte("x"), 0o644)) + + // Destination parents do not exist yet. + dst := filepath.Join(t.TempDir(), "a", "b", "c") + + require.NoError(t, CopyDir(src, dst, nil)) + + got, err := os.ReadFile(filepath.Join(dst, "f.txt")) + require.NoError(t, err) + assert.Equal(t, "x", string(got)) +} + +func TestCopyDir_MissingSource(t *testing.T) { + err := CopyDir(filepath.Join(t.TempDir(), "does-not-exist"), t.TempDir(), nil) + require.Error(t, err) +} diff --git a/pkg/genesis/override.go b/pkg/genesis/override.go new file mode 100644 index 000000000..c9e48ccd1 --- /dev/null +++ b/pkg/genesis/override.go @@ -0,0 +1,174 @@ +// Package genesis patches client genesis/chainspec JSON to activate forks that +// the file doesn't already schedule — the file-based equivalent of geth's +// --override. flag, for clients that read their fork schedule from the +// genesis. It is shared by the runner (boot genesis) and the eest_payloads +// builder (filler boot genesis) so both use the same vocabulary. +package genesis + +import ( + "bytes" + "encoding/json" + "fmt" +) + +// blobForkOrder lists the blob-bearing forks in activation order. When a fork +// override adds a fork the genesis blobSchedule doesn't yet cover, the new fork +// inherits the schedule of the latest preceding fork present here — geth-family +// clients reject an active fork that carries no blob schedule. +var blobForkOrder = []string{"cancun", "prague", "osaka", "amsterdam"} + +// ApplyForkOverrides patches a geth-format genesis JSON so the given forks +// activate at the given timestamps (fork name → unix-seconds). It is the +// genesis-file equivalent of geth's --override. flag, for clients that +// read their fork schedule from the genesis (besu, reth, ethrex). For each fork +// it sets config.Time and, when a blobSchedule is present but lacks the +// fork, inherits the latest preceding fork's blob parameters. +// +// Only the top-level "config" object is rewritten; every other field round-trips +// verbatim and existing numbers are preserved exactly (so the genesis block hash +// is unchanged). It returns an error if the genesis is not geth-format (has no +// top-level "config" object), since the patch shape is format-specific. +func ApplyForkOverrides(genesis []byte, overrides map[string]uint64) ([]byte, error) { + if len(overrides) == 0 { + return genesis, nil + } + + // Decode only the top level so untouched fields round-trip byte-for-byte. + var top map[string]json.RawMessage + if err := json.Unmarshal(genesis, &top); err != nil { + return nil, fmt.Errorf("parsing genesis json: %w", err) + } + + rawConfig, ok := top["config"] + if !ok { + return nil, fmt.Errorf( + "genesis has no \"config\" object; genesis_fork_override only " + + "supports geth-format genesis files", + ) + } + + // UseNumber keeps existing numbers as json.Number so they re-encode exactly + // rather than via lossy float64. + dec := json.NewDecoder(bytes.NewReader(rawConfig)) + dec.UseNumber() + + cfg := make(map[string]any) + if err := dec.Decode(&cfg); err != nil { + return nil, fmt.Errorf("parsing genesis config: %w", err) + } + + for fork, ts := range overrides { + cfg[fork+"Time"] = ts + inheritBlobSchedule(cfg, fork) + } + + patchedConfig, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis config: %w", err) + } + + top["config"] = patchedConfig + + patched, err := json.Marshal(top) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis: %w", err) + } + + return patched, nil +} + +// ApplyEIPOverrides patches a parity/nethermind-format chainspec so the given +// EIPs activate at timestamp. It is the parity-format counterpart of +// ApplyForkOverrides: parity chainspecs schedule forks per-EIP +// (params.eipTransitionTimestamp) rather than by fork name, so the +// devnet-specific EIP list comes from config. +// +// Only the "params" object is rewritten; every other field round-trips verbatim. +// It returns an error if the genesis is not parity-format (has no top-level +// "params" object). +func ApplyEIPOverrides(genesis []byte, timestamp uint64, eips []uint64) ([]byte, error) { + if len(eips) == 0 { + return genesis, nil + } + + var top map[string]json.RawMessage + if err := json.Unmarshal(genesis, &top); err != nil { + return nil, fmt.Errorf("parsing genesis json: %w", err) + } + + rawParams, ok := top["params"] + if !ok { + return nil, fmt.Errorf( + "genesis has no \"params\" object; genesis_eip_override only " + + "supports parity/nethermind-format chainspecs", + ) + } + + dec := json.NewDecoder(bytes.NewReader(rawParams)) + dec.UseNumber() + + params := make(map[string]any) + if err := dec.Decode(¶ms); err != nil { + return nil, fmt.Errorf("parsing genesis params: %w", err) + } + + // Parity transition timestamps are hex-encoded strings (e.g. "0x1"). + ts := fmt.Sprintf("0x%x", timestamp) + for _, eip := range eips { + params[fmt.Sprintf("eip%dTransitionTimestamp", eip)] = ts + } + + patchedParams, err := json.Marshal(params) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis params: %w", err) + } + + top["params"] = patchedParams + + patched, err := json.Marshal(top) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis: %w", err) + } + + return patched, nil +} + +// inheritBlobSchedule ensures cfg.blobSchedule covers fork by copying the +// schedule of the latest preceding fork in blobForkOrder. It is a no-op when the +// genesis has no blobSchedule, the fork already has a schedule, the fork is +// unknown to blobForkOrder, or no preceding schedule exists. +func inheritBlobSchedule(cfg map[string]any, fork string) { + schedule, ok := cfg["blobSchedule"].(map[string]any) + if !ok { + return + } + + if _, exists := schedule[fork]; exists { + return + } + + forkIdx := indexOf(blobForkOrder, fork) + if forkIdx < 0 { + return + } + + // Walk backwards to the nearest preceding fork that has a schedule. + for i := forkIdx - 1; i >= 0; i-- { + if prev, ok := schedule[blobForkOrder[i]]; ok { + schedule[fork] = prev + + return + } + } +} + +// indexOf returns the index of target in list, or -1 if absent. +func indexOf(list []string, target string) int { + for i, v := range list { + if v == target { + return i + } + } + + return -1 +} diff --git a/pkg/genesis/override_test.go b/pkg/genesis/override_test.go new file mode 100644 index 000000000..c676744f3 --- /dev/null +++ b/pkg/genesis/override_test.go @@ -0,0 +1,180 @@ +package genesis + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyForkOverrides(t *testing.T) { + t.Run("no overrides returns input unchanged", func(t *testing.T) { + in := []byte(`{"config":{"osakaTime":0}}`) + + out, err := ApplyForkOverrides(in, nil) + + require.NoError(t, err) + assert.Equal(t, in, out) + }) + + t.Run("non-geth genesis errors", func(t *testing.T) { + in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"}}`) + + _, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "geth-format") + }) + + t.Run("sets fork time and inherits blob schedule", func(t *testing.T) { + in := []byte(`{ + "config": { + "chainId": 1337, + "osakaTime": 0, + "blobSchedule": { + "osaka": {"baseFeeUpdateFraction": 5007716, "max": 12, "target": 9} + } + }, + "gasLimit": "0x11e1a300" + }`) + + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + cfg := decodeConfig(t, out) + assert.EqualValues(t, 1, cfg["amsterdamTime"]) + + bs, ok := cfg["blobSchedule"].(map[string]any) + require.True(t, ok) + + amsterdam, ok := bs["amsterdam"].(map[string]any) + require.True(t, ok, "amsterdam should inherit osaka's schedule") + assert.EqualValues(t, 12, amsterdam["max"]) + assert.EqualValues(t, 9, amsterdam["target"]) + + var top map[string]any + require.NoError(t, json.Unmarshal(out, &top)) + assert.Equal(t, "0x11e1a300", top["gasLimit"]) + }) + + t.Run("inherits latest preceding fork when several exist", func(t *testing.T) { + in := []byte(`{"config":{"blobSchedule":{ + "cancun":{"max":6}, + "prague":{"max":9}, + "osaka":{"max":12} + }}}`) + + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + bs := decodeConfig(t, out)["blobSchedule"].(map[string]any) + amsterdam := bs["amsterdam"].(map[string]any) + assert.EqualValues(t, 12, amsterdam["max"], "should inherit osaka, the latest") + }) + + t.Run("does not overwrite an existing fork schedule", func(t *testing.T) { + in := []byte(`{"config":{"blobSchedule":{ + "osaka":{"max":12}, + "amsterdam":{"max":99} + }}}`) + + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + bs := decodeConfig(t, out)["blobSchedule"].(map[string]any) + amsterdam := bs["amsterdam"].(map[string]any) + assert.EqualValues(t, 99, amsterdam["max"]) + }) + + t.Run("no blob schedule only sets the time", func(t *testing.T) { + in := []byte(`{"config":{"osakaTime":0}}`) + + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + cfg := decodeConfig(t, out) + assert.EqualValues(t, 1, cfg["amsterdamTime"]) + _, hasBlob := cfg["blobSchedule"] + assert.False(t, hasBlob) + }) + + t.Run("preserves large integers without float corruption", func(t *testing.T) { + in := []byte(`{"config":{"terminalTotalDifficulty":115792089237316195423570985008687907853269984665640564039457584007913129639936}}`) + + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + assert.Contains(t, string(out), + "115792089237316195423570985008687907853269984665640564039457584007913129639936") + }) +} + +func TestApplyEIPOverrides(t *testing.T) { + t.Run("no eips returns input unchanged", func(t *testing.T) { + in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"}}`) + + out, err := ApplyEIPOverrides(in, 1, nil) + require.NoError(t, err) + assert.Equal(t, in, out) + }) + + t.Run("non-parity genesis errors", func(t *testing.T) { + in := []byte(`{"config":{"osakaTime":0}}`) + + _, err := ApplyEIPOverrides(in, 1, []uint64{7928}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "parity") + }) + + t.Run("sets eip transition timestamps as hex", func(t *testing.T) { + in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"},"name":"x"}`) + + out, err := ApplyEIPOverrides(in, 1, []uint64{7928, 8037}) + require.NoError(t, err) + + params := decodeParams(t, out) + assert.Equal(t, "0x1", params["eip7928TransitionTimestamp"]) + assert.Equal(t, "0x1", params["eip8037TransitionTimestamp"]) + assert.Equal(t, "0x0", params["eip7825TransitionTimestamp"]) + + var top map[string]any + require.NoError(t, json.Unmarshal(out, &top)) + assert.Equal(t, "x", top["name"]) + }) + + t.Run("encodes larger timestamps as hex", func(t *testing.T) { + in := []byte(`{"params":{}}`) + + out, err := ApplyEIPOverrides(in, 1769856767, []uint64{7928}) + require.NoError(t, err) + + params := decodeParams(t, out) + assert.Equal(t, "0x697ddeff", params["eip7928TransitionTimestamp"]) + }) +} + +func decodeConfig(t *testing.T, genesis []byte) map[string]any { + t.Helper() + + var top map[string]json.RawMessage + require.NoError(t, json.Unmarshal(genesis, &top)) + + var cfg map[string]any + require.NoError(t, json.Unmarshal(top["config"], &cfg)) + + return cfg +} + +func decodeParams(t *testing.T, genesis []byte) map[string]any { + t.Helper() + + var top map[string]json.RawMessage + require.NoError(t, json.Unmarshal(genesis, &top)) + + var params map[string]any + require.NoError(t, json.Unmarshal(top["params"], ¶ms)) + + return params +} diff --git a/pkg/gitrepo/gitrepo.go b/pkg/gitrepo/gitrepo.go new file mode 100644 index 000000000..fc88c75f3 --- /dev/null +++ b/pkg/gitrepo/gitrepo.go @@ -0,0 +1,164 @@ +// Package gitrepo clones git repositories into an on-disk cache and keeps the +// cached checkout at a requested branch, tag, or commit. It is the shared +// implementation behind the test-source git clones and the EEST fill-repo +// clone, so recurring clones of the same repo are avoided. +package gitrepo + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +// CloneOrUpdate ensures repo is cloned under cacheDir and checked out at +// version (a branch, tag, or commit hash), reusing an existing cached clone +// when present. It returns the local path of the checkout. The cache key is +// derived from the repo URL, so different versions of the same repo share one +// directory (the checkout is moved to the requested version). +func CloneOrUpdate(ctx context.Context, log logrus.FieldLogger, repo, version, cacheDir string) (string, error) { + localPath := filepath.Join(cacheDir, hashRepoURL(repo)) + + log = log.WithFields(logrus.Fields{ + "repo": repo, + "version": version, + "path": localPath, + }) + + if _, err := os.Stat(localPath); os.IsNotExist(err) { + log.Info("Cloning repository") + + if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { + return "", fmt.Errorf("creating cache directory: %w", err) + } + + if looksLikeCommitHash(version) { + // Commit hashes can't be used with --branch, so init + fetch. + if err := cloneByCommitHash(ctx, repo, version, localPath); err != nil { + return "", err + } + } else if err := gitRun(ctx, "git", "clone", + "--depth=1", "--branch", version, "--single-branch", repo, localPath); err != nil { + return "", fmt.Errorf("cloning repository: %w", err) + } + + if err := initSubmodules(ctx, localPath); err != nil { + return "", err + } + + return localPath, nil + } + + // Cached clone exists. For commit hashes, skip the fetch when HEAD already + // matches the requested commit. + if looksLikeCommitHash(version) { + if sha, err := HeadSHA(ctx, localPath); err == nil && strings.HasPrefix(sha, version) { + log.Info("Cached repository already at requested version") + + if err := initSubmodules(ctx, localPath); err != nil { + return "", err + } + + return localPath, nil + } + } + + log.Info("Updating cached repository") + + if err := gitRun(ctx, "git", "-C", localPath, "fetch", "--depth=1", "origin", version); err != nil { + return "", fmt.Errorf("fetching version: %w", err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "checkout", "FETCH_HEAD"); err != nil { + return "", fmt.Errorf("checking out version: %w", err) + } + + if err := initSubmodules(ctx, localPath); err != nil { + return "", err + } + + return localPath, nil +} + +// HeadSHA returns the current HEAD commit hash for the repository at repoPath. +func HeadSHA(ctx context.Context, repoPath string) (string, error) { + out, err := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", "HEAD").Output() + if err != nil { + return "", fmt.Errorf("getting commit SHA: %w", err) + } + + return strings.TrimSpace(string(out)), nil +} + +// cloneByCommitHash initializes a repo and fetches a specific commit hash, then +// checks it out (a commit hash cannot be passed to `git clone --branch`). +func cloneByCommitHash(ctx context.Context, repo, version, localPath string) error { + if err := gitRun(ctx, "git", "init", localPath); err != nil { + return fmt.Errorf("initializing repository: %w", err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "remote", "add", "origin", repo); err != nil { + return fmt.Errorf("adding remote: %w", err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "fetch", "--depth=1", "origin", version); err != nil { + return fmt.Errorf("fetching commit %s: %w", version, err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "checkout", "FETCH_HEAD"); err != nil { + return fmt.Errorf("checking out commit %s: %w", version, err) + } + + return nil +} + +// initSubmodules fetches any git submodules declared by the checkout. It is a +// no-op for repos without submodules, so it is safe to call after every +// clone/checkout (some EEST test suites vendor fixtures as submodules and fail +// at fill time with "run git submodule update" if they are absent). +func initSubmodules(ctx context.Context, localPath string) error { + if err := gitRun(ctx, "git", "-C", localPath, + "submodule", "update", "--init", "--recursive"); err != nil { + return fmt.Errorf("updating submodules: %w", err) + } + + return nil +} + +// gitRun runs a git command, forwarding its output to the process stdio. +func gitRun(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} + +// looksLikeCommitHash returns true if s looks like a git commit hash +// (7-40 hex characters). +func looksLikeCommitHash(s string) bool { + if len(s) < 7 || len(s) > 40 { + return false + } + + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { + return false + } + } + + return true +} + +// hashRepoURL creates a short stable hash of the repository URL for caching. +func hashRepoURL(url string) string { + hash := sha256.Sum256([]byte(url)) + + return hex.EncodeToString(hash[:8]) +} diff --git a/pkg/gitrepo/gitrepo_test.go b/pkg/gitrepo/gitrepo_test.go new file mode 100644 index 000000000..22f223360 --- /dev/null +++ b/pkg/gitrepo/gitrepo_test.go @@ -0,0 +1,90 @@ +package gitrepo + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLooksLikeCommitHash(t *testing.T) { + tests := []struct { + name string + input string + expected bool + }{ + {name: "full sha1", input: "e5011aa5f75d7a1722481f25408347fadfb7fd3c", expected: true}, + {name: "short hash 7 chars", input: "e5011aa", expected: true}, + {name: "uppercase hex", input: "E5011AA5F75D7A17", expected: true}, + {name: "branch name", input: "main", expected: false}, + {name: "branch with slash", input: "feature/foo", expected: false}, + {name: "tag semver", input: "v1.0.0", expected: false}, + {name: "too short 6 chars", input: "e5011a", expected: false}, + {name: "empty string", input: "", expected: false}, + {name: "hex with non-hex char", input: "e5011gg", expected: false}, + {name: "7 char all digits", input: "1234567", expected: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, looksLikeCommitHash(tt.input)) + }) + } +} + +func TestHashRepoURLStable(t *testing.T) { + a := hashRepoURL("https://example.com/repo.git") + b := hashRepoURL("https://example.com/repo.git") + c := hashRepoURL("https://example.com/other.git") + assert.Equal(t, a, b, "same URL must hash the same") + assert.NotEqual(t, a, c, "different URLs must differ") + assert.Len(t, a, 16, "8-byte hex") +} + +// git runs a git command in dir, failing the test on error. +func git(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t") + out, err := cmd.CombinedOutput() + require.NoError(t, err, "git %v: %s", args, out) +} + +func TestCloneOrUpdate(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + // Build a tiny local upstream repo with a branch. + upstream := t.TempDir() + git(t, upstream, "init", "-b", "main") + require.NoError(t, os.WriteFile(filepath.Join(upstream, "README"), []byte("hello"), 0o644)) + git(t, upstream, "add", ".") + git(t, upstream, "commit", "-m", "init") + + cache := t.TempDir() + ctx := context.Background() + log := logrus.New() + + // First call clones. + path, err := CloneOrUpdate(ctx, log, upstream, "main", cache) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(path, "README")) + assert.Equal(t, cache, filepath.Dir(path), "checkout lives directly under the cache dir") + + sha, err := HeadSHA(ctx, path) + require.NoError(t, err) + assert.Len(t, sha, 40) + + // Second call reuses the same cached path. + path2, err := CloneOrUpdate(ctx, log, upstream, "main", cache) + require.NoError(t, err) + assert.Equal(t, path, path2) +} diff --git a/pkg/podman/podman.go b/pkg/podman/podman.go index 2d1acc412..3aa00f46e 100644 --- a/pkg/podman/podman.go +++ b/pkg/podman/podman.go @@ -164,6 +164,27 @@ func (m *manager) EnsureNetwork(ctx context.Context, name string) error { return nil } +// NetworkExists reports whether a network with the given name exists. +func (m *manager) NetworkExists(ctx context.Context, name string) (bool, error) { + conn, cancel := m.connWithCtx(ctx) + defer cancel() + + nets, err := network.List(conn, &network.ListOptions{ + Filters: map[string][]string{"name": {name}}, + }) + if err != nil { + return false, fmt.Errorf("listing networks: %w", err) + } + + for _, n := range nets { + if n.Name == name { + return true, nil + } + } + + return false, nil +} + // RemoveNetwork removes a Podman network. func (m *manager) RemoveNetwork(ctx context.Context, name string) error { conn, cancel := m.connWithCtx(ctx) diff --git a/pkg/runner/lifecycle.go b/pkg/runner/lifecycle.go index af3dbe2a1..e3613f487 100644 --- a/pkg/runner/lifecycle.go +++ b/pkg/runner/lifecycle.go @@ -22,6 +22,7 @@ import ( "github.com/ethpandaops/benchmarkoor/pkg/docker" "github.com/ethpandaops/benchmarkoor/pkg/executor" "github.com/ethpandaops/benchmarkoor/pkg/fsutil" + "github.com/ethpandaops/benchmarkoor/pkg/genesis" "github.com/ethpandaops/benchmarkoor/pkg/podman" "github.com/ethpandaops/benchmarkoor/pkg/version" "github.com/shirou/gopsutil/v4/cpu" @@ -111,6 +112,13 @@ func (r *runner) runContainerLifecycle( Source: prepared.MountPath, Target: containerDir, } + + // Surface any state-actor provenance dropped at the snapshot root + // (state-actor-manifest.json + its content-addressed spec sidecar) into + // the run output under .state-actor/ so the UI can render the State Actor + // Configuration. Read from the original source dir (the manifest lives + // there, not necessarily in the copied/overlaid datadir). + copyStateActorFiles(log, datadirCfg.SourceDir, runResultsDir, r.cfg.ResultsOwner) } else if r.cfg.FullConfig != nil && r.cfg.FullConfig.GetRollbackStrategy(instance) == config.RollbackStrategyCheckpointRestore { // Checkpoint-restore without a pre-populated datadir uses a bind @@ -187,6 +195,59 @@ func (r *runner) runContainerLifecycle( log.Info("No genesis configured, skipping genesis setup") } + // Apply per-instance genesis fork-time overrides (the genesis-file + // equivalent of geth's --override. for clients that read forks from + // the genesis, e.g. besu/reth/ethrex). + if len(instance.GenesisForkOverride) > 0 { + if len(genesisContent) == 0 { + return fmt.Errorf( + "instance %s sets genesis_fork_override but has no genesis file", + instance.ID, + ) + } + + patched, overrideErr := genesis.ApplyForkOverrides( + genesisContent, instance.GenesisForkOverride, + ) + if overrideErr != nil { + return fmt.Errorf( + "applying genesis fork override for %s: %w", instance.ID, overrideErr, + ) + } + + genesisContent = patched + + log.WithField("forks", instance.GenesisForkOverride). + Info("Applied genesis fork-time overrides") + } + + // Apply per-instance genesis EIP overrides (parity/nethermind-format + // chainspecs schedule forks per-EIP rather than by fork name). + if instance.GenesisEIPOverride != nil && len(instance.GenesisEIPOverride.EIPs) > 0 { + if len(genesisContent) == 0 { + return fmt.Errorf( + "instance %s sets genesis_eip_override but has no genesis file", + instance.ID, + ) + } + + patched, overrideErr := genesis.ApplyEIPOverrides( + genesisContent, instance.GenesisEIPOverride.Timestamp, instance.GenesisEIPOverride.EIPs, + ) + if overrideErr != nil { + return fmt.Errorf( + "applying genesis eip override for %s: %w", instance.ID, overrideErr, + ) + } + + genesisContent = patched + + log.WithFields(logrus.Fields{ + "eips": instance.GenesisEIPOverride.EIPs, + "timestamp": instance.GenesisEIPOverride.Timestamp, + }).Info("Applied genesis EIP-time overrides") + } + // Fail if neither genesis nor datadir is configured. if genesisSource == "" && !useDataDir { return fmt.Errorf( @@ -201,7 +262,7 @@ func (r *runner) runContainerLifecycle( // Create temp files for genesis and JWT. tempDir, err := os.MkdirTemp( - r.cfg.TmpCacheDir, "benchmarkoor-"+instance.ID+"-", + r.cfg.CacheDir, "benchmarkoor-"+instance.ID+"-", ) if err != nil { return fmt.Errorf("creating temp directory: %w", err) @@ -1391,3 +1452,70 @@ func writeRunConfig(resultsDir string, cfg *RunConfig, owner *fsutil.OwnerConfig return nil } + +// stateActorFilePrefix matches the provenance files a recent state-actor drops +// at the datadir root: state-actor-manifest.json and its content-addressed spec +// sidecar (state-actor-spec-.yaml). +const stateActorFilePrefix = "state-actor-" + +// copyStateActorFiles copies any state-actor provenance files from the snapshot +// source dir's root into /.state-actor/. Best-effort: a missing +// source, no matching files, or copy errors are logged but never fail the run — +// the metadata is auxiliary and only present for snapshots built by a recent +// state-actor. +func copyStateActorFiles( + log logrus.FieldLogger, + sourceDir, runResultsDir string, + owner *fsutil.OwnerConfig, +) { + if sourceDir == "" { + return + } + + entries, err := os.ReadDir(sourceDir) + if err != nil { + log.WithError(err).Debug("state-actor: reading snapshot source dir") + + return + } + + names := make([]string, 0, 2) + + for _, e := range entries { + if !e.IsDir() && strings.HasPrefix(e.Name(), stateActorFilePrefix) { + names = append(names, e.Name()) + } + } + + if len(names) == 0 { + return + } + + dst := filepath.Join(runResultsDir, ".state-actor") + if err := fsutil.MkdirAll(dst, 0o755, owner); err != nil { + log.WithError(err).Warn("state-actor: creating .state-actor output dir") + + return + } + + copied := 0 + + for _, name := range names { + data, readErr := os.ReadFile(filepath.Join(sourceDir, name)) + if readErr != nil { + log.WithError(readErr).WithField("file", name).Warn("state-actor: reading file") + + continue + } + + if writeErr := fsutil.WriteFile(filepath.Join(dst, name), data, 0o644, owner); writeErr != nil { + log.WithError(writeErr).WithField("file", name).Warn("state-actor: writing file") + + continue + } + + copied++ + } + + log.WithField("count", copied).Debug("Copied state-actor provenance into run output") +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index f42f9568c..1ac1fa74d 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -60,7 +60,7 @@ type Config struct { GenesisURLs map[string]string DataDirs map[string]*config.DataDirConfig TmpDataDir string // Directory for temporary datadir copies (empty = system default) - TmpCacheDir string // Directory for temporary cache files (empty = system default) + CacheDir string // Shared on-disk cache dir (git/archive clones, etc.) ReadyTimeout time.Duration TestFilter string FullConfig *config.Config // Full config for resolving per-instance settings diff --git a/pkg/runner/stateactor_test.go b/pkg/runner/stateactor_test.go new file mode 100644 index 000000000..94a4bcbde --- /dev/null +++ b/pkg/runner/stateactor_test.go @@ -0,0 +1,64 @@ +package runner + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyStateActorFiles(t *testing.T) { + src := t.TempDir() + // state-actor provenance at the datadir root. + require.NoError(t, os.WriteFile( + filepath.Join(src, "state-actor-manifest.json"), []byte(`{"schema_version":1}`), 0o644)) + require.NoError(t, os.WriteFile( + filepath.Join(src, "state-actor-spec-abc123.yaml"), []byte("entities: []\n"), 0o644)) + // Unrelated files + a dir must not be copied. + require.NoError(t, os.WriteFile(filepath.Join(src, "geth-genesis.json"), []byte("{}"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(src, "state-actor-not-a-file"), 0o755)) + + runDir := t.TempDir() + + copyStateActorFiles(logrus.New(), src, runDir, nil) + + metaDir := filepath.Join(runDir, ".state-actor") + + got, err := os.ReadFile(filepath.Join(metaDir, "state-actor-manifest.json")) + require.NoError(t, err) + assert.Equal(t, `{"schema_version":1}`, string(got)) + + _, err = os.Stat(filepath.Join(metaDir, "state-actor-spec-abc123.yaml")) + require.NoError(t, err) + + // Non-matching file is not copied. + _, err = os.Stat(filepath.Join(metaDir, "geth-genesis.json")) + assert.True(t, os.IsNotExist(err)) +} + +func TestCopyStateActorFiles_NoFilesNoDir(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "geth-genesis.json"), []byte("{}"), 0o644)) + + runDir := t.TempDir() + + copyStateActorFiles(logrus.New(), src, runDir, nil) + + // With no state-actor files present, the .state-actor dir is not created. + _, err := os.Stat(filepath.Join(runDir, ".state-actor")) + assert.True(t, os.IsNotExist(err)) +} + +func TestCopyStateActorFiles_MissingSource(t *testing.T) { + // Must not panic or create anything for a missing/empty source. + runDir := t.TempDir() + + copyStateActorFiles(logrus.New(), filepath.Join(t.TempDir(), "nope"), runDir, nil) + copyStateActorFiles(logrus.New(), "", runDir, nil) + + _, err := os.Stat(filepath.Join(runDir, ".state-actor")) + assert.True(t, os.IsNotExist(err)) +} diff --git a/pkg/stats/docker_reader.go b/pkg/stats/docker_reader.go index 7551719a7..06f4c8856 100644 --- a/pkg/stats/docker_reader.go +++ b/pkg/stats/docker_reader.go @@ -4,23 +4,60 @@ import ( "context" "encoding/json" "fmt" + "io" + "sync" + "time" "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" "github.com/sirupsen/logrus" ) -// dockerReader implements Reader using Docker Stats API. +const ( + // firstSampleTimeout bounds how long newDockerReader waits for the first + // streamed stats sample before returning. The stream usually delivers a + // sample within ~1s; this is a one-time wait per reader, not per ReadStats. + firstSampleTimeout = 5 * time.Second + + // streamRetryDelay is the backoff between stats-stream reconnect attempts + // (e.g. after a transient daemon error or a container restart). + streamRetryDelay = 1 * time.Second + + // closeTimeout bounds how long Close waits for the streaming goroutine to + // exit after cancellation. + closeTimeout = 2 * time.Second +) + +// dockerReader implements Reader using the Docker Stats API. +// +// A one-shot stats request (ContainerStats with stream=false) blocks on the +// daemon's next collection cycle — ~1-2s on Docker Desktop/OrbStack for macOS. +// Issuing one per ReadStats made benchmark runs crawl and, when read inside a +// timing window, inflated the measured RPC duration. Instead this reader opens a +// single streaming stats connection in the background and caches the most recent +// sample; ReadStats returns that cached snapshot without touching the daemon, so +// it is effectively instant. The trade-off is granularity: samples refresh at +// the daemon's cadence (~1/s), so per-RPC deltas for sub-second calls are coarse +// — acceptable on the macOS fallback path, where the cgroup reader is unavailable. type dockerReader struct { log logrus.FieldLogger - client *client.Client containerID string + + cancel context.CancelFunc + done chan struct{} + + mu sync.RWMutex + latest *Stats + + readyOnce sync.Once + ready chan struct{} // closed once the first sample has been cached } // Ensure interface compliance. var _ Reader = (*dockerReader)(nil) -// newDockerReader creates a new Docker Stats API reader. +// newDockerReader creates a streaming Docker Stats API reader and waits briefly +// for the first sample so the initial ReadStats has data. func newDockerReader( log logrus.FieldLogger, dockerClient *client.Client, @@ -30,11 +67,25 @@ func newDockerReader( return nil, fmt.Errorf("docker client is nil") } - return &dockerReader{ + ctx, cancel := context.WithCancel(context.Background()) + + r := &dockerReader{ log: log.WithField("reader", "docker"), - client: dockerClient, containerID: containerID, - }, nil + cancel: cancel, + done: make(chan struct{}), + ready: make(chan struct{}), + } + + go r.stream(ctx, dockerClient) + + select { + case <-r.ready: + case <-time.After(firstSampleTimeout): + r.log.Warn("Timed out waiting for first Docker stats sample; metrics may be delayed") + } + + return r, nil } // Type returns the reader implementation type. @@ -42,43 +93,120 @@ func (r *dockerReader) Type() string { return "docker" } -// Close releases any resources held by the reader. +// Close cancels the streaming goroutine and waits briefly for it to exit. func (r *dockerReader) Close() error { + r.cancel() + + select { + case <-r.done: + case <-time.After(closeTimeout): + } + return nil } -// ReadStats returns current resource metrics using Docker Stats API. +// ReadStats returns the most recent cached sample. It never blocks on the Docker +// daemon. Returns an error only if no sample has been collected yet. func (r *dockerReader) ReadStats() (*Stats, error) { - // Use one-shot stats (stream=false) for lower overhead. - ctx := context.Background() + r.mu.RLock() + defer r.mu.RUnlock() + + if r.latest == nil { + return nil, fmt.Errorf("no docker stats sample available yet") + } + + // Return a copy so callers cannot mutate the cached snapshot. + snapshot := *r.latest + + return &snapshot, nil +} + +// stream maintains a long-lived ContainerStats connection, decoding samples into +// the cache until the context is cancelled. It reconnects on transient errors +// and container restarts. +func (r *dockerReader) stream(ctx context.Context, dockerClient *client.Client) { + defer close(r.done) + + for ctx.Err() == nil { + resp, err := dockerClient.ContainerStats(ctx, r.containerID, true) + if err != nil { + if ctx.Err() != nil { + return + } + + r.log.WithError(err).Debug("Docker stats stream failed; retrying") + + if !sleepCtx(ctx, streamRetryDelay) { + return + } + + continue + } + + r.consume(ctx, resp.Body) + _ = resp.Body.Close() + + if ctx.Err() != nil { + return + } - statsResp, err := r.client.ContainerStats(ctx, r.containerID, false) - if err != nil { - return nil, fmt.Errorf("getting container stats: %w", err) + // The stream ended (e.g. container stop/restart); back off and retry. + if !sleepCtx(ctx, streamRetryDelay) { + return + } } - defer func() { _ = statsResp.Body.Close() }() +} + +// consume decodes stats samples from a single stream connection until it ends or +// the context is cancelled, updating the cache for each sample. +func (r *dockerReader) consume(ctx context.Context, body io.Reader) { + dec := json.NewDecoder(body) + + for ctx.Err() == nil { + var ds container.StatsResponse + if err := dec.Decode(&ds); err != nil { + return + } - var dockerStats container.StatsResponse - if err := json.NewDecoder(statsResp.Body).Decode(&dockerStats); err != nil { - return nil, fmt.Errorf("decoding stats response: %w", err) + r.update(&ds) } +} - stats := &Stats{ +// update caches a freshly decoded sample and signals readiness on the first one. +func (r *dockerReader) update(ds *container.StatsResponse) { + snapshot := &Stats{ // Memory usage in bytes. - Memory: dockerStats.MemoryStats.Usage, - // CPU usage: Docker reports in nanoseconds, convert to microseconds. - CPUUsage: dockerStats.CPUStats.CPUUsage.TotalUsage / 1000, + Memory: ds.MemoryStats.Usage, + // CPU usage: Docker reports nanoseconds, convert to microseconds. + CPUUsage: ds.CPUStats.CPUUsage.TotalUsage / 1000, } - // Sum disk I/O from BlkioStats. - stats.DiskRead, stats.DiskWrite = r.extractBlkioBytes(&dockerStats) - stats.DiskReadOps, stats.DiskWriteOps = r.extractBlkioOps(&dockerStats) + snapshot.DiskRead, snapshot.DiskWrite = extractBlkioBytes(ds) + snapshot.DiskReadOps, snapshot.DiskWriteOps = extractBlkioOps(ds) + + r.mu.Lock() + r.latest = snapshot + r.mu.Unlock() - return stats, nil + r.readyOnce.Do(func() { close(r.ready) }) +} + +// sleepCtx sleeps for d or until ctx is cancelled. It returns false if the +// context was cancelled (caller should stop). +func sleepCtx(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } } // extractBlkioBytes extracts read/write bytes from BlkioStats. -func (r *dockerReader) extractBlkioBytes(stats *container.StatsResponse) (readBytes, writeBytes uint64) { +func extractBlkioBytes(stats *container.StatsResponse) (readBytes, writeBytes uint64) { for _, entry := range stats.BlkioStats.IoServiceBytesRecursive { switch entry.Op { case "Read", "read": @@ -92,7 +220,7 @@ func (r *dockerReader) extractBlkioBytes(stats *container.StatsResponse) (readBy } // extractBlkioOps extracts read/write I/O operations from BlkioStats. -func (r *dockerReader) extractBlkioOps(stats *container.StatsResponse) (readOps, writeOps uint64) { +func extractBlkioOps(stats *container.StatsResponse) (readOps, writeOps uint64) { for _, entry := range stats.BlkioStats.IoServicedRecursive { switch entry.Op { case "Read", "read": diff --git a/pkg/stats/docker_reader_test.go b/pkg/stats/docker_reader_test.go new file mode 100644 index 000000000..e4b8c04a4 --- /dev/null +++ b/pkg/stats/docker_reader_test.go @@ -0,0 +1,107 @@ +package stats + +import ( + "sync" + "testing" + + "github.com/docker/docker/api/types/container" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestReader builds a dockerReader without starting the streaming goroutine, +// so the cache logic can be exercised without a Docker daemon. +func newTestReader() *dockerReader { + return &dockerReader{ + log: logrus.New(), + ready: make(chan struct{}), + } +} + +func TestDockerReader_ReadStatsBeforeFirstSample(t *testing.T) { + r := newTestReader() + + _, err := r.ReadStats() + require.Error(t, err, "ReadStats must error until the first sample is cached") +} + +func TestDockerReader_UpdateCachesSampleAndSignalsReady(t *testing.T) { + r := newTestReader() + + ds := &container.StatsResponse{} + ds.MemoryStats.Usage = 2048 + ds.CPUStats.CPUUsage.TotalUsage = 5000 // nanoseconds + ds.BlkioStats.IoServiceBytesRecursive = []container.BlkioStatEntry{ + {Op: "Read", Value: 100}, + {Op: "Write", Value: 200}, + } + ds.BlkioStats.IoServicedRecursive = []container.BlkioStatEntry{ + {Op: "Read", Value: 3}, + {Op: "Write", Value: 4}, + } + + r.update(ds) + + // readyOnce should have closed the ready channel. + select { + case <-r.ready: + default: + t.Fatal("ready channel was not closed after first update") + } + + got, err := r.ReadStats() + require.NoError(t, err) + assert.Equal(t, uint64(2048), got.Memory) + assert.Equal(t, uint64(5), got.CPUUsage, "nanoseconds should convert to microseconds") + assert.Equal(t, uint64(100), got.DiskRead) + assert.Equal(t, uint64(200), got.DiskWrite) + assert.Equal(t, uint64(3), got.DiskReadOps) + assert.Equal(t, uint64(4), got.DiskWriteOps) +} + +func TestDockerReader_ReadStatsReturnsCopy(t *testing.T) { + r := newTestReader() + + ds := &container.StatsResponse{} + ds.MemoryStats.Usage = 1000 + r.update(ds) + + first, err := r.ReadStats() + require.NoError(t, err) + + // Mutating the returned snapshot must not affect the cached sample. + first.Memory = 9999 + + second, err := r.ReadStats() + require.NoError(t, err) + assert.Equal(t, uint64(1000), second.Memory, "cached sample must be isolated from callers") +} + +func TestDockerReader_UpdateReadConcurrent(t *testing.T) { + r := newTestReader() + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for i := range 1000 { + ds := &container.StatsResponse{} + ds.MemoryStats.Usage = uint64(i) + r.update(ds) + } + }() + + go func() { + defer wg.Done() + + for range 1000 { + _, _ = r.ReadStats() + } + }() + + wg.Wait() +} diff --git a/ui/src/api/hooks/useStateActorManifest.ts b/ui/src/api/hooks/useStateActorManifest.ts new file mode 100644 index 000000000..52b63fb98 --- /dev/null +++ b/ui/src/api/hooks/useStateActorManifest.ts @@ -0,0 +1,23 @@ +import { useQuery } from '@tanstack/react-query' +import { fetchData } from '../client' +import type { StateActorManifest } from '../types' + +// useStateActorManifest fetches a run's state-actor provenance manifest. It is +// optional — only present when the run's snapshot was built by a state-actor +// that emits it — so a 404 resolves to null instead of erroring. +export function useStateActorManifest(runId: string, enabled = true) { + return useQuery({ + queryKey: ['run', runId, 'state-actor-manifest'], + queryFn: async () => { + const { data, status } = await fetchData( + `runs/${runId}/.state-actor/state-actor-manifest.json`, + ) + if (!data) { + if (status === 404) return null + throw new Error(`Failed to fetch state-actor manifest: ${status}`) + } + return data + }, + enabled: !!runId && enabled, + }) +} diff --git a/ui/src/api/types.ts b/ui/src/api/types.ts index 154ea67cf..8daabf9eb 100644 --- a/ui/src/api/types.ts +++ b/ui/src/api/types.ts @@ -189,6 +189,55 @@ export interface RunConfig { } } +// .state-actor/state-actor-manifest.json per run (present only when the run's +// snapshot was built by a state-actor that emits the manifest). Mirrors the +// state-actor manifest schema (github.com/ethereum/state-actor). +export interface StateActorManifest { + schema_version: number + state_actor: { + version: string + go_version: string + os: string + arch: string + vcs_revision?: string + vcs_time?: string + vcs_modified: boolean + } + generated_at: string + command: string[] + flags: { + client: string + db: string + seed: number + seed_input: number + fork: string + fork_input: string + chain_id: number + gas_limit: number + timestamp: number + extra_data?: string + target_size?: string + binary_trie: boolean + group_depth: number + archive: boolean + spec_path?: string + } + spec?: { + input_path: string + sha256: string + output_file: string + } + result?: { + state_root: string + accounts_created: number + contracts_created: number + storage_slots: number + total_db_size_bytes: number + elapsed_ms: number + } + reproduced_from?: string +} + export interface SystemInfo { hostname: string os: string @@ -439,6 +488,9 @@ export interface SuiteInfo { } pre_run_steps?: SuiteFile[] tests: SuiteTest[] + // eest_metadata is true when the suite output carries an .eest-meta directory + // (EEST fill provenance) copied from the fixtures' .meta. + eest_metadata?: boolean } export interface SuiteTestEEST { @@ -547,7 +599,7 @@ export interface SourceInfo { } } eest?: { - github_repo: string + github_repo?: string github_release?: string fixtures_url?: string genesis_url?: string @@ -556,6 +608,10 @@ export interface SourceInfo { genesis_artifact_name?: string fixtures_artifact_run_id?: string genesis_artifact_run_id?: string + local_fixtures_dir?: string + local_genesis_dir?: string + local_fixtures_tarball?: string + local_genesis_tarball?: string } } diff --git a/ui/src/components/run-detail/StateActorConfiguration.tsx b/ui/src/components/run-detail/StateActorConfiguration.tsx new file mode 100644 index 000000000..b64c814f8 --- /dev/null +++ b/ui/src/components/run-detail/StateActorConfiguration.tsx @@ -0,0 +1,284 @@ +import { useState, type ReactNode } from 'react' + +import { useQuery } from '@tanstack/react-query' +import clsx from 'clsx' +import { Check, Copy, Database, ExternalLink } from 'lucide-react' +import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter' +import { oneDark } from 'react-syntax-highlighter/dist/esm/styles/prism' + +import { fetchText } from '@/api/client' +import type { StateActorManifest } from '@/api/types' +import { Badge } from '@/components/shared/Badge' +import { Card } from '@/components/shared/Card' +import { getDataUrl, loadRuntimeConfig } from '@/config/runtime' +import { formatBytes, formatNumber } from '@/utils/format' + +interface StateActorConfigurationProps { + manifest: StateActorManifest + runId: string +} + +function CopyButton({ text }: { text: string }) { + const [copied, setCopied] = useState(false) + + const handleCopy = async () => { + await navigator.clipboard.writeText(text) + setCopied(true) + setTimeout(() => setCopied(false), 2000) + } + + return ( + + ) +} + +// compactNumber abbreviates large counts for compact header display +// (e.g. 300000000 -> "300M"). +function compactNumber(n: number): string { + if (n >= 1e9) return `${+(n / 1e9).toFixed(1)}B` + if (n >= 1e6) return `${+(n / 1e6).toFixed(1)}M` + if (n >= 1e3) return `${+(n / 1e3).toFixed(1)}K` + + return String(n) +} + +// shortHash truncates a 0x hash to first-6…last-4 for inline display. +function shortHash(h: string): string { + return h.length > 12 ? `${h.slice(0, 6)}…${h.slice(-4)}` : h +} + +function langForFile(name: string): string { + if (name.endsWith('.json')) return 'json' + if (name.endsWith('.yaml') || name.endsWith('.yml')) return 'yaml' + + return 'text' +} + +// RawFile shows a single .state-actor file (the manifest or a spec sidecar) +// with copy, open-raw and an expandable, syntax-highlighted raw-content view. +function RawFile({ runId, name }: { runId: string; name: string }) { + const [open, setOpen] = useState(false) + + const { data: config } = useQuery({ + queryKey: ['runtime-config'], + queryFn: loadRuntimeConfig, + staleTime: Infinity, + }) + + const { data: file, isLoading } = useQuery({ + queryKey: ['run', runId, 'state-actor-file', name], + queryFn: () => fetchText(`runs/${runId}/.state-actor/${name}`), + }) + + const text = file?.data ?? '' + const url = config ? getDataUrl(`runs/${runId}/.state-actor/${name}`, config) : undefined + + return ( +
+
+ + {name} + +
+ {text && } + {url && ( + + + + )} + +
+
+ {open && ( + + {isLoading ? 'Loading…' : text || '(empty)'} + + )} +
+ ) +} + +function Field({ label, value, mono }: { label: string; value: ReactNode; mono?: boolean }) { + return ( +
+
{label}
+
+ {value} +
+
+ ) +} + +function Section({ title, children }: { title: string; children: ReactNode }) { + return ( +
+

+ {title} +

+
{children}
+
+ ) +} + +// StateActorConfiguration renders a run's state-actor provenance manifest +// (build, resolved flags, generation result, optional spec and command). +export function StateActorConfiguration({ manifest, runId }: StateActorConfigurationProps) { + const { state_actor: sa, flags, result, spec } = manifest + + return ( + + + Database + + } + headerExtra={ + + {result && ( + <> + DB size: + + {formatBytes(result.total_db_size_bytes)} + + · + + )} + Gas limit: + {compactNumber(flags.gas_limit)} + {result?.state_root && ( + <> + · + State root: + + {shortHash(result.state_root)} + + + )} + + } + collapsible + defaultCollapsed + > +
+

+ + state-actor + {' '} + is the tool that builds a client's data directory — the synthetic + genesis state the benchmark boots from and replays payloads on. The + manifest below records how this run's snapshot was generated. +

+ +
+ + {sa.version || 'unknown'} + {sa.vcs_modified && modified} + + } + /> + + + {sa.vcs_revision && } + {sa.vcs_time && } + +
+ +
+ + + + + + {flags.target_size && } + + + + {flags.spec_path && } + +
+ + {result && ( +
+ + + + + + +
+ )} + + {spec && ( +
+ + + +
+ )} + + {manifest.reproduced_from && ( +
+ +
+ )} + +
+

+ Command +

+ 
+            {manifest.command.join(' ')}
+
+
+ +
+

+ Raw files +

+ + {spec?.output_file && } +
+
+ + ) +} diff --git a/ui/src/components/shared/Card.tsx b/ui/src/components/shared/Card.tsx index 54f427ce7..5da404828 100644 --- a/ui/src/components/shared/Card.tsx +++ b/ui/src/components/shared/Card.tsx @@ -8,23 +8,40 @@ interface CardProps { collapsible?: boolean defaultCollapsed?: boolean className?: string + // headerExtra renders a summary on the right side of the header (before the + // collapse chevron), e.g. a size or status shown even while collapsed. + headerExtra?: React.ReactNode } -export function Card({ title, children, collapsible = false, defaultCollapsed = false, className }: CardProps) { +export function Card({ + title, + children, + collapsible = false, + defaultCollapsed = false, + className, + headerExtra, +}: CardProps) { const [isCollapsed, setIsCollapsed] = useState(defaultCollapsed) return (
setIsCollapsed(!isCollapsed) : undefined} > -

{title}

- {collapsible && ( - +

{title}

+ {(headerExtra || collapsible) && ( +
+ {headerExtra} + {collapsible && ( + + )} +
)}
{!isCollapsed &&
{children}
} diff --git a/ui/src/components/suite-detail/EESTMetadata.tsx b/ui/src/components/suite-detail/EESTMetadata.tsx new file mode 100644 index 000000000..1512fa09c --- /dev/null +++ b/ui/src/components/suite-detail/EESTMetadata.tsx @@ -0,0 +1,167 @@ +import { useMemo } from 'react' + +import { useQuery } from '@tanstack/react-query' +import { ExternalLink } from 'lucide-react' + +import { fetchText } from '@/api/client' +import { Card } from '@/components/shared/Card' +import { getDataUrl, loadRuntimeConfig } from '@/config/runtime' + +interface EESTMetadataProps { + suiteHash: string +} + +interface IniSection { + name: string + entries: [string, string][] +} + +// parseIni parses a minimal INI document (sections + key=value pairs) into +// ordered sections, preserving order and skipping comments and blank lines. +function parseIni(text: string): IniSection[] { + const sections: IniSection[] = [] + let current: IniSection | null = null + + for (const rawLine of text.split('\n')) { + const line = rawLine.trim() + if (line === '' || line.startsWith(';') || line.startsWith('#')) continue + + const section = line.match(/^\[(.+)\]$/) + if (section) { + current = { name: section[1], entries: [] } + sections.push(current) + + continue + } + + const eq = line.indexOf('=') + if (eq === -1) continue + + if (!current) { + current = { name: '', entries: [] } + sections.push(current) + } + + current.entries.push([line.slice(0, eq).trim(), line.slice(eq + 1).trim()]) + } + + return sections +} + +// EESTMetadata renders the EEST fill provenance copied into a suite's +// .eest-meta directory: the parsed fixtures.ini (tool/python versions, fill +// command, packages/plugins) plus links to and an embed of the fill report. +// Rendered as a single collapsible card so it sits inside the Source tab. +export function EESTMetadata({ suiteHash }: EESTMetadataProps) { + const iniQuery = useQuery({ + queryKey: ['eest-meta-fixtures-ini', suiteHash], + queryFn: () => fetchText(`suites/${suiteHash}/.eest-meta/fixtures.ini`), + }) + + const { data: config } = useQuery({ + queryKey: ['runtime-config'], + queryFn: loadRuntimeConfig, + staleTime: Infinity, + }) + + const sections = useMemo( + () => (iniQuery.data?.data ? parseIni(iniQuery.data.data) : []), + [iniQuery.data], + ) + + // Don't render the section at all while loading or if the metadata is + // unreadable — the caller only mounts this when the suite advertises it. + if (iniQuery.isLoading || iniQuery.data?.data == null) { + return null + } + + const reportUrl = config + ? getDataUrl(`suites/${suiteHash}/.eest-meta/report_fill.html`, config) + : undefined + const indexUrl = config + ? getDataUrl(`suites/${suiteHash}/.eest-meta/index.json`, config) + : undefined + + return ( + +
+

+ Provenance recorded by execution-spec-tests when these fixtures were filled + (.eest-meta/fixtures.ini). +

+ + {sections.map((section) => ( +
+

+ {section.name || 'general'} +

+
+ {section.entries.map(([key, value]) => ( +
+
+ {key} +
+
+ {value} +
+
+ ))} +
+
+ ))} + + {(reportUrl || indexUrl) && ( +
+

+ fill report +

+
+ {reportUrl && ( + + Open fill report + + + )} + {indexUrl && ( + + View fixture index (JSON) + + + )} +
+ {reportUrl && ( +
+ {/* The fill report is a light-mode HTML page rendered in an iframe + (often cross-origin, so its document can't be styled from here). + Approximate dark mode by inverting the rendered frame when the + UI is in dark mode. invert(.9) (not full 1.0) softens the + extremes — white bg → dark gray (~#1a1a1a) and black text → + near-white — so it blends with the UI palette instead of going + stark black/white; hue-rotate keeps hues roughly correct. Tracks + the UI theme automatically via the dark variant. */} +