From 3a3224ca4165cab14d975a9a8afbad92cc51baa0 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 16 Jun 2026 12:41:53 +0200 Subject: [PATCH 01/83] feat(eest): generate and replay stateful EEST benchmark fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the stateful EEST fixture pipeline end-to-end: a builder that generates `blockchain_test_stateful_engine` fixtures and runner support that replays them. Generation (`benchmarkoor build`): - `builder.eest_payloads` boots a geth filler on a writable copy of a snapshot datadir and runs fill-stateful (execution-specs #2637) against it, writing stateful fixtures. Only geth supports testing_buildBlockV1. - Shared builder helpers extracted to pkg/builder/util.go; Dockerfile.eest-filler builds the fill image. Replay (`benchmarkoor run`): - Parse the stateful-engine format (snapshot datadir, no genesis) plus the shared pre_run/.json files; convert pre_run + per-test setup payloads into setup steps and the benchmark payloads into the measured test step (pkg/eest, pkg/executor/eest_source.go). - Make eest_fixtures.local_genesis_dir optional — the runner already boots datadir-only with no genesis, so stateful sources need no genesis dir. --- Dockerfile.eest-filler | 63 ++++ cmd/benchmarkoor/build.go | 210 +++++++---- config.example.yaml | 30 ++ docs/configuration.md | 106 +++++- pkg/builder/eest_payloads.go | 582 ++++++++++++++++++++++++++++++ pkg/builder/eest_payloads_test.go | 182 ++++++++++ pkg/builder/rpc.go | 127 +++++++ pkg/builder/state_actor.go | 117 +----- pkg/builder/util.go | 123 +++++++ pkg/config/config.go | 385 +++++++++++++++++++- pkg/config/config_test.go | 263 +++++++++++++- pkg/eest/converter.go | 67 ++++ pkg/eest/converter_test.go | 154 ++++++++ pkg/eest/fixture.go | 53 ++- pkg/executor/eest_source.go | 88 ++++- 15 files changed, 2350 insertions(+), 200 deletions(-) create mode 100644 Dockerfile.eest-filler create mode 100644 pkg/builder/eest_payloads.go create mode 100644 pkg/builder/eest_payloads_test.go create mode 100644 pkg/builder/rpc.go create mode 100644 pkg/builder/util.go diff --git a/Dockerfile.eest-filler b/Dockerfile.eest-filler new file mode 100644 index 000000000..d9538cc67 --- /dev/null +++ b/Dockerfile.eest-filler @@ -0,0 +1,63 @@ +# Dockerfile.eest-filler +# +# Builds the image referenced by `builder.eest_payloads.fill_image`: it carries +# the EEST `fill-stateful` command (execution-specs PR #2637) plus `uv`. +# +# fill-stateful drives a *live* EL client over testing_buildBlockV1, so — unlike +# the t8n-based `fill` — it needs NO evmone/eels transition-tool binary. The +# image is just `uv` + the execution-specs checkout. +# +# Build: +# docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +# # (podman build works identically) +# +# Then point your config at it: +# builder: +# eest_payloads: +# fill_image: ghcr.io/your-org/eest-fill-stateful:latest +# +# benchmarkoor invokes it as: `uv run fill-stateful ` +# (the default builder.eest_payloads.fill_command), with the test paths resolved +# relative to the execution-specs checkout (WORKDIR below). + +# --- Source selection ------------------------------------------------------- +# fill-stateful is merged upstream on the forks/amsterdam branch. Override the +# ref to pin a specific tag/commit, e.g. --build-arg EEST_REF=. +ARG EEST_REPO=https://github.com/ethereum/execution-specs.git +ARG EEST_REF=forks/amsterdam +ARG PYTHON_VERSION=3.12 + +FROM python:${PYTHON_VERSION}-slim-bookworm + +# uv: copy the static binary from the official image (recommended pattern). +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# Build deps: git to fetch the source; build-essential/headers for any EEST +# dependencies without prebuilt wheels. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + git \ + ca-certificates \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +ARG EEST_REPO +ARG EEST_REF + +WORKDIR /eest + +# Clone and check out the requested ref (branch, tag, or commit SHA). `uv run +# fill-stateful` and the relative test paths (e.g. tests/benchmark/stateful) +# both resolve against this directory. +RUN git clone "${EEST_REPO}" . \ + && git checkout "${EEST_REF}" + +# Pre-create the virtualenv at build time so runs don't sync on every invocation. +ENV UV_NO_PROGRESS=1 +RUN uv sync + +# Record the exact source revision for traceability. +RUN git rev-parse HEAD > /eest/.fill-stateful-rev + +# Standalone default; benchmarkoor overrides this with the full fill-stateful argv. +CMD ["uv", "run", "fill-stateful", "--help"] diff --git a/cmd/benchmarkoor/build.go b/cmd/benchmarkoor/build.go index 79ee8e1e0..1401abadd 100644 --- a/cmd/benchmarkoor/build.go +++ b/cmd/benchmarkoor/build.go @@ -24,12 +24,18 @@ var ( var buildCmd = &cobra.Command{ Use: "build", - Short: "Build client datadirs declared in builder.state_actor.targets", - Long: `Materialise each datadir declared under builder.state_actor.targets -by invoking state-actor (https://github.com/ethereum/state-actor) via the -configured container runtime. Builds are decoupled from "benchmarkoor run": -this command produces datadirs on disk that subsequent runs consume via -their normal datadir.* providers.`, + Short: "Build datadirs and fixtures declared under the builder.* config blocks", + Long: `Run each configured builder: + + - builder.state_actor materialises pre-populated client datadirs by invoking + state-actor (https://github.com/ethereum/state-actor). + - builder.eest_payloads generates stateful EEST benchmark fixtures by running + fill-stateful against a filler client booted on a snapshot. + +Builds are decoupled from "benchmarkoor run": this command produces artifacts on +disk that subsequent runs consume via their normal datadir.* / test source providers. +Builders run in declaration order (state_actor before eest_payloads) so a fixture +build can consume a datadir produced earlier in the same invocation.`, RunE: runBuild, } @@ -55,18 +61,29 @@ func runBuild(_ *cobra.Command, _ []string) error { return fmt.Errorf("validating config: %w", err) } - if cfg.Builder == nil || cfg.Builder.StateActor == nil || len(cfg.Builder.StateActor.Targets) == 0 { - return fmt.Errorf("builder.state_actor.targets is empty or unset; nothing to build") + if cfg.Builder == nil || + (cfg.Builder.StateActor == nil && cfg.Builder.EESTPayloads == nil) { + return fmt.Errorf("no builders configured; nothing to build") } - targets, err := selectTargets(cfg.Builder.StateActor.Targets, buildTargetFilter) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + installSignalHandler(cancel) + + builders, stop, err := buildBuilders(ctx, cfg) if err != nil { return err } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + defer stop() + return runBuilders(ctx, builders) +} + +// installSignalHandler cancels the context on the first SIGINT/SIGTERM and +// force-exits on the second. +func installSignalHandler(cancel context.CancelFunc) { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) @@ -78,45 +95,102 @@ func runBuild(_ *cobra.Command, _ []string) error { sig = <-sigCh log.WithField("signal", sig).Fatal("Received second signal, forcing exit") }() +} - runtime := cfg.GetStateActorContainerRuntime() +// buildBuilders constructs every configured builder, creating and starting a +// container manager per distinct runtime. The returned stop func stops all +// managers and must be deferred by the caller. +func buildBuilders(ctx context.Context, cfg *config.Config) ([]builder.Builder, func(), error) { + managers := make(map[string]docker.ContainerManager, 2) + + stop := func() { + for _, mgr := range managers { + if err := mgr.Stop(); err != nil { + log.WithError(err).Warn("Failed to stop container manager") + } + } + } - var mgr docker.ContainerManager + getManager := func(runtime string) (docker.ContainerManager, error) { + if mgr, ok := managers[runtime]; ok { + return mgr, nil + } - switch runtime { - case "podman": - mgr, err = podman.NewManager(log) - default: - mgr, err = docker.NewManager(log) - } + mgr, err := newContainerManager(runtime) + if err != nil { + return nil, err + } - if err != nil { - return fmt.Errorf("creating container manager: %w", err) + if err := mgr.Start(ctx); err != nil { + return nil, fmt.Errorf("starting %s container manager: %w", runtime, err) + } + + managers[runtime] = mgr + + return mgr, nil } - if err := mgr.Start(ctx); err != nil { - return fmt.Errorf("starting container manager: %w", err) + var builders []builder.Builder + + if cfg.Builder.StateActor != nil { + runtime := cfg.GetStateActorContainerRuntime() + + mgr, err := getManager(runtime) + if err != nil { + stop() + + return nil, nil, err + } + + builders = append(builders, builder.NewStateActorBuilder(log, cfg.Builder.StateActor, runtime, mgr)) } - defer func() { - if err := mgr.Stop(); err != nil { - log.WithError(err).Warn("Failed to stop container manager") + if cfg.Builder.EESTPayloads != nil { + runtime := cfg.GetEESTPayloadsContainerRuntime() + + mgr, err := getManager(runtime) + if err != nil { + stop() + + return nil, nil, err } - }() - b := builder.NewStateActorBuilder(log, cfg.Builder.StateActor, runtime, mgr) + builders = append(builders, builder.NewEESTPayloadsBuilder(log, cfg.Builder.EESTPayloads, runtime, mgr)) + } + + return builders, stop, nil +} + +// newContainerManager creates a container manager for the given runtime. +func newContainerManager(runtime string) (docker.ContainerManager, error) { + switch runtime { + case "podman": + return podman.NewManager(log) + default: + return docker.NewManager(log) + } +} + +// buildResult captures the outcome of a single target build. +type buildResult struct { + name string + client string + outputDir string + skipped bool + err error +} - type result struct { - name string - client string - outputDir string - skipped bool - err error +// runBuilders selects and builds the requested targets across all builders, +// preserving declaration order, then prints a summary. +func runBuilders(ctx context.Context, builders []builder.Builder) error { + targets, err := selectTargets(builders, buildTargetFilter) + if err != nil { + return err } - results := make([]result, 0, len(targets)) + results := make([]buildResult, 0, len(targets)) - for _, t := range targets { + for _, sel := range targets { select { case <-ctx.Done(): return ctx.Err() @@ -124,27 +198,31 @@ func runBuild(_ *cobra.Command, _ []string) error { } log.WithFields(logrus.Fields{ - "target": t.EffectiveName(), - "client": t.Client, - "output_dir": t.OutputDir, + "target": sel.info.Name, + "client": sel.info.Client, + "output_dir": sel.info.OutputDir, }).Info("Building target") - skipped, buildErr := b.Build(ctx, t.EffectiveName(), builder.BuildOptions{Force: buildForce}) + skipped, buildErr := sel.builder.Build(ctx, sel.info.Name, builder.BuildOptions{Force: buildForce}) - results = append(results, result{ - name: t.EffectiveName(), - client: t.Client, - outputDir: t.OutputDir, + results = append(results, buildResult{ + name: sel.info.Name, + client: sel.info.Client, + outputDir: sel.info.OutputDir, skipped: skipped, err: buildErr, }) if buildErr != nil { - log.WithError(buildErr).WithField("target", t.EffectiveName()).Error("Build failed") + log.WithError(buildErr).WithField("target", sel.info.Name).Error("Build failed") } } - // Summary. + return summarise(results) +} + +// summarise logs the per-target outcome and returns an error if any failed. +func summarise(results []buildResult) error { var failed []string log.Info("Build summary:") @@ -155,6 +233,7 @@ func runBuild(_ *cobra.Command, _ []string) error { switch { case r.err != nil: status = "ERR " + failed = append(failed, r.name) case r.skipped: status = "SKIP" @@ -170,37 +249,42 @@ func runBuild(_ *cobra.Command, _ []string) error { } if len(failed) > 0 { - return fmt.Errorf("%d target(s) failed: %s", - len(failed), strings.Join(failed, ", ")) + return fmt.Errorf("%d target(s) failed: %s", len(failed), strings.Join(failed, ", ")) } return nil } -// selectTargets filters `all` by the names in `filter`, preserving the -// order targets were declared in. An empty filter returns all targets. -// Unmatched filter values produce an error so typos surface immediately. -func selectTargets(all []config.StateActorTarget, filter []string) ([]config.StateActorTarget, error) { - if len(filter) == 0 { - return all, nil - } +// selectedTarget pairs a target with the builder that owns it. +type selectedTarget struct { + builder builder.Builder + info builder.TargetInfo +} +// selectTargets flattens all builders' targets in declaration order and +// filters them by the names in filter. An empty filter returns every target. +// Unmatched filter values produce an error so typos surface immediately. +func selectTargets(builders []builder.Builder, filter []string) ([]selectedTarget, error) { wanted := make(map[string]bool, len(filter)) + for _, f := range filter { - f = strings.TrimSpace(f) - if f != "" { + if f = strings.TrimSpace(f); f != "" { wanted[f] = true } } - out := make([]config.StateActorTarget, 0, len(all)) + var out []selectedTarget + matched := make(map[string]bool, len(wanted)) - for _, t := range all { - name := t.EffectiveName() - if wanted[name] { - out = append(out, t) - matched[name] = true + for _, b := range builders { + for _, info := range b.Targets() { + if len(wanted) > 0 && !wanted[info.Name] { + continue + } + + out = append(out, selectedTarget{builder: b, info: info}) + matched[info.Name] = true } } diff --git a/config.example.yaml b/config.example.yaml index b92371d5e..c2dfd9ec3 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -266,6 +266,36 @@ runner: # client: reth # output_dir: /srv/state/reth-spec # # inherits the top-level spec/spec_file and every field from `config` +# +# # Optional: generate stateful EEST benchmark fixtures with fill-stateful. +# # Boots a filler client on a copy of a snapshot datadir (e.g. a state_actor +# # output_dir above), records engine-API payloads, and writes fixtures that +# # `run` consumes via tests.source.eest_fixtures. Runs after state_actor. +# # Only geth supports fill-stateful's testing_buildBlockV1 today. Build the +# # fill image from the repo's Dockerfile.eest-filler (execution-specs #2637): +# # docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +# eest_payloads: +# fill_image: ghcr.io/your-org/eest-fill-stateful:latest +# # pull_policy: always # always | if-not-present | never +# # container_runtime: docker # defaults to runner.container_runtime, then docker +# # jwt: # Engine API secret shared with the filler (default: built-in) +# # fill_command: [uv, run, fill-stateful] # argv prefix inside fill_image (default) +# config: # shared per-target defaults; targets override when set +# filler_image: ethpandaops/geth:master +# fork: Osaka +# gas_benchmark_values: "10,30" # millions of gas to parametrise against +# datadir_method: copy # copy | overlayfs | fuse-overlayfs | zfs | direct | schelk +# targets: +# - name: compute-geth +# filler_client: geth # only geth is supported today +# source_dir: /srv/state/geth-5g # PRISTINE snapshot (never mutated) +# genesis_file: /srv/state/geth-5g/genesis.json # chain config for the filler boot +# output_dir: /srv/fixtures/compute +# tests: +# - tests/benchmark/compute # pytest paths inside the fill image +# # filter: bn128 # optional pytest -k expression +# # force: true # wipe output_dir before filling +# # address_stubs_file: /etc/benchmarkoor/stubs.json # for stub-dependent tests # Optional: API server for authentication and user management. # When configured, the UI can integrate with the API for login, admin, and role-based access. diff --git a/docs/configuration.md b/docs/configuration.md index 5e5184423..16b89583a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1382,9 +1382,12 @@ runner: ## Builder -The `builder` section configures tools that pre-populate client datadirs on disk. Today the only builder is `state-actor` (https://github.com/ethereum/state-actor), which writes per-client genesis state directly in each EL's native on-disk format — geth Pebble, reth MDBX, besu/nethermind RocksDB — bypassing the client's normal genesis-replay path. +The `builder` section configures tools that pre-populate benchmark inputs on disk. There are two builders: -Builds are **decoupled from `benchmarkoor run`**: invoke `benchmarkoor build` to materialise datadirs, then run benchmarks against them via the regular `datadir.method: copy|zfs|schelk|…` providers. A missing datadir at `run` time is an error — it is never auto-built. +- **`state_actor`** (https://github.com/ethereum/state-actor) writes per-client genesis state directly in each EL's native on-disk format — geth Pebble, reth MDBX, besu/nethermind RocksDB — bypassing the client's normal genesis-replay path. +- **`eest_payloads`** generates stateful EEST benchmark fixtures by running [`fill-stateful`](https://github.com/ethereum/execution-specs/pull/2637) against a filler client booted on a pre-populated snapshot (typically one produced by `state_actor`). The fixtures are replayed by `benchmarkoor run`. + +Builds are **decoupled from `benchmarkoor run`**: invoke `benchmarkoor build` to materialise the artifacts, then run benchmarks against them via the regular `datadir.method: copy|zfs|schelk|…` providers and test-source config. A missing datadir at `run` time is an error — it is never auto-built. When both builders are configured, they run in declaration order (`state_actor` before `eest_payloads`) so a fixture build can consume a datadir produced earlier in the same `benchmarkoor build` invocation. ### `builder.state_actor` options @@ -1550,6 +1553,105 @@ builder: output_dir: /srv/state/geth-spec ``` +### `builder.eest_payloads` options + +`eest_payloads` generates **stateful** EEST benchmark fixtures: it boots a filler EL client on a *writable copy* of a pre-populated snapshot datadir, runs `fill-stateful` against the live client (recording engine-API payloads anchored to the snapshot's head block), and writes the fixtures to each target's `output_dir`. `fill-stateful` itself does not manage datadirs — benchmarkoor boots the filler and snapshots it. + +> **Filler client:** only `geth` implements the `testing_buildBlockV1` API that `fill-stateful` drives, so `filler_client` must be `geth` today; `ethpandaops/geth:master` is the production-ready image. +> +> **Fill image:** there is no published `fill-stateful` image yet (the command lands in execution-specs [#2637](https://github.com/ethereum/execution-specs/pull/2637)). Build one from the repo's `Dockerfile.eest-filler` (it bundles `uv` + execution-specs) and point `fill_image` at it: +> ```bash +> docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +> ``` + +```yaml +builder: + eest_payloads: + fill_image: ghcr.io/your-org/eest-fill-stateful:latest # image carrying `uv run fill-stateful` + pull_policy: always # always | if-not-present | never (default: always) + container_runtime: docker # docker | podman (default: inherits runner.container_runtime, then docker) + # jwt: # Engine API secret, shared with the filler (default: benchmarkoor's DefaultJWT) + # fill_command: [uv, run, fill-stateful] # argv prefix inside fill_image (this is the default) + config: # shared per-target defaults; targets override when set + filler_image: ethpandaops/geth:master + fork: Osaka + gas_benchmark_values: "10,30" # millions of gas to parametrise against + datadir_method: copy # copy | overlayfs | fuse-overlayfs | zfs | direct | schelk + targets: + - name: compute-geth + filler_client: geth + source_dir: /srv/state/geth-archive # PRISTINE snapshot (never mutated; a writable copy is filled) + genesis_file: /srv/state/geth-archive/genesis.json # chain config for the filler boot + output_dir: /srv/fixtures/compute + tests: + - tests/benchmark/compute # pytest paths inside the fill image + filter: bn128 # optional pytest -k expression +``` + +| Option | Type | Default | Description | +|---|---|---|---| +| `fill_image` | string | – | **Required.** Container image carrying the `fill-stateful` command (`uv` + execution-specs). | +| `pull_policy` | string | `always` | One of `always`, `if-not-present`, `never`. Applies to both the fill image and the filler image. | +| `container_runtime` | string | runner's runtime, then `docker` | Container runtime for the filler + fill containers. | +| `jwt` | string | benchmarkoor's `DefaultJWT` | Engine API JWT secret; shared between the filler client and `fill-stateful`. | +| `fill_command` | []string | `[uv, run, fill-stateful]` | argv prefix invoked inside `fill_image` before the `fill-stateful` flags. Override if your image exposes the command differently. | +| `config` | object | – | Shared defaults for the per-target parameters. See below. | +| `targets` | []object | – | Required when invoking `benchmarkoor build`. See below. | + +### `builder.eest_payloads.config` options + +Every field below is also available per-target; a non-nil/non-empty value on a target overrides the default. Use this block to avoid repeating `fork`, `gas_benchmark_values`, etc. + +| Option | Type | Default | Description | +|---|---|---|---| +| `filler_image` | string | – | Docker image for the filler client (e.g. `ethpandaops/geth:master`). | +| `fork` | string | – | Fork to fill against, e.g. `Osaka` (passed to `fill-stateful --fork`). | +| `gas_benchmark_values` | string | – | Comma-separated gas budgets in millions, e.g. `10,30` (`--gas-benchmark-values`). | +| `datadir_method` | string | `copy` | How the filler's writable copy of `source_dir` is prepared: `copy`, `overlayfs`, `fuse-overlayfs`, `zfs`, `direct`, `schelk`. Use `zfs`/`overlayfs` to avoid a full copy of a large snapshot. | +| `max_gas_per_test` | uint64 | – | Overrides the fork's transaction gas-limit cap (`--max-gas-per-test`). | +| `rpc_seed_key` | string | – | Pin the seed EOA for reproducible fills (`--rpc-seed-key`); otherwise one is generated and funded via CL withdrawal. | +| `filler_extra_args` | []string | – | Extra argv appended to the filler client command. | + +### `builder.eest_payloads.targets[]` options + +Identity/locator fields are target-only; the rest mirror `config` and are resolved with per-target precedence. + +| Option | Type | Default | Description | +|---|---|---|---| +| `name` | string | `filler_client` | Used by `--target` to filter. Must be unique across targets. | +| `filler_client` | string | – | Client booted as the filler. Must be `geth` (only client supporting `testing_buildBlockV1`). | +| `source_dir` | string | – | **Absolute** host path to the pristine snapshot datadir (e.g. a `state_actor` `output_dir`). Never mutated — a writable copy is filled. Existence is checked at build time. | +| `genesis_file` | string | – | **Absolute** host path to the genesis/chain-config the filler boots with (`--override.genesis`). Must match the chain config used to produce `source_dir`. | +| `output_dir` | string | – | **Absolute** host path for the generated fixtures. Skipped if already populated unless `--force` / `force: true`. Written under `/blockchain_tests_stateful_engine/`. | +| `tests` | []string | – | **Required.** pytest paths inside the fill image, e.g. `tests/benchmark/compute`. | +| `filter` | string | – | Optional pytest `-k` expression. | +| `address_stubs_file` | string | – | **Absolute** host path to a `--address-stubs` JSON map, required by stub-dependent tests (e.g. bloatnet opcode tests). | +| `force` | bool | `false` | Per-target override of `--force`: wipe `output_dir` before filling. | +| `filler_image`, `fork`, `gas_benchmark_values`, `datadir_method`, `max_gas_per_test`, `rpc_seed_key`, `filler_extra_args` | — | from `config` | See the `config` table above. `fork` and `filler_image` are required after resolution. | + +### Replaying generated fixtures + +Point `benchmarkoor run` at the **pristine** snapshot (never the copy the filler mutated) and at the fixture output: + +```yaml +runner: + client: + datadirs: + geth: + source_dir: /srv/state/geth-archive # the pristine snapshot + method: zfs # or copy/overlayfs/… + benchmark: + tests: + source: + eest_fixtures: + local_fixtures_dir: /srv/fixtures/compute + fixtures_subdir: blockchain_tests_stateful_engine +``` + +> Stateful replay needs the new fixture format support — see benchmarkoor [#182](https://github.com/ethpandaops/benchmarkoor/pull/182). + +As a sanity check, each fixture's recorded `benchmarkGasUsed` should match benchmarkoor's measured `gas_used_total` for that test. + ## API Server See [API Server documentation](api.md) for the full reference on the `api` config section, including server settings, authentication, database, storage, endpoints, and UI integration. diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go new file mode 100644 index 000000000..ee7df1c98 --- /dev/null +++ b/pkg/builder/eest_payloads.go @@ -0,0 +1,582 @@ +package builder + +import ( + "context" + "fmt" + "io" + "os" + "strconv" + "time" + + "github.com/ethpandaops/benchmarkoor/pkg/client" + "github.com/ethpandaops/benchmarkoor/pkg/config" + "github.com/ethpandaops/benchmarkoor/pkg/datadir" + "github.com/ethpandaops/benchmarkoor/pkg/docker" + "github.com/sirupsen/logrus" +) + +// EESTPayloadsBuilderName is the value of the benchmarkoor.builder label +// set on every eest_payloads container, and the value returned by Name(). +const EESTPayloadsBuilderName = "eest-payloads" + +const ( + // eestBuildNetwork is the docker/podman network shared by the filler + // client and the fill-stateful container so they can reach each other. + eestBuildNetwork = "benchmarkoor-build" + + // fillerReadyTimeout bounds how long we wait for the filler client's RPC + // to answer — opening a large archive snapshot can take several minutes. + fillerReadyTimeout = 15 * time.Minute + + // fillerStopTimeoutSec is the graceful-stop window for the filler client. + fillerStopTimeoutSec = 30 + + // In-container paths used by the fill-stateful container. + fillJWTPath = "/jwt/jwtsecret" + fillOutputPath = "/out" + fillStubsPath = "/stubs.json" + + // minerGasLimit is the huge gas limit the filler geth is started with so + // benchmark blocks of any size can be built (mirrors the fill-stateful docs). + minerGasLimit = "1000000000000" +) + +// EESTPayloadsBuilder generates stateful EEST benchmark fixtures. Per target +// it boots a filler EL client on a writable copy of a pre-populated snapshot +// datadir, runs fill-stateful against the live client, then tears it down. +type EESTPayloadsBuilder struct { + log logrus.FieldLogger + cfg *config.EESTPayloadsConfig + runtime string + mgr docker.ContainerManager + registry client.Registry +} + +// NewEESTPayloadsBuilder constructs a builder bound to a specific container +// manager. The caller is expected to have Start()'d the manager and to +// Stop() it after the last Build() call. +func NewEESTPayloadsBuilder( + log logrus.FieldLogger, + cfg *config.EESTPayloadsConfig, + runtime string, + mgr docker.ContainerManager, +) *EESTPayloadsBuilder { + return &EESTPayloadsBuilder{ + log: log.WithField("component", "builder.eest_payloads"), + cfg: cfg, + runtime: runtime, + mgr: mgr, + registry: client.NewRegistry(), + } +} + +// Name implements Builder. +func (b *EESTPayloadsBuilder) Name() string { + return EESTPayloadsBuilderName +} + +// Targets implements Builder. +func (b *EESTPayloadsBuilder) Targets() []TargetInfo { + out := make([]TargetInfo, 0, len(b.cfg.Targets)) + + for i := range b.cfg.Targets { + t := &b.cfg.Targets[i] + out = append(out, TargetInfo{ + Name: t.EffectiveName(), + Client: t.FillerClient, + OutputDir: t.OutputDir, + }) + } + + return out +} + +// Build implements Builder. +func (b *EESTPayloadsBuilder) Build(ctx context.Context, name string, opts BuildOptions) (bool, error) { + idx := b.findTargetIndex(name) + if idx < 0 { + return false, fmt.Errorf("no target named %q", name) + } + + resolved := b.cfg.ResolveTarget(idx) + target := &resolved + + log := b.log.WithFields(logrus.Fields{ + "target": target.EffectiveName(), + "filler_client": target.FillerClient, + "source_dir": target.SourceDir, + "output_dir": target.OutputDir, + "fork": target.Fork, + }) + + force := opts.Force || target.Force + + if !force { + populated, err := isPopulated(target.OutputDir) + if err != nil { + return false, err + } + + if populated { + log.Info("Skipping build: output_dir already populated " + + "(pass --force or set force: true on the target to rebuild)") + + return true, nil + } + } + + if err := b.checkInputs(target); err != nil { + return false, err + } + + if err := prepareOutputDir(target.OutputDir, force); err != nil { + return false, err + } + + return false, b.run(ctx, log, target) +} + +// findTargetIndex returns the index of the first target whose EffectiveName +// matches name, or -1 when nothing matches. +func (b *EESTPayloadsBuilder) findTargetIndex(name string) int { + for i := range b.cfg.Targets { + if b.cfg.Targets[i].EffectiveName() == name { + return i + } + } + + return -1 +} + +// checkInputs verifies the build-time inputs exist. Existence is checked +// here (not at config-validation time) because a state-actor target earlier +// in the same config may still need to produce source_dir. +func (b *EESTPayloadsBuilder) checkInputs(t *config.EESTPayloadTarget) error { + if info, err := os.Stat(t.SourceDir); err != nil { + return fmt.Errorf("source_dir %q: %w", t.SourceDir, err) + } else if !info.IsDir() { + return fmt.Errorf("source_dir %q is not a directory", t.SourceDir) + } + + if t.GenesisFile != "" { + if _, err := os.Stat(t.GenesisFile); err != nil { + return fmt.Errorf("genesis_file: %w", err) + } + } + + if t.AddressStubsFile != "" { + if _, err := os.Stat(t.AddressStubsFile); err != nil { + return fmt.Errorf("address_stubs_file: %w", err) + } + } + + return nil +} + +// run performs the orchestration: temp JWT, network, datadir copy, filler +// boot, fill-stateful, and teardown. +func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t *config.EESTPayloadTarget) error { + spec, err := b.registry.Get(client.ClientType(t.FillerClient)) + if err != nil { + return fmt.Errorf("resolving filler client %q: %w", t.FillerClient, err) + } + + jwtPath, cleanupJWT, err := writeTempJWT(b.cfg.JWT) + if err != nil { + return err + } + + defer cleanupJWT() + + if err := b.mgr.EnsureNetwork(ctx, eestBuildNetwork); err != nil { + return fmt.Errorf("ensuring network %q: %w", eestBuildNetwork, err) + } + + provider, err := datadir.NewProvider(b.log, t.DataDirMethod) + if err != nil { + return fmt.Errorf("creating datadir provider: %w", err) + } + + log.Info("Preparing writable copy of snapshot datadir") + + prepared, err := provider.Prepare(ctx, &datadir.ProviderConfig{ + SourceDir: t.SourceDir, + InstanceID: "eest-fill-" + t.EffectiveName(), + TmpDir: os.TempDir(), + }) + if err != nil { + return fmt.Errorf("preparing datadir copy: %w", err) + } + + defer func() { + if cleanupErr := prepared.Cleanup(); cleanupErr != nil { + log.WithError(cleanupErr).Warn("Failed to clean up datadir copy") + } + }() + + // Stream the filler's logs for the lifetime of this build. + streamCtx, streamCancel := context.WithCancel(ctx) + defer streamCancel() + + fillerID, fillerIP, err := b.startFiller(ctx, streamCtx, log, t, spec, prepared.MountPath, jwtPath) + if err != nil { + return err + } + + defer b.stopFiller(log, fillerID) + + log.Info("Waiting for filler client RPC to become ready") + + readyCtx, cancel := context.WithTimeout(ctx, fillerReadyTimeout) + version, err := waitForRPC(readyCtx, fillerIP, spec.RPCPort()) + cancel() + + if err != nil { + return fmt.Errorf("filler client never became ready: %w", err) + } + + snapshotHash, err := getLatestBlockHash(ctx, fillerIP, spec.RPCPort()) + if err != nil { + return fmt.Errorf("fetching snapshot block hash: %w", err) + } + + log.WithFields(logrus.Fields{ + "client_version": version, + "snapshot_block": snapshotHash, + }).Info("Filler client ready; running fill-stateful") + + return b.runFill(ctx, log, t, fillerIP, spec, jwtPath, snapshotHash) +} + +// startFiller boots the filler EL client and returns its container ID and IP. +func (b *EESTPayloadsBuilder) startFiller( + ctx, streamCtx context.Context, + log logrus.FieldLogger, + t *config.EESTPayloadTarget, + spec client.Spec, + dataMount, jwtPath string, +) (string, string, error) { + if err := b.mgr.PullImage(ctx, t.FillerImage, b.cfg.PullPolicy); err != nil { + return "", "", fmt.Errorf("pulling filler image %q: %w", t.FillerImage, err) + } + + mounts := []docker.Mount{ + {Source: dataMount, Target: spec.DataDir(), Type: "bind"}, + {Source: jwtPath, Target: spec.JWTPath(), Type: "bind", ReadOnly: true}, + } + + configCleanup := func() {} + + if files := spec.DefaultConfigFiles(); len(files) > 0 { + configMounts, cleanup, err := writeTempConfigFiles(files) + if err != nil { + return "", "", err + } + + mounts = append(mounts, configMounts...) + configCleanup = cleanup + } + + defer configCleanup() + + if t.GenesisFile != "" { + mounts = append(mounts, docker.Mount{ + Source: t.GenesisFile, Target: spec.GenesisPath(), Type: "bind", ReadOnly: true, + }) + } + + suffix, err := randSuffix() + if err != nil { + return "", "", fmt.Errorf("generating container name suffix: %w", err) + } + + cmd := fillerGethCommand(t, spec) + + containerSpec := &docker.ContainerSpec{ + Name: fmt.Sprintf("benchmarkoor-build-eest-filler-%s-%s", t.FillerClient, suffix), + Image: t.FillerImage, + Command: cmd, + Mounts: mounts, + NetworkName: eestBuildNetwork, + SecurityOpt: []string{"seccomp=unconfined"}, + Labels: b.labels(t), + } + + log.WithField("argv", cmd).Info("Starting filler client") + + id, err := b.mgr.CreateContainer(ctx, containerSpec) + if err != nil { + return "", "", fmt.Errorf("creating filler container: %w", err) + } + + if err := b.mgr.StartContainer(ctx, id); err != nil { + _ = b.mgr.RemoveContainer(context.Background(), id) + + return "", "", fmt.Errorf("starting filler container: %w", err) + } + + go func() { + w := logWriter(log.WithField("filler", t.FillerClient), logrus.InfoLevel) + if streamErr := b.mgr.StreamLogs(streamCtx, id, w, w); streamErr != nil { + log.WithError(streamErr).Debug("Filler log streaming stopped") + } + }() + + ip, err := b.mgr.GetContainerIP(ctx, id, eestBuildNetwork) + if err != nil { + _ = b.mgr.RemoveContainer(context.Background(), id) + + return "", "", fmt.Errorf("getting filler container IP: %w", err) + } + + return id, ip, nil +} + +// stopFiller stops and removes the filler container. It uses a background +// context so cleanup still runs when the build context is cancelled. +func (b *EESTPayloadsBuilder) stopFiller(log logrus.FieldLogger, id string) { + timeout := fillerStopTimeoutSec + if err := b.mgr.StopContainer(context.Background(), id, &timeout); err != nil { + log.WithError(err).Warn("Failed to stop filler container") + } + + if err := b.mgr.RemoveContainer(context.Background(), id); err != nil { + log.WithError(err).Warn("Failed to remove filler container") + } +} + +// runFill runs the fill-stateful container against the live filler client. +func (b *EESTPayloadsBuilder) runFill( + ctx context.Context, + log logrus.FieldLogger, + t *config.EESTPayloadTarget, + fillerIP string, + spec client.Spec, + jwtPath, snapshotHash string, +) error { + if err := b.mgr.PullImage(ctx, b.cfg.FillImage, b.cfg.PullPolicy); err != nil { + return fmt.Errorf("pulling fill image %q: %w", b.cfg.FillImage, err) + } + + args := buildFillArgs(b.cfg.ResolveFillCommand(), t, fillerIP, spec, snapshotHash) + + mounts := []docker.Mount{ + {Source: jwtPath, Target: fillJWTPath, Type: "bind", ReadOnly: true}, + {Source: t.OutputDir, Target: fillOutputPath, Type: "bind"}, + } + + if t.AddressStubsFile != "" { + mounts = append(mounts, docker.Mount{ + Source: t.AddressStubsFile, Target: fillStubsPath, Type: "bind", ReadOnly: true, + }) + } + + suffix, err := randSuffix() + if err != nil { + return fmt.Errorf("generating container name suffix: %w", err) + } + + containerSpec := &docker.ContainerSpec{ + Name: fmt.Sprintf("benchmarkoor-build-eest-fill-%s-%s", t.FillerClient, suffix), + Image: b.cfg.FillImage, + Command: args, + Mounts: mounts, + NetworkName: eestBuildNetwork, + Labels: b.labels(t), + } + + tail := newTailBuffer(64 * 1024) + out := io.MultiWriter(logWriter(log, logrus.InfoLevel), tail) + + log.WithField("argv", args).Info("Running fill-stateful") + + if err := b.mgr.RunInitContainer(ctx, containerSpec, out, out); err != nil { + return fmt.Errorf("running fill-stateful: %w (output tail: %s)", err, tail.String()) + } + + log.Info("Build completed") + + return nil +} + +// labels returns the standard label set for a builder container. +func (b *EESTPayloadsBuilder) labels(t *config.EESTPayloadTarget) map[string]string { + return map[string]string{ + "benchmarkoor.managed-by": "benchmarkoor", + "benchmarkoor.builder": EESTPayloadsBuilderName, + "benchmarkoor.client": t.FillerClient, + "benchmarkoor.target": t.EffectiveName(), + "benchmarkoor.output-dir": t.OutputDir, + } +} + +// fillerGethCommand builds the geth argv for the filler client. Only geth is +// supported today (validated in config), so the namespaces and flags are +// geth-specific: the http API exposes the testing/engine/miner namespaces +// fill-stateful needs, archive gcmode keeps full state, and peering is +// disabled. spec supplies the in-container paths and ports. +func fillerGethCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + args := []string{ + "--config=/tmp/config.toml", + "--datadir=" + spec.DataDir(), + "--port=0", + "--nodiscover", + "--maxpeers=0", + "--bootnodes=", + "--nat=none", + "--syncmode=full", + "--gcmode=archive", + "--snapshot=false", + "--http", + "--http.addr=0.0.0.0", + "--http.vhosts=*", + "--http.corsdomain=*", + "--http.api=admin,debug,eth,miner,net,txpool,web3,testing,engine", + "--http.port=" + strconv.Itoa(spec.RPCPort()), + "--authrpc.jwtsecret=" + spec.JWTPath(), + "--authrpc.addr=0.0.0.0", + "--authrpc.port=" + strconv.Itoa(spec.EnginePort()), + "--authrpc.vhosts=*", + "--miner.gaslimit=" + minerGasLimit, + } + + if t.GenesisFile != "" { + args = append(args, spec.GenesisFlag()+spec.GenesisPath()) + } + + return append(args, t.FillerExtraArgs...) +} + +// buildFillArgs assembles the fill-stateful argv: the configured command +// prefix, the live-client endpoints, the run knobs, and the test selection. +func buildFillArgs( + prefix []string, + t *config.EESTPayloadTarget, + fillerIP string, + spec client.Spec, + snapshotHash string, +) []string { + // NB: we deliberately do NOT pass --clean. fill-stateful's --clean does + // shutil.rmtree(output), which fails with EBUSY when output is a bind + // mount (it can't remove the mountpoint). benchmarkoor already owns the + // output_dir lifecycle — Build leaves it empty (skip-if-populated; --force + // wipes it) before we get here, so fill-stateful just mkdirs into it. + args := append([]string{}, prefix...) + args = append(args, + fmt.Sprintf("--rpc-endpoint=http://%s:%d", fillerIP, spec.RPCPort()), + fmt.Sprintf("--engine-endpoint=http://%s:%d", fillerIP, spec.EnginePort()), + "--engine-jwt-secret-file="+fillJWTPath, + "--fork="+t.Fork, + "--snapshot-block="+snapshotHash, + "--output="+fillOutputPath, + ) + + if t.GasBenchmarkValues != "" { + args = append(args, "--gas-benchmark-values="+t.GasBenchmarkValues) + } + + if t.MaxGasPerTest != nil { + args = append(args, fmt.Sprintf("--max-gas-per-test=%d", *t.MaxGasPerTest)) + } + + if t.RPCSeedKey != "" { + args = append(args, "--rpc-seed-key="+t.RPCSeedKey) + } + + if t.AddressStubsFile != "" { + args = append(args, "--address-stubs="+fillStubsPath) + } + + args = append(args, t.Tests...) + + if t.Filter != "" { + args = append(args, "-k", t.Filter) + } + + return args +} + +// writeTempJWT writes the JWT secret to a temp file readable by the +// container UID (0644) and returns its path plus a cleanup callback. +func writeTempJWT(secret string) (string, func(), error) { + f, err := os.CreateTemp("", "benchmarkoor-eest-jwt-*") + if err != nil { + return "", nil, fmt.Errorf("creating temp jwt file: %w", err) + } + + path := f.Name() + + cleanup := func() { _ = os.Remove(path) } + + if _, err := f.WriteString(secret); err != nil { + _ = f.Close() + cleanup() + + return "", nil, fmt.Errorf("writing temp jwt file: %w", err) + } + + if err := f.Close(); err != nil { + cleanup() + + return "", nil, fmt.Errorf("closing temp jwt file: %w", err) + } + + if err := os.Chmod(path, 0o644); err != nil { + cleanup() + + return "", nil, fmt.Errorf("chmod temp jwt file: %w", err) + } + + return path, cleanup, nil +} + +// writeTempConfigFiles materialises a client's in-container config files +// (path → content) to host temp files and returns the corresponding +// read-only bind mounts plus a cleanup callback. +func writeTempConfigFiles(files map[string]string) ([]docker.Mount, func(), error) { + mounts := make([]docker.Mount, 0, len(files)) + paths := make([]string, 0, len(files)) + + cleanup := func() { + for _, p := range paths { + _ = os.Remove(p) + } + } + + for target, content := range files { + f, err := os.CreateTemp("", "benchmarkoor-eest-config-*") + if err != nil { + cleanup() + + return nil, nil, fmt.Errorf("creating temp config file: %w", err) + } + + path := f.Name() + paths = append(paths, path) + + if _, err := f.WriteString(content); err != nil { + _ = f.Close() + cleanup() + + return nil, nil, fmt.Errorf("writing temp config file: %w", err) + } + + if err := f.Close(); err != nil { + cleanup() + + return nil, nil, fmt.Errorf("closing temp config file: %w", err) + } + + if err := os.Chmod(path, 0o644); err != nil { + cleanup() + + return nil, nil, fmt.Errorf("chmod temp config file: %w", err) + } + + mounts = append(mounts, docker.Mount{ + Source: path, Target: target, Type: "bind", ReadOnly: true, + }) + } + + return mounts, cleanup, nil +} diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go new file mode 100644 index 000000000..ab9bf3936 --- /dev/null +++ b/pkg/builder/eest_payloads_test.go @@ -0,0 +1,182 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/ethpandaops/benchmarkoor/pkg/client" + "github.com/ethpandaops/benchmarkoor/pkg/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildFillArgs(t *testing.T) { + spec := client.NewGethSpec() + prefix := []string{"uv", "run", "fill-stateful"} + + tests := []struct { + name string + target *config.EESTPayloadTarget + wantContain []string + wantAbsent []string + }{ + { + name: "minimal", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + Tests: []string{"tests/benchmark/compute"}, + }, + wantContain: []string{ + "uv", "run", "fill-stateful", + "--rpc-endpoint=http://10.0.0.5:8545", + "--engine-endpoint=http://10.0.0.5:8551", + "--engine-jwt-secret-file=" + fillJWTPath, + "--fork=Osaka", + "--snapshot-block=0xabc", + "--output=" + fillOutputPath, + "tests/benchmark/compute", + }, + wantAbsent: []string{ + "--clean", "--gas-benchmark-values", "--max-gas-per-test", "--rpc-seed-key", "--address-stubs", "-k", + }, + }, + { + name: "full", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + GasBenchmarkValues: "10,30", + MaxGasPerTest: u64(45000000), + RPCSeedKey: "0xdead", + AddressStubsFile: "/host/stubs.json", + Tests: []string{"tests/benchmark/compute", "tests/benchmark/stateful"}, + Filter: "bn128", + }, + wantContain: []string{ + "--gas-benchmark-values=10,30", + "--max-gas-per-test=45000000", + "--rpc-seed-key=0xdead", + "--address-stubs=" + fillStubsPath, + "tests/benchmark/compute", + "tests/benchmark/stateful", + "-k", "bn128", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildFillArgs(prefix, tt.target, "10.0.0.5", spec, "0xabc") + + for _, want := range tt.wantContain { + assert.Contains(t, got, want) + } + + for _, absent := range tt.wantAbsent { + for _, arg := range got { + assert.NotContains(t, arg, absent) + } + } + }) + } + + // The -k filter value must immediately follow the -k flag. + got := buildFillArgs(prefix, &config.EESTPayloadTarget{ + FillerClient: "geth", Fork: "Osaka", Tests: []string{"t"}, Filter: "expr", + }, "1.2.3.4", spec, "0x1") + idx := slices.Index(got, "-k") + require.GreaterOrEqual(t, idx, 0, "-k flag must be present") + require.Less(t, idx+1, len(got)) + assert.Equal(t, "expr", got[idx+1]) +} + +func TestFillerGethCommand(t *testing.T) { + spec := client.NewGethSpec() + + t.Run("without genesis", func(t *testing.T) { + cmd := fillerGethCommand(&config.EESTPayloadTarget{FillerClient: "geth"}, spec) + + assert.Contains(t, cmd, "--datadir=/data") + assert.Contains(t, cmd, "--http.api=admin,debug,eth,miner,net,txpool,web3,testing,engine") + assert.Contains(t, cmd, "--authrpc.jwtsecret=/tmp/jwtsecret") + assert.Contains(t, cmd, "--authrpc.port=8551") + assert.Contains(t, cmd, "--http.port=8545") + assert.Contains(t, cmd, "--gcmode=archive") + + for _, arg := range cmd { + assert.NotContains(t, arg, "--override.genesis", "no genesis flag when genesis_file unset") + } + }) + + t.Run("with genesis and extra args", func(t *testing.T) { + cmd := fillerGethCommand(&config.EESTPayloadTarget{ + FillerClient: "geth", + GenesisFile: "/host/genesis.json", + FillerExtraArgs: []string{"--verbosity=5"}, + }, spec) + + assert.Contains(t, cmd, "--override.genesis=/tmp/genesis.json") + assert.Contains(t, cmd, "--verbosity=5") + assert.Equal(t, "--verbosity=5", cmd[len(cmd)-1], "extra args are appended last") + }) +} + +func TestEESTPayloadsBuilder_Targets(t *testing.T) { + cfg := &config.EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []config.EESTPayloadTarget{ + {Name: "compute", FillerClient: "geth", OutputDir: "/srv/c"}, + {FillerClient: "geth", OutputDir: "/srv/g"}, + }, + } + + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}) + + got := b.Targets() + require.Len(t, got, 2) + assert.Equal(t, TargetInfo{Name: "compute", Client: "geth", OutputDir: "/srv/c"}, got[0]) + assert.Equal(t, TargetInfo{Name: "geth", Client: "geth", OutputDir: "/srv/g"}, got[1]) +} + +func TestEESTPayloadsBuilder_BuildUnknownTarget(t *testing.T) { + cfg := &config.EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []config.EESTPayloadTarget{{FillerClient: "geth", OutputDir: "/srv/g"}}, + } + + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}) + + _, err := b.Build(context.Background(), "nope", BuildOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no target named") +} + +func TestEESTPayloadsBuilder_BuildSkipsPopulatedDir(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "leftover"), []byte("x"), 0o644)) + + // fakeMgr panics on the orchestration methods; reaching them would fail + // the test, proving the skip path returns before any container work. + cfg := &config.EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []config.EESTPayloadTarget{{ + Name: "compute", FillerClient: "geth", SourceDir: "/snap", OutputDir: dir, + Fork: "Osaka", FillerImage: "geth:master", Tests: []string{"tests/benchmark/compute"}, + }}, + } + + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}) + + skipped, err := b.Build(context.Background(), "compute", BuildOptions{}) + require.NoError(t, err) + assert.True(t, skipped, "Build should report skipped=true when output_dir is non-empty") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "leftover", entries[0].Name()) +} diff --git a/pkg/builder/rpc.go b/pkg/builder/rpc.go new file mode 100644 index 000000000..f63f69960 --- /dev/null +++ b/pkg/builder/rpc.go @@ -0,0 +1,127 @@ +package builder + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// healthCheckInterval is how often waitForRPC polls the endpoint. +const healthCheckInterval = time.Second + +// waitForRPC blocks until the EL client at host:port answers +// web3_clientVersion or ctx is cancelled, returning the client version. +// These helpers mirror pkg/runner/rpc.go but are standalone so the builder +// does not depend on the runner. +func waitForRPC(ctx context.Context, host string, port int) (string, error) { + url := fmt.Sprintf("http://%s:%d", host, port) + + ticker := time.NewTicker(healthCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return "", fmt.Errorf("timeout waiting for RPC at %s: %w", url, ctx.Err()) + case <-ticker.C: + if version, ok := checkRPCHealth(ctx, url); ok { + return version, nil + } + } + } +} + +// getLatestBlockHash returns the hash of the latest block at host:port. +func getLatestBlockHash(ctx context.Context, host string, port int) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + url := fmt.Sprintf("http://%s:%d", host, port) + body := `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}` + + resp, err := postJSON(ctx, url, body) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("reading response: %w", err) + } + + var rpcResp struct { + Result struct { + Hash string `json:"hash"` + } `json:"result"` + } + + if err := json.Unmarshal(respBody, &rpcResp); err != nil { + return "", fmt.Errorf("parsing response: %w", err) + } + + if rpcResp.Result.Hash == "" { + return "", fmt.Errorf("latest block has no hash (client not ready?)") + } + + return rpcResp.Result.Hash, nil +} + +// checkRPCHealth performs a single web3_clientVersion call, returning the +// version on success. +func checkRPCHealth(ctx context.Context, url string) (string, bool) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + body := `{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":1}` + + resp, err := postJSON(ctx, url, body) + if err != nil { + return "", false + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", false + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", false + } + + var rpcResp struct { + Result string `json:"result"` + } + + if err := json.Unmarshal(respBody, &rpcResp); err != nil { + return "", false + } + + return rpcResp.Result, true +} + +// postJSON issues a JSON-RPC POST and returns the response. +func postJSON(ctx context.Context, url, body string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("executing request: %w", err) + } + + return resp, nil +} diff --git a/pkg/builder/state_actor.go b/pkg/builder/state_actor.go index 3c8fed08a..a95bb25be 100644 --- a/pkg/builder/state_actor.go +++ b/pkg/builder/state_actor.go @@ -1,10 +1,7 @@ package builder import ( - "bytes" "context" - "crypto/rand" - "encoding/hex" "fmt" "io" "os" @@ -115,7 +112,7 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp } } - if err := b.prepareOutputDir(target.OutputDir, force); err != nil { + if err := prepareOutputDir(target.OutputDir, force); err != nil { return false, err } @@ -193,38 +190,6 @@ func (b *StateActorBuilder) findTargetIndex(name string) int { return -1 } -// isPopulated reports whether dir exists and contains at least one -// entry. A missing dir returns (false, nil). -func isPopulated(dir string) (bool, error) { - entries, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - - return false, fmt.Errorf("reading output_dir %q: %w", dir, err) - } - - return len(entries) > 0, nil -} - -// prepareOutputDir ensures the target's output_dir exists. When force is -// true the directory is removed first; the populated check that gates -// skip-vs-build lives in Build, not here. -func (b *StateActorBuilder) prepareOutputDir(dir string, force bool) error { - if force { - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("removing existing output_dir %q: %w", dir, err) - } - } - - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("creating output_dir %q: %w", dir, err) - } - - return nil -} - // buildMounts assembles the bind mounts state-actor needs: an identity // mount of the output_dir (RW) and, when a spec path is resolved, a // read-only mount of that file at the same path inside the container so @@ -388,81 +353,5 @@ func dbPath(target *config.StateActorTarget) string { return target.OutputDir } -// randSuffix returns a 6-hex-character random string used to keep -// concurrent build container names unique. -func randSuffix() (string, error) { - var b [3]byte - - if _, err := rand.Read(b[:]); err != nil { - return "", err - } - - return hex.EncodeToString(b[:]), nil -} - -// logWriter returns an io.Writer that forwards each line written to it -// to the supplied logger at the given level. Used to stream container -// stdout/stderr without writing the bytes directly to the global -// stdout/stderr. -func logWriter(log logrus.FieldLogger, level logrus.Level) io.Writer { - return &lineLogger{log: log, level: level} -} - -type lineLogger struct { - log logrus.FieldLogger - level logrus.Level - buf bytes.Buffer -} - -func (w *lineLogger) Write(p []byte) (int, error) { - w.buf.Write(p) - - for { - i := bytes.IndexByte(w.buf.Bytes(), '\n') - if i < 0 { - break - } - - line := w.buf.Next(i + 1) - msg := string(bytes.TrimRight(line, "\r\n")) - - switch w.level { - case logrus.ErrorLevel: - w.log.Error(msg) - case logrus.WarnLevel: - w.log.Warn(msg) - case logrus.DebugLevel: - w.log.Debug(msg) - default: - w.log.Info(msg) - } - } - - return len(p), nil -} - -// tailBuffer is an io.Writer that retains at most `max` bytes of the -// most recent input. Useful for surfacing the trailing output of a -// failing container in the resulting error. -type tailBuffer struct { - buf bytes.Buffer - max int -} - -func newTailBuffer(maxBytes int) *tailBuffer { - return &tailBuffer{max: maxBytes} -} - -func (t *tailBuffer) Write(p []byte) (int, error) { - t.buf.Write(p) - - if excess := t.buf.Len() - t.max; excess > 0 { - t.buf.Next(excess) - } - - return len(p), nil -} - -func (t *tailBuffer) String() string { - return t.buf.String() -} +// randSuffix, logWriter/lineLogger and tailBuffer live in util.go — shared +// across builders. diff --git a/pkg/builder/util.go b/pkg/builder/util.go new file mode 100644 index 000000000..41fbcd59a --- /dev/null +++ b/pkg/builder/util.go @@ -0,0 +1,123 @@ +package builder + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "os" + + "github.com/sirupsen/logrus" +) + +// isPopulated reports whether dir exists and contains at least one +// entry. A missing dir returns (false, nil). +func isPopulated(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + + return false, fmt.Errorf("reading output_dir %q: %w", dir, err) + } + + return len(entries) > 0, nil +} + +// prepareOutputDir ensures dir exists. When force is true the directory is +// removed first; the populated check that gates skip-vs-build lives in each +// builder's Build, not here. +func prepareOutputDir(dir string, force bool) error { + if force { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("removing existing output_dir %q: %w", dir, err) + } + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating output_dir %q: %w", dir, err) + } + + return nil +} + +// randSuffix returns a 6-hex-character random string used to keep +// concurrent build container names unique. +func randSuffix() (string, error) { + var b [3]byte + + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + + return hex.EncodeToString(b[:]), nil +} + +// logWriter returns an io.Writer that forwards each line written to it to +// the supplied logger at the given level. Used to stream container +// stdout/stderr without writing the bytes directly to the global +// stdout/stderr. +func logWriter(log logrus.FieldLogger, level logrus.Level) io.Writer { + return &lineLogger{log: log, level: level} +} + +type lineLogger struct { + log logrus.FieldLogger + level logrus.Level + buf bytes.Buffer +} + +func (w *lineLogger) Write(p []byte) (int, error) { + w.buf.Write(p) + + for { + i := bytes.IndexByte(w.buf.Bytes(), '\n') + if i < 0 { + break + } + + line := w.buf.Next(i + 1) + msg := string(bytes.TrimRight(line, "\r\n")) + + switch w.level { + case logrus.ErrorLevel: + w.log.Error(msg) + case logrus.WarnLevel: + w.log.Warn(msg) + case logrus.DebugLevel: + w.log.Debug(msg) + default: + w.log.Info(msg) + } + } + + return len(p), nil +} + +// tailBuffer is an io.Writer that retains at most `max` bytes of the most +// recent input. Useful for surfacing the trailing output of a failing +// container in the resulting error. +type tailBuffer struct { + buf bytes.Buffer + max int +} + +func newTailBuffer(maxBytes int) *tailBuffer { + return &tailBuffer{max: maxBytes} +} + +func (t *tailBuffer) Write(p []byte) (int, error) { + t.buf.Write(p) + + if excess := t.buf.Len() - t.max; excess > 0 { + t.buf.Next(excess) + } + + return len(p), nil +} + +func (t *tailBuffer) String() string { + return t.buf.String() +} diff --git a/pkg/config/config.go b/pkg/config/config.go index a67410365..4d610070a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -73,11 +73,13 @@ type Config struct { Builder *BuilderConfig `yaml:"builder,omitempty" mapstructure:"builder"` } -// BuilderConfig is the top-level builder block. Today it only houses -// state-actor; future builders (e.g. geth-import, snap-sync) plug in -// alongside. +// BuilderConfig is the top-level builder block. It houses state-actor +// (materialises pre-populated datadirs) and eest_payloads (generates +// stateful EEST benchmark fixtures against such a datadir). Future +// builders plug in alongside. type BuilderConfig struct { - StateActor *StateActorConfig `yaml:"state_actor,omitempty" mapstructure:"state_actor"` + StateActor *StateActorConfig `yaml:"state_actor,omitempty" mapstructure:"state_actor"` + EESTPayloads *EESTPayloadsConfig `yaml:"eest_payloads,omitempty" mapstructure:"eest_payloads"` } // StateActorConfig configures how the state-actor binary is invoked via @@ -276,6 +278,147 @@ var stateActorValidPullPolicies = map[string]bool{ "never": true, } +// EESTPayloadsConfig configures generation of stateful EEST benchmark +// fixtures via the `fill-stateful` command (execution-specs). See +// https://github.com/ethereum/execution-specs/pull/2637. +// +// Unlike state-actor, this builder is an orchestrator: per target it +// boots a filler EL client on a copy of a pre-populated snapshot datadir +// (e.g. produced by builder.state_actor), runs fill-stateful against the +// live client, and writes the resulting fixtures to the target's +// output_dir. The fixtures are later replayed by `benchmarkoor run`. +// +// FillImage is the container image carrying the `fill-stateful` command +// (uv + execution-specs). Config holds per-target defaults that can be +// hoisted to avoid repetition; any field set on a target wins. +type EESTPayloadsConfig struct { + ContainerRuntime string `yaml:"container_runtime,omitempty" mapstructure:"container_runtime"` + FillImage string `yaml:"fill_image,omitempty" mapstructure:"fill_image"` + PullPolicy string `yaml:"pull_policy,omitempty" mapstructure:"pull_policy"` + JWT string `yaml:"jwt,omitempty" mapstructure:"jwt"` + // FillCommand is the argv prefix invoked inside FillImage before the + // fill-stateful flags. Defaults to ["uv", "run", "fill-stateful"]. + FillCommand []string `yaml:"fill_command,omitempty" mapstructure:"fill_command"` + Config *EESTPayloadDefaults `yaml:"config,omitempty" mapstructure:"config"` + Targets []EESTPayloadTarget `yaml:"targets,omitempty" mapstructure:"targets"` +} + +// DefaultFillCommand is the argv prefix used to invoke fill-stateful inside +// the fill image when EESTPayloadsConfig.FillCommand is unset. +var DefaultFillCommand = []string{"uv", "run", "fill-stateful"} + +// ResolveFillCommand returns the configured fill-stateful argv prefix, or +// DefaultFillCommand when unset. +func (e *EESTPayloadsConfig) ResolveFillCommand() []string { + if len(e.FillCommand) > 0 { + return e.FillCommand + } + + return DefaultFillCommand +} + +// EESTPayloadDefaults are the per-target build parameters that may be +// hoisted to the top level under `builder.eest_payloads.config`. Every +// field is also present on EESTPayloadTarget; a non-nil/non-empty value +// on the target wins over the corresponding default. See ResolveTarget. +type EESTPayloadDefaults struct { + FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` + Fork string `yaml:"fork,omitempty" mapstructure:"fork"` + GasBenchmarkValues string `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` + DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` + MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` + RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` + FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` +} + +// EESTPayloadTarget is one fixture-generation run. Identity/locator fields +// (Name, FillerClient, SourceDir, OutputDir, GenesisFile, Tests, Filter, +// AddressStubsFile) live exclusively on the target; the remaining fields +// mirror EESTPayloadDefaults and are resolved via ResolveTarget. +type EESTPayloadTarget struct { + Name string `yaml:"name,omitempty" mapstructure:"name"` + FillerClient string `yaml:"filler_client" mapstructure:"filler_client"` + SourceDir string `yaml:"source_dir" mapstructure:"source_dir"` + OutputDir string `yaml:"output_dir" mapstructure:"output_dir"` + GenesisFile string `yaml:"genesis_file,omitempty" mapstructure:"genesis_file"` + AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` + // Tests are pytest paths inside the fill image, e.g. tests/benchmark/compute. + Tests []string `yaml:"tests,omitempty" mapstructure:"tests"` + Filter string `yaml:"filter,omitempty" mapstructure:"filter"` + Force bool `yaml:"force,omitempty" mapstructure:"force"` + + // Hoistable fields (mirror EESTPayloadDefaults). + FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` + Fork string `yaml:"fork,omitempty" mapstructure:"fork"` + GasBenchmarkValues string `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` + DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` + MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` + RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` + FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` +} + +// ResolveTarget returns a copy of the i-th target with any unset hoistable +// fields filled in from EESTPayloadsConfig.Config. Identity/locator fields +// are never touched. Per-target value wins when set (non-nil for pointer +// types, non-empty for strings/slices); otherwise the value from Config is +// used. When Config is nil, the target is returned unchanged. +func (e *EESTPayloadsConfig) ResolveTarget(i int) EESTPayloadTarget { + t := e.Targets[i] + if e.Config == nil { + return t + } + + g := e.Config + + if t.FillerImage == "" { + t.FillerImage = g.FillerImage + } + + if t.Fork == "" { + t.Fork = g.Fork + } + + if t.GasBenchmarkValues == "" { + t.GasBenchmarkValues = g.GasBenchmarkValues + } + + if t.DataDirMethod == "" { + t.DataDirMethod = g.DataDirMethod + } + + if t.MaxGasPerTest == nil { + t.MaxGasPerTest = g.MaxGasPerTest + } + + if t.RPCSeedKey == "" { + t.RPCSeedKey = g.RPCSeedKey + } + + if len(t.FillerExtraArgs) == 0 { + t.FillerExtraArgs = g.FillerExtraArgs + } + + return t +} + +// EffectiveName returns the target's user-facing name, defaulting to the +// filler client when Name was not set — matching the `--target` filter +// behaviour in the build command. +func (t *EESTPayloadTarget) EffectiveName() string { + if t.Name != "" { + return t.Name + } + + return t.FillerClient +} + +// eestFillerSupportedClients lists the clients that can act as the +// fill-stateful filler. Only geth implements testing_buildBlockV1 today +// (ethpandaops/geth:master is the production-ready filler image). +var eestFillerSupportedClients = map[string]struct{}{ + "geth": {}, +} + // RunnerConfig contains all run-specific configuration settings. type RunnerConfig struct { ContainerRuntime string `yaml:"container_runtime,omitempty" mapstructure:"container_runtime"` @@ -551,19 +694,20 @@ func (e *EESTFixturesSource) validate() error { // Validate local dir mode. if hasLocalDir { if e.LocalFixturesDir == "" { - return fmt.Errorf("eest_fixtures: local_fixtures_dir is required when local_genesis_dir is set") - } - - if e.LocalGenesisDir == "" { - return fmt.Errorf("eest_fixtures: local_genesis_dir is required when local_fixtures_dir is set") + return fmt.Errorf("eest_fixtures: local_fixtures_dir is required for local directory mode") } if err := validateDirExists(e.LocalFixturesDir, "eest_fixtures.local_fixtures_dir"); err != nil { return err } - if err := validateDirExists(e.LocalGenesisDir, "eest_fixtures.local_genesis_dir"); err != nil { - return err + // local_genesis_dir is optional: stateful-engine fixtures boot from a + // pre-populated snapshot datadir (configured via runner.client.datadirs) + // and carry no genesis. Validate it only when provided. + if e.LocalGenesisDir != "" { + if err := validateDirExists(e.LocalGenesisDir, "eest_fixtures.local_genesis_dir"); err != nil { + return err + } } } @@ -1182,6 +1326,8 @@ func bindEnvKeys(v *viper.Viper) { // Builder settings "builder.state_actor.container_runtime", "builder.state_actor.pull_policy", + "builder.eest_payloads.container_runtime", + "builder.eest_payloads.pull_policy", } for _, key := range keys { @@ -1310,6 +1456,19 @@ func (c *Config) applyDefaults() { c.Builder.StateActor.PullPolicy = DefaultPullPolicy } } + + // Apply builder.eest_payloads defaults. ContainerRuntime is left empty + // so GetEESTPayloadsContainerRuntime can fall back at call time; JWT + // defaults to DefaultJWT so the filler client and fill-stateful share it. + if c.Builder != nil && c.Builder.EESTPayloads != nil { + if c.Builder.EESTPayloads.PullPolicy == "" { + c.Builder.EESTPayloads.PullPolicy = DefaultPullPolicy + } + + if c.Builder.EESTPayloads.JWT == "" { + c.Builder.EESTPayloads.JWT = DefaultJWT + } + } } // GetStateActorContainerRuntime returns the container runtime to use for @@ -1323,6 +1482,17 @@ func (c *Config) GetStateActorContainerRuntime() string { return c.GetContainerRuntime() } +// GetEESTPayloadsContainerRuntime returns the container runtime to use for +// eest_payloads builds. Falls back to the runner's runtime when the builder +// block does not override it. +func (c *Config) GetEESTPayloadsContainerRuntime() string { + if c.Builder != nil && c.Builder.EESTPayloads != nil && c.Builder.EESTPayloads.ContainerRuntime != "" { + return c.Builder.EESTPayloads.ContainerRuntime + } + + return c.GetContainerRuntime() +} + // ValidateOpts controls optional validation behavior. type ValidateOpts struct { // ActiveInstanceIDs limits validation to instances with these IDs. @@ -1565,11 +1735,24 @@ func (c *Config) ValidateBuilder() error { return c.validateBuilder() } -// validateBuilder enforces the builder.state_actor rules: supported +// validateBuilder validates each configured builder block. +func (c *Config) validateBuilder() error { + if c.Builder == nil { + return nil + } + + if err := c.validateStateActor(); err != nil { + return err + } + + return c.validateEESTPayloads() +} + +// validateStateActor enforces the builder.state_actor rules: supported // clients, single-source-of-truth target_size XOR spec, archive/binary-trie // applicability, group_depth range, image resolvability, and uniqueness of // target names and output_dirs. -func (c *Config) validateBuilder() error { +func (c *Config) validateStateActor() error { if c.Builder == nil || c.Builder.StateActor == nil { return nil } @@ -1701,6 +1884,182 @@ func (c *Config) validateBuilder() error { return nil } +// validateEESTPayloads enforces the builder.eest_payloads rules: a +// configured fill_image, supported filler clients, required locator fields +// (source_dir/output_dir/tests/fork), valid datadir method and +// gas-benchmark values, absolute paths, and uniqueness of target names and +// output_dirs. Existence of source_dir/genesis_file/address_stubs_file is +// checked at build time, not here — a state-actor target earlier in the +// same config may still need to produce them. +func (c *Config) validateEESTPayloads() error { + if c.Builder == nil || c.Builder.EESTPayloads == nil { + return nil + } + + ep := c.Builder.EESTPayloads + + if !validContainerRuntimes[ep.ContainerRuntime] { + return fmt.Errorf( + "builder.eest_payloads.container_runtime: invalid value %q "+ + "(must be \"docker\" or \"podman\")", ep.ContainerRuntime, + ) + } + + if !stateActorValidPullPolicies[ep.PullPolicy] { + return fmt.Errorf( + "builder.eest_payloads.pull_policy: invalid value %q "+ + "(must be \"always\", \"if-not-present\", or \"never\")", + ep.PullPolicy, + ) + } + + if ep.FillImage == "" { + return fmt.Errorf( + "builder.eest_payloads.fill_image is required " + + "(the container image carrying the fill-stateful command)", + ) + } + + seenOutputs := make(map[string]int, len(ep.Targets)) + seenNames := make(map[string]int, len(ep.Targets)) + + for i := range ep.Targets { + t := ep.ResolveTarget(i) + prefix := fmt.Sprintf("builder.eest_payloads.targets[%d]", i) + + if _, ok := eestFillerSupportedClients[t.FillerClient]; !ok { + return fmt.Errorf( + "%s.filler_client: %q cannot act as the fill-stateful filler "+ + "(only geth implements testing_buildBlockV1 today)", + prefix, t.FillerClient, + ) + } + + name := t.EffectiveName() + if prev, dup := seenNames[name]; dup { + return fmt.Errorf( + "%s: name %q duplicates targets[%d] (set an explicit name to disambiguate)", + prefix, name, prev, + ) + } + + seenNames[name] = i + + if err := validateEESTPayloadPaths(&t, prefix, seenOutputs, i); err != nil { + return err + } + + if len(t.Tests) == 0 { + return fmt.Errorf( + "%s.tests is required (at least one pytest path, e.g. tests/benchmark/compute)", + prefix, + ) + } + + if t.Fork == "" { + return fmt.Errorf( + "%s.fork is required (set it on the target or builder.eest_payloads.config.fork)", + prefix, + ) + } + + if t.FillerImage == "" { + return fmt.Errorf( + "%s.filler_image is required (e.g. ethpandaops/geth:master)", prefix, + ) + } + + if !validDataDirMethods[t.DataDirMethod] { + return fmt.Errorf( + "%s.datadir_method: invalid value %q "+ + "(must be copy, overlayfs, fuse-overlayfs, zfs, direct, or schelk)", + prefix, t.DataDirMethod, + ) + } + + if err := validateGasBenchmarkValues(t.GasBenchmarkValues, prefix); err != nil { + return err + } + } + + return nil +} + +// validateEESTPayloadPaths checks output_dir / genesis_file / +// address_stubs_file are absolute and output_dir is unique. +func validateEESTPayloadPaths(t *EESTPayloadTarget, prefix string, seenOutputs map[string]int, i int) error { + if t.SourceDir == "" { + return fmt.Errorf("%s.source_dir is required", prefix) + } + + if !filepath.IsAbs(t.SourceDir) { + return fmt.Errorf("%s.source_dir must be an absolute path, got %q", prefix, t.SourceDir) + } + + if t.OutputDir == "" { + return fmt.Errorf("%s.output_dir is required", prefix) + } + + if !filepath.IsAbs(t.OutputDir) { + return fmt.Errorf("%s.output_dir must be an absolute path, got %q", prefix, t.OutputDir) + } + + if prev, dup := seenOutputs[t.OutputDir]; dup { + return fmt.Errorf( + "%s.output_dir %q duplicates targets[%d].output_dir", prefix, t.OutputDir, prev, + ) + } + + seenOutputs[t.OutputDir] = i + + if t.GenesisFile != "" && !filepath.IsAbs(t.GenesisFile) { + return fmt.Errorf("%s.genesis_file must be an absolute path, got %q", prefix, t.GenesisFile) + } + + if t.AddressStubsFile != "" && !filepath.IsAbs(t.AddressStubsFile) { + return fmt.Errorf( + "%s.address_stubs_file must be an absolute path, got %q", prefix, t.AddressStubsFile, + ) + } + + return nil +} + +// validateGasBenchmarkValues checks a comma-separated list of positive +// integers (millions of gas), e.g. "10,30". Empty is allowed. +func validateGasBenchmarkValues(values, prefix string) error { + if values == "" { + return nil + } + + for _, v := range strings.Split(values, ",") { + v = strings.TrimSpace(v) + if v == "" { + return fmt.Errorf("%s.gas_benchmark_values: empty value in %q", prefix, values) + } + + if _, err := strconv.ParseUint(v, 10, 64); err != nil { + return fmt.Errorf( + "%s.gas_benchmark_values: %q is not a comma-separated list of integers", prefix, values, + ) + } + } + + return nil +} + +// validDataDirMethods mirrors the datadir.method vocabulary accepted by +// pkg/datadir.NewProvider and DataDirConfig validation. +var validDataDirMethods = map[string]bool{ + "": true, // unset → copy + "copy": true, + "overlayfs": true, + "fuse-overlayfs": true, + "zfs": true, + "direct": true, + "schelk": true, +} + // validateLiveReporting checks the runner.live_reporting config when enabled. func (c *Config) validateLiveReporting() error { lr := c.Runner.LiveReporting diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 934cb1452..7f4256bce 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -452,14 +452,13 @@ func TestSourceConfig_Validate(t *testing.T) { wantErr: false, }, { - name: "eest_fixtures local dir missing local_genesis_dir", + name: "eest_fixtures local dir without local_genesis_dir (stateful)", source: SourceConfig{ EESTFixtures: &EESTFixturesSource{ LocalFixturesDir: tmpDir, }, }, - wantErr: true, - errSubstr: "local_genesis_dir is required", + wantErr: false, }, { name: "eest_fixtures local dir missing local_fixtures_dir", @@ -3665,3 +3664,261 @@ func TestGetStateActorContainerRuntime(t *testing.T) { assert.Equal(t, "docker", cfg.GetStateActorContainerRuntime()) }) } + +func TestValidateEESTPayloads(t *testing.T) { + dirA := t.TempDir() + dirB := t.TempDir() + + // mkCfg builds a Config with only the eest_payloads builder block so the + // builder validation runs in isolation. + mkCfg := func(ep *EESTPayloadsConfig) *Config { + return &Config{Builder: &BuilderConfig{EESTPayloads: ep}} + } + + // base returns a minimal valid target rooted at dir. + base := func(dir string) EESTPayloadTarget { + return EESTPayloadTarget{ + FillerClient: "geth", + FillerImage: "ethpandaops/geth:master", + SourceDir: "/snap", + OutputDir: dir, + Fork: "Osaka", + Tests: []string{"tests/benchmark/compute"}, + } + } + + tests := []struct { + name string + ep *EESTPayloadsConfig + wantErr bool + errSubstr string + }{ + { + name: "nil builder is fine", + ep: nil, + }, + { + name: "valid minimal", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, + { + name: "missing fill_image", + ep: &EESTPayloadsConfig{ + Targets: []EESTPayloadTarget{base(dirA)}, + }, + wantErr: true, + errSubstr: "fill_image", + }, + { + name: "invalid container_runtime", + ep: &EESTPayloadsConfig{ + ContainerRuntime: "lima", + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + wantErr: true, + errSubstr: "container_runtime", + }, + { + name: "unsupported filler client", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FillerClient = "reth" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "testing_buildBlockV1", + }, + { + name: "missing tests", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.Tests = nil + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "tests is required", + }, + { + name: "missing fork", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.Fork = "" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "fork is required", + }, + { + name: "fork hoisted from config defaults", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.Fork = "" + + return &EESTPayloadsConfig{ + FillImage: "fill:latest", + Config: &EESTPayloadDefaults{Fork: "Osaka"}, + Targets: []EESTPayloadTarget{tgt}, + } + }(), + }, + { + name: "relative source_dir", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.SourceDir = "relative/path" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "source_dir must be an absolute path", + }, + { + name: "relative output_dir", + ep: func() *EESTPayloadsConfig { + tgt := base("relative/out") + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "output_dir must be an absolute path", + }, + { + name: "duplicate output_dir", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{ + func() EESTPayloadTarget { t := base(dirA); t.Name = "a"; return t }(), + func() EESTPayloadTarget { t := base(dirA); t.Name = "b"; return t }(), + }, + }, + wantErr: true, + errSubstr: "duplicates", + }, + { + name: "duplicate name", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + Targets: []EESTPayloadTarget{ + func() EESTPayloadTarget { t := base(dirA); t.Name = "dup"; return t }(), + func() EESTPayloadTarget { t := base(dirB); t.Name = "dup"; return t }(), + }, + }, + wantErr: true, + errSubstr: "duplicates", + }, + { + name: "invalid datadir_method", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.DataDirMethod = "btrfs" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "datadir_method", + }, + { + name: "invalid gas_benchmark_values", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.GasBenchmarkValues = "10,abc" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "gas_benchmark_values", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := mkCfg(tt.ep).validateBuilder() + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + + return + } + + require.NoError(t, err) + }) + } +} + +func TestEESTPayloadsResolveTarget(t *testing.T) { + ep := &EESTPayloadsConfig{ + Config: &EESTPayloadDefaults{ + FillerImage: "ethpandaops/geth:master", + Fork: "Osaka", + GasBenchmarkValues: "10,30", + DataDirMethod: "zfs", + MaxGasPerTest: u64Cfg(45000000), + RPCSeedKey: "0xseed", + FillerExtraArgs: []string{"--verbosity=3"}, + }, + Targets: []EESTPayloadTarget{ + // Inherits everything from config. + {Name: "inherit", FillerClient: "geth", SourceDir: "/s", OutputDir: "/o"}, + // Overrides fork and gas values. + {Name: "override", FillerClient: "geth", SourceDir: "/s2", OutputDir: "/o2", + Fork: "Prague", GasBenchmarkValues: "60"}, + }, + } + + inherit := ep.ResolveTarget(0) + assert.Equal(t, "ethpandaops/geth:master", inherit.FillerImage) + assert.Equal(t, "Osaka", inherit.Fork) + assert.Equal(t, "10,30", inherit.GasBenchmarkValues) + assert.Equal(t, "zfs", inherit.DataDirMethod) + require.NotNil(t, inherit.MaxGasPerTest) + assert.Equal(t, uint64(45000000), *inherit.MaxGasPerTest) + assert.Equal(t, []string{"--verbosity=3"}, inherit.FillerExtraArgs) + + override := ep.ResolveTarget(1) + assert.Equal(t, "Prague", override.Fork, "per-target fork wins") + assert.Equal(t, "60", override.GasBenchmarkValues, "per-target gas values win") + assert.Equal(t, "ethpandaops/geth:master", override.FillerImage, "still inherits unset fields") +} + +func TestEESTPayloadsEffectiveName(t *testing.T) { + withName := EESTPayloadTarget{Name: "compute", FillerClient: "geth"} + assert.Equal(t, "compute", withName.EffectiveName()) + + noName := EESTPayloadTarget{FillerClient: "geth"} + assert.Equal(t, "geth", noName.EffectiveName()) +} + +func TestEESTPayloadsResolveFillCommand(t *testing.T) { + def := (&EESTPayloadsConfig{}).ResolveFillCommand() + assert.Equal(t, []string{"uv", "run", "fill-stateful"}, def) + + custom := (&EESTPayloadsConfig{FillCommand: []string{"fill-stateful"}}).ResolveFillCommand() + assert.Equal(t, []string{"fill-stateful"}, custom) +} + +func TestGetEESTPayloadsContainerRuntime(t *testing.T) { + t.Run("builder override wins", func(t *testing.T) { + cfg := &Config{ + Runner: RunnerConfig{ContainerRuntime: "docker"}, + Builder: &BuilderConfig{EESTPayloads: &EESTPayloadsConfig{ContainerRuntime: "podman"}}, + } + assert.Equal(t, "podman", cfg.GetEESTPayloadsContainerRuntime()) + }) + + t.Run("falls back to runner runtime", func(t *testing.T) { + cfg := &Config{ + Runner: RunnerConfig{ContainerRuntime: "podman"}, + Builder: &BuilderConfig{EESTPayloads: &EESTPayloadsConfig{}}, + } + assert.Equal(t, "podman", cfg.GetEESTPayloadsContainerRuntime()) + }) +} + +func u64Cfg(v uint64) *uint64 { return &v } diff --git a/pkg/eest/converter.go b/pkg/eest/converter.go index 063671aca..0ee72f4a5 100644 --- a/pkg/eest/converter.go +++ b/pkg/eest/converter.go @@ -63,6 +63,73 @@ func ConvertFixture(name string, fixture *Fixture) (*ConvertedTest, error) { return result, nil } +// ConvertStatefulFixture converts a stateful-engine fixture to JSON-RPC calls. +// Replay boots from a snapshot datadir rather than a genesis, so the setup +// phase is the shared pre_run payloads (snapshot → start block, preRun may be +// nil) followed by the fixture's own setupEngineNewPayloads (start block → +// per-test pre-state). The fixture's engineNewPayloads (the benchmark block) +// become the measured test step. Each payload still emits an +// engine_newPayload + engine_forkchoiceUpdated pair, so the chain head +// advances naturally and no separate forkchoice injection is needed. +func ConvertStatefulFixture(name string, fixture *Fixture, preRun *StatefulPreRun) (*ConvertedTest, error) { + if fixture == nil { + return nil, fmt.Errorf("fixture is nil") + } + + if len(fixture.EngineNewPayloads) == 0 { + return nil, fmt.Errorf("fixture has no benchmark payloads") + } + + // Setup = shared pre_run payloads, then the fixture's own setup payloads. + setupPayloads := make([]*EngineNewPayload, 0, + len(fixture.SetupEngineNewPayloads)+preRunPayloadCount(preRun)) + + if preRun != nil { + setupPayloads = append(setupPayloads, preRun.EngineNewPayloads...) + } + + setupPayloads = append(setupPayloads, fixture.SetupEngineNewPayloads...) + + result := &ConvertedTest{ + Name: name, + SetupLines: make([]string, 0, len(setupPayloads)*2), + TestLines: make([]string, 0, len(fixture.EngineNewPayloads)*2), + GenesisHash: fixture.SnapshotBlockHash, + PayloadCount: len(setupPayloads) + len(fixture.EngineNewPayloads), + } + + for i, payload := range setupPayloads { + lines, err := convertPayload(payload, i+1) + if err != nil { + return nil, fmt.Errorf("converting setup payload %d: %w", i, err) + } + + result.SetupLines = append(result.SetupLines, lines...) + } + + for i, payload := range fixture.EngineNewPayloads { + lines, err := convertPayload(payload, len(setupPayloads)+i+1) + if err != nil { + return nil, fmt.Errorf("converting benchmark payload %d: %w", i, err) + } + + result.TestLines = append(result.TestLines, lines...) + result.FinalHash = payload.ExecutionPayload.BlockHash + } + + return result, nil +} + +// preRunPayloadCount returns the number of pre_run payloads, tolerating a nil +// pre_run (capacity hint only). +func preRunPayloadCount(preRun *StatefulPreRun) int { + if preRun == nil { + return 0 + } + + return len(preRun.EngineNewPayloads) +} + // convertPayload generates JSON-RPC lines for a single payload. func convertPayload(payload *EngineNewPayload, id int) ([]string, error) { if payload.ExecutionPayload == nil { diff --git a/pkg/eest/converter_test.go b/pkg/eest/converter_test.go index fa4b7a513..a70d650f0 100644 --- a/pkg/eest/converter_test.go +++ b/pkg/eest/converter_test.go @@ -220,6 +220,160 @@ func TestConvertFixture_PayloadVersions(t *testing.T) { } } +// statefulPayload builds a minimal EngineNewPayload for stateful conversion +// tests, with the given block number/hash/parent. +func statefulPayload(number, hash, parent string) *EngineNewPayload { + return &EngineNewPayload{ + ExecutionPayload: &ExecutionPayload{ + ParentHash: parent, + FeeRecipient: "0xfee", + StateRoot: "0xstate", + ReceiptsRoot: "0xreceipts", + LogsBloom: "0xbloom", + PrevRandao: "0xrandao", + BlockNumber: number, + GasLimit: "0x1000000", + GasUsed: "0x0", + Timestamp: "0x100", + ExtraData: "0x", + BaseFeePerGas: "0x7", + BlockHash: hash, + Transactions: []string{}, + }, + NewPayloadVersion: 4, + ForkchoiceUpdatedVersion: 3, + BlobVersionedHashes: []string{}, + ParentBeaconBlockRoot: "0xbeacon", + ExecutionRequests: []string{}, + } +} + +func TestConvertStatefulFixture(t *testing.T) { + preRun := &StatefulPreRun{ + SnapshotBlockHash: "0xsnapshot", + StartBlockHash: "0xstart", + // snapshot (0x0) -> start (0x3): three pre_run blocks. + EngineNewPayloads: []*EngineNewPayload{ + statefulPayload("0x1", "0xb1", "0xsnapshot"), + statefulPayload("0x2", "0xb2", "0xb1"), + statefulPayload("0x3", "0xstart", "0xb2"), + }, + } + + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + Network: "Osaka", + SnapshotBlockHash: "0xsnapshot", + StartBlockHash: "0xstart", + LastBlockHash: "0xbench", + // start (0x3) -> setup (0x4). + SetupEngineNewPayloads: []*EngineNewPayload{ + statefulPayload("0x4", "0xsetup", "0xstart"), + }, + // setup (0x4) -> benchmark (0x5): the measured block. + EngineNewPayloads: []*EngineNewPayload{ + statefulPayload("0x5", "0xbench", "0xsetup"), + }, + } + + result, err := ConvertStatefulFixture("test_stateful", fixture, preRun) + require.NoError(t, err) + + assert.Equal(t, "test_stateful", result.Name) + // GenesisHash carries the snapshot hash for reporting. + assert.Equal(t, "0xsnapshot", result.GenesisHash) + assert.Equal(t, "0xbench", result.FinalHash) + // 3 pre_run + 1 setup + 1 benchmark = 5 payloads. + assert.Equal(t, 5, result.PayloadCount) + // Setup = (3 pre_run + 1 setup) * 2 lines (newPayload + fcU). + assert.Len(t, result.SetupLines, 8) + // Test = 1 benchmark * 2 lines. + assert.Len(t, result.TestLines, 2) + + // First setup line replays the first pre_run block. + var rpcCall map[string]any + require.NoError(t, json.Unmarshal([]byte(result.SetupLines[0]), &rpcCall)) + assert.Equal(t, "engine_newPayloadV4", rpcCall["method"]) + + // The benchmark newPayload is the test step. + require.NoError(t, json.Unmarshal([]byte(result.TestLines[0]), &rpcCall)) + assert.Equal(t, "engine_newPayloadV4", rpcCall["method"]) +} + +func TestConvertStatefulFixture_NilPreRun(t *testing.T) { + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + SnapshotBlockHash: "0xsnapshot", + SetupEngineNewPayloads: []*EngineNewPayload{statefulPayload("0x4", "0xsetup", "0xstart")}, + EngineNewPayloads: []*EngineNewPayload{statefulPayload("0x5", "0xbench", "0xsetup")}, + } + + result, err := ConvertStatefulFixture("test_stateful", fixture, nil) + require.NoError(t, err) + + // Without pre_run, only the fixture's own setup payload is replayed. + assert.Len(t, result.SetupLines, 2) + assert.Len(t, result.TestLines, 2) + assert.Equal(t, 2, result.PayloadCount) +} + +func TestConvertStatefulFixture_NoBenchmarkPayloads(t *testing.T) { + fixture := &Fixture{ + Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}, + SetupEngineNewPayloads: []*EngineNewPayload{statefulPayload("0x4", "0xsetup", "0xstart")}, + EngineNewPayloads: []*EngineNewPayload{}, + } + + _, err := ConvertStatefulFixture("test", fixture, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no benchmark payloads") +} + +func TestConvertStatefulFixture_NilFixture(t *testing.T) { + _, err := ConvertStatefulFixture("test", nil, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "fixture is nil") +} + +func TestParsePreRunFile(t *testing.T) { + jsonData := `{ + "network": "Osaka", + "snapshotBlockHash": "0xsnapshot", + "startBlockHash": "0xstart", + "engineNewPayloads": [ + { + "newPayloadVersion": "4", + "forkchoiceUpdatedVersion": "3", + "params": [ + {"parentHash":"0xsnapshot","feeRecipient":"0xfee","stateRoot":"0xs", + "receiptsRoot":"0xr","logsBloom":"0xb","prevRandao":"0xrd", + "blockNumber":"0x1","gasLimit":"0x1000000","gasUsed":"0x0", + "timestamp":"0x100","extraData":"0x","baseFeePerGas":"0x7", + "blockHash":"0xstart","transactions":[]}, + [], "0xbeacon", [] + ] + } + ] + }` + + preRun, err := ParsePreRunFile([]byte(jsonData)) + require.NoError(t, err) + assert.Equal(t, "0xstart", preRun.StartBlockHash) + assert.Equal(t, "0xsnapshot", preRun.SnapshotBlockHash) + require.Len(t, preRun.EngineNewPayloads, 1) + assert.Equal(t, 4, preRun.EngineNewPayloads[0].NewPayloadVersion) +} + +func TestFixture_IsStateful(t *testing.T) { + stateful := &Fixture{Info: &FixtureInfo{FixtureFormat: SupportedStatefulFixtureFormat}} + assert.True(t, stateful.IsStateful()) + assert.True(t, stateful.IsSupportedFormat()) + + genesisBased := &Fixture{Info: &FixtureInfo{FixtureFormat: SupportedFixtureFormat}} + assert.False(t, genesisBased.IsStateful()) + assert.True(t, genesisBased.IsSupportedFormat()) +} + func TestParseFixtureFile(t *testing.T) { jsonData := `{ "test_one": { diff --git a/pkg/eest/fixture.go b/pkg/eest/fixture.go index 37cd563e5..9b2877de4 100644 --- a/pkg/eest/fixture.go +++ b/pkg/eest/fixture.go @@ -6,15 +6,42 @@ import ( "strconv" ) -// SupportedFixtureFormat is the fixture format we support. +// SupportedFixtureFormat is the genesis-based fixture format we support. const SupportedFixtureFormat = "blockchain_test_engine_x" +// SupportedStatefulFixtureFormat is the stateful-engine fixture format. Unlike +// the genesis-based format, these fixtures boot from a pre-populated snapshot +// datadir (no genesis): shared pre_run payloads advance the snapshot to a +// common start block, SetupEngineNewPayloads bring the per-test pre-state into +// place, and EngineNewPayloads carry the benchmark block. +const SupportedStatefulFixtureFormat = "blockchain_test_stateful_engine" + // Fixture represents a single EEST test fixture. type Fixture struct { Info *FixtureInfo `json:"_info"` Network string `json:"network"` GenesisBlockHeader *BlockHeader `json:"genesisBlockHeader"` EngineNewPayloads []*EngineNewPayload `json:"engineNewPayloads"` + + // Stateful-engine fields (only set for SupportedStatefulFixtureFormat). + // SetupEngineNewPayloads run after the shared pre_run payloads to build the + // per-test pre-state; StartBlockHash links the fixture to its pre_run file. + SetupEngineNewPayloads []*EngineNewPayload `json:"setupEngineNewPayloads"` + SnapshotBlockHash string `json:"snapshotBlockHash"` + StartBlockHash string `json:"startBlockHash"` + LastBlockHash string `json:"lastblockhash"` +} + +// StatefulPreRun is a shared pre_run file referenced by stateful fixtures via +// their startBlockHash. Its EngineNewPayloads advance the snapshot datadir to +// the common start block before each fixture's per-test setup runs. The same +// pre_run is reused by every fixture that starts from the same block, which is +// why EEST writes it once under pre_run/.json. +type StatefulPreRun struct { + Network string `json:"network"` + SnapshotBlockHash string `json:"snapshotBlockHash"` + StartBlockHash string `json:"startBlockHash"` + EngineNewPayloads []*EngineNewPayload `json:"engineNewPayloads"` } // FixtureInfo contains metadata about the fixture. @@ -30,7 +57,18 @@ type FixtureInfo struct { // IsSupportedFormat returns true if the fixture has a supported format. func (f *Fixture) IsSupportedFormat() bool { - return f.Info != nil && f.Info.FixtureFormat == SupportedFixtureFormat + if f.Info == nil { + return false + } + + return f.Info.FixtureFormat == SupportedFixtureFormat || + f.Info.FixtureFormat == SupportedStatefulFixtureFormat +} + +// IsStateful reports whether the fixture uses the stateful-engine format, which +// replays against a snapshot datadir and carries no genesis. +func (f *Fixture) IsStateful() bool { + return f.Info != nil && f.Info.FixtureFormat == SupportedStatefulFixtureFormat } // BlockHeader represents an Ethereum block header. @@ -216,3 +254,14 @@ func ParseFixtureFile(data []byte) (map[string]*Fixture, error) { return fixtures, nil } + +// ParsePreRunFile parses a stateful pre_run JSON file. Unlike a fixture file, +// it holds a single object (not a map keyed by test name). +func ParsePreRunFile(data []byte) (*StatefulPreRun, error) { + var preRun StatefulPreRun + if err := json.Unmarshal(data, &preRun); err != nil { + return nil, err + } + + return &preRun, nil +} diff --git a/pkg/executor/eest_source.go b/pkg/executor/eest_source.go index a4b8f8dd3..1b0a0a391 100644 --- a/pkg/executor/eest_source.go +++ b/pkg/executor/eest_source.go @@ -560,6 +560,62 @@ func (s *EESTSource) downloadAndExtractTarball(ctx context.Context, url, targetD return nil } +// loadPreRuns reads the shared pre_run files for stateful-engine fixtures from +// /pre_run/*.json, keyed by start block hash. A missing pre_run +// directory is not an error (the genesis-based format has none) and yields an +// empty map. +func (s *EESTSource) loadPreRuns(searchDir string) (map[string]*eest.StatefulPreRun, error) { + preRunDir := filepath.Join(searchDir, "pre_run") + + entries, err := os.ReadDir(preRunDir) + if err != nil { + if os.IsNotExist(err) { + return map[string]*eest.StatefulPreRun{}, nil + } + + return nil, fmt.Errorf("reading pre_run directory: %w", err) + } + + preRuns := make(map[string]*eest.StatefulPreRun, len(entries)) + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + + path := filepath.Join(preRunDir, entry.Name()) + + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading pre_run file %s: %w", path, err) + } + + preRun, err := eest.ParsePreRunFile(data) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": path, + "error": err, + }).Warn("Failed to parse pre_run file, skipping") + + continue + } + + // Key by the start block hash the fixtures reference. Fall back to the + // file name (which EEST names .json) if the field is + // absent. + key := preRun.StartBlockHash + if key == "" { + key = strings.TrimSuffix(entry.Name(), ".json") + } + + preRuns[key] = preRun + } + + s.log.WithField("count", len(preRuns)).Debug("Loaded stateful pre_run files") + + return preRuns, nil +} + // discoverTests parses fixture files and creates test entries. func (s *EESTSource) discoverTests() (*PreparedSource, error) { // Determine the fixtures search directory. @@ -583,17 +639,26 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { s.log.WithField("path", searchDir).Info("Searching for fixtures") + // Load shared pre_run files (stateful-engine format). Keyed by start block + // hash; empty for the genesis-based format, which has no pre_run dir. + preRuns, err := s.loadPreRuns(searchDir) + if err != nil { + return nil, err + } + // Map fixture keys (testIds) to their TestWithSteps for pre_alloc matching. testsByFixtureKey := make(map[string]*TestWithSteps, 256) // Walk fixture directory for JSON files. - err := filepath.Walk(searchDir, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(searchDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { - if info.Name() == "pre_alloc" { + // pre_alloc holds genesis groups; pre_run holds stateful setup + // payloads loaded separately above. Neither contains fixtures. + if info.Name() == "pre_alloc" || info.Name() == "pre_run" { return filepath.SkipDir } @@ -643,7 +708,24 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { continue } - converted, err := eest.ConvertFixture(name, fixture) + var converted *eest.ConvertedTest + + if fixture.IsStateful() { + preRun := preRuns[fixture.StartBlockHash] + if preRun == nil && fixture.StartBlockHash != "" { + s.log.WithFields(logrus.Fields{ + "file": path, + "fixture": name, + "start_block": fixture.StartBlockHash, + }).Warn("No pre_run file for stateful fixture's start block; " + + "replaying setup payloads only") + } + + converted, err = eest.ConvertStatefulFixture(name, fixture, preRun) + } else { + converted, err = eest.ConvertFixture(name, fixture) + } + if err != nil { s.log.WithFields(logrus.Fields{ "file": path, From d9860751b2402bb9e0e5f03763893359734ff823 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 16 Jun 2026 14:55:21 +0200 Subject: [PATCH 02/83] feat(builder): run build containers as host user; rework fill-stateful knobs Run the state-actor and fill-stateful build containers as the invoking host user so their on-disk output (snapshot datadirs, EEST fixtures) is owned by that user instead of root. This avoids permission-denied failures when a later non-root step reads or cleans that output (e.g. the datadir copy, or wiping output_dir on --force). - docker.ContainerSpec gains a User field; container creation honours it and defaults to root when empty (existing containers unchanged). - state-actor and the eest_payloads filler/fill containers set User to the current uid:gid. The uv/pytest-based fill image is made non-root-friendly by redirecting its writable paths (uv cache, pytest cache, $HOME) to /tmp, skipping the runtime venv re-sync, and marking the root-owned /eest git checkout as a git safe.directory. Also rework the fill-stateful benchmark parametrisation: - gas_benchmark_values is now a typed int list ([10, 30]) instead of a comma-separated string (joined into --gas-benchmark-values). - add fixed_opcode_count (float list, thousands of opcodes), mutually exclusive with gas_benchmark_values; an empty list passes the bare --fixed-opcode-count flag (uses the image's .fixed_opcode_counts.json). --- config.example.yaml | 6 ++- docs/configuration.md | 8 +-- pkg/builder/eest_payloads.go | 56 +++++++++++++++++++-- pkg/builder/eest_payloads_test.go | 27 +++++++++- pkg/builder/state_actor.go | 3 ++ pkg/builder/util.go | 9 ++++ pkg/config/config.go | 82 +++++++++++++++++++++---------- pkg/config/config_test.go | 43 ++++++++++++++-- pkg/docker/docker.go | 11 ++++- 9 files changed, 202 insertions(+), 43 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index c2dfd9ec3..1520023e7 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -283,7 +283,11 @@ runner: # config: # shared per-target defaults; targets override when set # filler_image: ethpandaops/geth:master # fork: Osaka -# gas_benchmark_values: "10,30" # millions of gas to parametrise against +# gas_benchmark_values: [10, 30] # millions of gas to parametrise against +# # fixed_opcode_count parametrises by opcode count (thousands) instead of gas; +# # mutually exclusive with gas_benchmark_values. [] passes the flag bare, which +# # uses the fill image's .fixed_opcode_counts.json default. +# # fixed_opcode_count: [0.5, 1, 2] # datadir_method: copy # copy | overlayfs | fuse-overlayfs | zfs | direct | schelk # targets: # - name: compute-geth diff --git a/docs/configuration.md b/docs/configuration.md index 16b89583a..e3e46b5fc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1575,7 +1575,8 @@ builder: config: # shared per-target defaults; targets override when set filler_image: ethpandaops/geth:master fork: Osaka - gas_benchmark_values: "10,30" # millions of gas to parametrise against + gas_benchmark_values: [10, 30] # millions of gas to parametrise against + # fixed_opcode_count: [0.5, 1, 2] # thousands of opcodes; mutually exclusive with gas_benchmark_values datadir_method: copy # copy | overlayfs | fuse-overlayfs | zfs | direct | schelk targets: - name: compute-geth @@ -1606,7 +1607,8 @@ Every field below is also available per-target; a non-nil/non-empty value on a t |---|---|---|---| | `filler_image` | string | – | Docker image for the filler client (e.g. `ethpandaops/geth:master`). | | `fork` | string | – | Fork to fill against, e.g. `Osaka` (passed to `fill-stateful --fork`). | -| `gas_benchmark_values` | string | – | Comma-separated gas budgets in millions, e.g. `10,30` (`--gas-benchmark-values`). | +| `gas_benchmark_values` | int[] | – | Gas budgets in millions, e.g. `[10, 30]`; joined into `--gas-benchmark-values`. Mutually exclusive with `fixed_opcode_count`. | +| `fixed_opcode_count` | float[] | – | Opcode counts in thousands, e.g. `[0.5, 1, 2]`; joined into `--fixed-opcode-count`. An empty list (`[]`) passes the flag bare, using the fill image's `.fixed_opcode_counts.json` default. Mutually exclusive with `gas_benchmark_values`. | | `datadir_method` | string | `copy` | How the filler's writable copy of `source_dir` is prepared: `copy`, `overlayfs`, `fuse-overlayfs`, `zfs`, `direct`, `schelk`. Use `zfs`/`overlayfs` to avoid a full copy of a large snapshot. | | `max_gas_per_test` | uint64 | – | Overrides the fork's transaction gas-limit cap (`--max-gas-per-test`). | | `rpc_seed_key` | string | – | Pin the seed EOA for reproducible fills (`--rpc-seed-key`); otherwise one is generated and funded via CL withdrawal. | @@ -1627,7 +1629,7 @@ Identity/locator fields are target-only; the rest mirror `config` and are resolv | `filter` | string | – | Optional pytest `-k` expression. | | `address_stubs_file` | string | – | **Absolute** host path to a `--address-stubs` JSON map, required by stub-dependent tests (e.g. bloatnet opcode tests). | | `force` | bool | `false` | Per-target override of `--force`: wipe `output_dir` before filling. | -| `filler_image`, `fork`, `gas_benchmark_values`, `datadir_method`, `max_gas_per_test`, `rpc_seed_key`, `filler_extra_args` | — | from `config` | See the `config` table above. `fork` and `filler_image` are required after resolution. | +| `filler_image`, `fork`, `gas_benchmark_values`, `fixed_opcode_count`, `datadir_method`, `max_gas_per_test`, `rpc_seed_key`, `filler_extra_args` | — | from `config` | See the `config` table above. `fork` and `filler_image` are required after resolution. | ### Replaying generated fixtures diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index ee7df1c98..d0b8bbf35 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -6,6 +6,7 @@ import ( "io" "os" "strconv" + "strings" "time" "github.com/ethpandaops/benchmarkoor/pkg/client" @@ -299,7 +300,11 @@ func (b *EESTPayloadsBuilder) startFiller( Mounts: mounts, NetworkName: eestBuildNetwork, SecurityOpt: []string{"seccomp=unconfined"}, - Labels: b.labels(t), + // Run as the invoking host user so the state the filler writes into the + // copied datadir is owned by that user and can be cleaned up afterwards + // (the copy is made by the host user, not root). + User: currentUserSpec(), + Labels: b.labels(t), } log.WithField("argv", cmd).Info("Starting filler client") @@ -382,7 +387,27 @@ func (b *EESTPayloadsBuilder) runFill( Command: args, Mounts: mounts, NetworkName: eestBuildNetwork, - Labels: b.labels(t), + // Run as the invoking host user so the fixtures written to /out are + // owned by that user rather than root. The default fill image is + // uv/pytest based with a root-owned /eest checkout and venv, so + // redirect every writable path it touches to /tmp (world-writable) and + // skip the runtime venv re-sync (the venv is already built into the + // image) so it runs cleanly as a non-root UID. + User: currentUserSpec(), + Env: map[string]string{ + "HOME": "/tmp", + "UV_CACHE_DIR": "/tmp/uv-cache", + "UV_NO_SYNC": "1", + "PYTEST_ADDOPTS": "-o cache_dir=/tmp/.pytest_cache", + // fill-stateful reads its commit hash from the root-owned /eest git + // checkout; as a non-root user git refuses with "dubious ownership". + // Inject safe.directory=* via git's env-based config so it trusts the + // repo regardless of ownership (* avoids hardcoding the repo path). + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "safe.directory", + "GIT_CONFIG_VALUE_0": "*", + }, + Labels: b.labels(t), } tail := newTailBuffer(64 * 1024) @@ -471,8 +496,31 @@ func buildFillArgs( "--output="+fillOutputPath, ) - if t.GasBenchmarkValues != "" { - args = append(args, "--gas-benchmark-values="+t.GasBenchmarkValues) + if len(t.GasBenchmarkValues) > 0 { + vals := make([]string, len(t.GasBenchmarkValues)) + for i, v := range t.GasBenchmarkValues { + vals[i] = strconv.Itoa(v) + } + + // fill-stateful's --gas-benchmark-values takes one comma-separated + // argument, e.g. "10,30" for 10M and 30M gas. + args = append(args, "--gas-benchmark-values="+strings.Join(vals, ",")) + } + + // --fixed-opcode-count is mutually exclusive with --gas-benchmark-values + // (config validation enforces this). A non-nil but empty list passes the + // flag bare, which makes fill-stateful use its .fixed_opcode_counts.json. + if t.FixedOpcodeCount != nil { + if counts := *t.FixedOpcodeCount; len(counts) > 0 { + vals := make([]string, len(counts)) + for i, v := range counts { + vals[i] = strconv.FormatFloat(v, 'g', -1, 64) + } + + args = append(args, "--fixed-opcode-count="+strings.Join(vals, ",")) + } else { + args = append(args, "--fixed-opcode-count") + } } if t.MaxGasPerTest != nil { diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go index ab9bf3936..0c25a0ac3 100644 --- a/pkg/builder/eest_payloads_test.go +++ b/pkg/builder/eest_payloads_test.go @@ -41,15 +41,38 @@ func TestBuildFillArgs(t *testing.T) { "tests/benchmark/compute", }, wantAbsent: []string{ - "--clean", "--gas-benchmark-values", "--max-gas-per-test", "--rpc-seed-key", "--address-stubs", "-k", + "--clean", "--gas-benchmark-values", "--fixed-opcode-count", "--max-gas-per-test", + "--rpc-seed-key", "--address-stubs", "-k", }, }, + { + name: "fixed opcode count values", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + FixedOpcodeCount: &[]float64{0.5, 1, 2}, + Tests: []string{"tests/benchmark/compute"}, + }, + wantContain: []string{"--fixed-opcode-count=0.5,1,2"}, + wantAbsent: []string{"--gas-benchmark-values"}, + }, + { + name: "fixed opcode count bare (default json)", + target: &config.EESTPayloadTarget{ + FillerClient: "geth", + Fork: "Osaka", + FixedOpcodeCount: &[]float64{}, + Tests: []string{"tests/benchmark/compute"}, + }, + wantContain: []string{"--fixed-opcode-count"}, + wantAbsent: []string{"--fixed-opcode-count=", "--gas-benchmark-values"}, + }, { name: "full", target: &config.EESTPayloadTarget{ FillerClient: "geth", Fork: "Osaka", - GasBenchmarkValues: "10,30", + GasBenchmarkValues: []int{10, 30}, MaxGasPerTest: u64(45000000), RPCSeedKey: "0xdead", AddressStubsFile: "/host/stubs.json", diff --git a/pkg/builder/state_actor.go b/pkg/builder/state_actor.go index a95bb25be..db675917c 100644 --- a/pkg/builder/state_actor.go +++ b/pkg/builder/state_actor.go @@ -150,6 +150,9 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp Image: image, Command: args, Mounts: mounts, + // Run as the invoking host user so the output datadir is owned by that + // user (not root) and is readable when a later step copies it. + User: currentUserSpec(), Labels: map[string]string{ "benchmarkoor.managed-by": "benchmarkoor", "benchmarkoor.builder": StateActorBuilderName, diff --git a/pkg/builder/util.go b/pkg/builder/util.go index 41fbcd59a..993bdb272 100644 --- a/pkg/builder/util.go +++ b/pkg/builder/util.go @@ -43,6 +43,15 @@ func prepareOutputDir(dir string, force bool) error { return nil } +// currentUserSpec returns the invoking process's user as a docker "uid:gid" +// string. Builder containers run as this user so the datadirs and fixtures +// they write are owned by the host user instead of root — avoiding +// permission-denied failures when a later non-root step (e.g. the datadir +// copy) reads that output. +func currentUserSpec() string { + return fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) +} + // randSuffix returns a 6-hex-character random string used to keep // concurrent build container names unique. func randSuffix() (string, error) { diff --git a/pkg/config/config.go b/pkg/config/config.go index 4d610070a..e4170ae5d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -322,13 +322,14 @@ func (e *EESTPayloadsConfig) ResolveFillCommand() []string { // field is also present on EESTPayloadTarget; a non-nil/non-empty value // on the target wins over the corresponding default. See ResolveTarget. type EESTPayloadDefaults struct { - FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` - Fork string `yaml:"fork,omitempty" mapstructure:"fork"` - GasBenchmarkValues string `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` - DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` - MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` - RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` - FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` + FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` + Fork string `yaml:"fork,omitempty" mapstructure:"fork"` + GasBenchmarkValues []int `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` + FixedOpcodeCount *[]float64 `yaml:"fixed_opcode_count,omitempty" mapstructure:"fixed_opcode_count"` + DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` + MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` + RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` + FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` } // EESTPayloadTarget is one fixture-generation run. Identity/locator fields @@ -348,13 +349,14 @@ type EESTPayloadTarget struct { Force bool `yaml:"force,omitempty" mapstructure:"force"` // Hoistable fields (mirror EESTPayloadDefaults). - FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` - Fork string `yaml:"fork,omitempty" mapstructure:"fork"` - GasBenchmarkValues string `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` - DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` - MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` - RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` - FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` + FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` + Fork string `yaml:"fork,omitempty" mapstructure:"fork"` + GasBenchmarkValues []int `yaml:"gas_benchmark_values,omitempty" mapstructure:"gas_benchmark_values"` + FixedOpcodeCount *[]float64 `yaml:"fixed_opcode_count,omitempty" mapstructure:"fixed_opcode_count"` + DataDirMethod string `yaml:"datadir_method,omitempty" mapstructure:"datadir_method"` + MaxGasPerTest *uint64 `yaml:"max_gas_per_test,omitempty" mapstructure:"max_gas_per_test"` + RPCSeedKey string `yaml:"rpc_seed_key,omitempty" mapstructure:"rpc_seed_key"` + FillerExtraArgs []string `yaml:"filler_extra_args,omitempty" mapstructure:"filler_extra_args"` } // ResolveTarget returns a copy of the i-th target with any unset hoistable @@ -378,10 +380,14 @@ func (e *EESTPayloadsConfig) ResolveTarget(i int) EESTPayloadTarget { t.Fork = g.Fork } - if t.GasBenchmarkValues == "" { + if len(t.GasBenchmarkValues) == 0 { t.GasBenchmarkValues = g.GasBenchmarkValues } + if t.FixedOpcodeCount == nil { + t.FixedOpcodeCount = g.FixedOpcodeCount + } + if t.DataDirMethod == "" { t.DataDirMethod = g.DataDirMethod } @@ -1980,6 +1986,17 @@ func (c *Config) validateEESTPayloads() error { if err := validateGasBenchmarkValues(t.GasBenchmarkValues, prefix); err != nil { return err } + + if err := validateFixedOpcodeCount(t.FixedOpcodeCount, prefix); err != nil { + return err + } + + if len(t.GasBenchmarkValues) > 0 && t.FixedOpcodeCount != nil { + return fmt.Errorf( + "%s: gas_benchmark_values and fixed_opcode_count are mutually exclusive "+ + "(fill-stateful rejects both)", prefix, + ) + } } return nil @@ -2025,22 +2042,33 @@ func validateEESTPayloadPaths(t *EESTPayloadTarget, prefix string, seenOutputs m return nil } -// validateGasBenchmarkValues checks a comma-separated list of positive -// integers (millions of gas), e.g. "10,30". Empty is allowed. -func validateGasBenchmarkValues(values, prefix string) error { - if values == "" { - return nil +// validateGasBenchmarkValues checks a list of positive integers (millions of +// gas), e.g. [10, 30]. An empty list is allowed. +func validateGasBenchmarkValues(values []int, prefix string) error { + for _, v := range values { + if v < 1 { + return fmt.Errorf( + "%s.gas_benchmark_values: %d is not a positive integer (millions of gas)", prefix, v, + ) + } } - for _, v := range strings.Split(values, ",") { - v = strings.TrimSpace(v) - if v == "" { - return fmt.Errorf("%s.gas_benchmark_values: empty value in %q", prefix, values) - } + return nil +} + +// validateFixedOpcodeCount checks a list of positive numbers (thousands of +// opcodes), e.g. [0.5, 1, 2]. A nil pointer (unset) is allowed, as is a +// non-nil empty list (passes the bare --fixed-opcode-count flag, which uses +// the fill image's .fixed_opcode_counts.json default). +func validateFixedOpcodeCount(values *[]float64, prefix string) error { + if values == nil { + return nil + } - if _, err := strconv.ParseUint(v, 10, 64); err != nil { + for _, v := range *values { + if v <= 0 { return fmt.Errorf( - "%s.gas_benchmark_values: %q is not a comma-separated list of integers", prefix, values, + "%s.fixed_opcode_count: %v is not a positive number (thousands of opcodes)", prefix, v, ) } } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 7f4256bce..5ec0b2599 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3828,13 +3828,46 @@ func TestValidateEESTPayloads(t *testing.T) { name: "invalid gas_benchmark_values", ep: func() *EESTPayloadsConfig { tgt := base(dirA) - tgt.GasBenchmarkValues = "10,abc" + tgt.GasBenchmarkValues = []int{10, 0} return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} }(), wantErr: true, errSubstr: "gas_benchmark_values", }, + { + name: "invalid fixed_opcode_count", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FixedOpcodeCount = &[]float64{0.5, -1} + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "fixed_opcode_count", + }, + { + name: "gas_benchmark_values and fixed_opcode_count are mutually exclusive", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.GasBenchmarkValues = []int{10} + tgt.FixedOpcodeCount = &[]float64{0.5} + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: true, + errSubstr: "mutually exclusive", + }, + { + name: "fixed_opcode_count bare (empty list) is valid", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FixedOpcodeCount = &[]float64{} + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + wantErr: false, + }, } for _, tt := range tests { @@ -3857,7 +3890,7 @@ func TestEESTPayloadsResolveTarget(t *testing.T) { Config: &EESTPayloadDefaults{ FillerImage: "ethpandaops/geth:master", Fork: "Osaka", - GasBenchmarkValues: "10,30", + GasBenchmarkValues: []int{10, 30}, DataDirMethod: "zfs", MaxGasPerTest: u64Cfg(45000000), RPCSeedKey: "0xseed", @@ -3868,14 +3901,14 @@ func TestEESTPayloadsResolveTarget(t *testing.T) { {Name: "inherit", FillerClient: "geth", SourceDir: "/s", OutputDir: "/o"}, // Overrides fork and gas values. {Name: "override", FillerClient: "geth", SourceDir: "/s2", OutputDir: "/o2", - Fork: "Prague", GasBenchmarkValues: "60"}, + Fork: "Prague", GasBenchmarkValues: []int{60}}, }, } inherit := ep.ResolveTarget(0) assert.Equal(t, "ethpandaops/geth:master", inherit.FillerImage) assert.Equal(t, "Osaka", inherit.Fork) - assert.Equal(t, "10,30", inherit.GasBenchmarkValues) + assert.Equal(t, []int{10, 30}, inherit.GasBenchmarkValues) assert.Equal(t, "zfs", inherit.DataDirMethod) require.NotNil(t, inherit.MaxGasPerTest) assert.Equal(t, uint64(45000000), *inherit.MaxGasPerTest) @@ -3883,7 +3916,7 @@ func TestEESTPayloadsResolveTarget(t *testing.T) { override := ep.ResolveTarget(1) assert.Equal(t, "Prague", override.Fork, "per-target fork wins") - assert.Equal(t, "60", override.GasBenchmarkValues, "per-target gas values win") + assert.Equal(t, []int{60}, override.GasBenchmarkValues, "per-target gas values win") assert.Equal(t, "ethpandaops/geth:master", override.FillerImage, "still inherits unset fields") } diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index ed65ae23c..066d63064 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -106,6 +106,7 @@ type ContainerSpec struct { ResourceLimits *ResourceLimits CapAdd []string // Additional Linux capabilities (e.g., "SYS_PTRACE" for CRIU). SecurityOpt []string // Security options (e.g., "seccomp=unconfined"). + User string // Container user (e.g., "1000:1000"); defaults to "root" when empty. } // Mount defines a volume mount. @@ -247,9 +248,17 @@ func (m *manager) CreateContainer(ctx context.Context, spec *ContainerSpec) (str }) } + // Containers run as root unless the spec requests a specific user (e.g. the + // state-actor builder runs as the invoking host user so its output datadir + // is owned by that user rather than root). + user := spec.User + if user == "" { + user = "root" + } + containerCfg := &container.Config{ Image: spec.Image, - User: "root", + User: user, Env: env, Labels: spec.Labels, Entrypoint: spec.Entrypoint, From c47fd2efad2934926d756e3ed017e176eada7b86 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 16 Jun 2026 14:55:29 +0200 Subject: [PATCH 03/83] fix(ui): render local EEST fixture sources without crashing Local-directory / tarball EEST sources (e.g. stateful fixtures built locally) have no github_repo, so getGitHubUrl(undefined) threw on the suite-detail page. Guard the helper, make the GitHub repo/link rendering conditional, surface the local_* fields, and badge these sources as "EEST Local". --- ui/src/api/types.ts | 6 +- .../components/suite-detail/SuiteSource.tsx | 60 ++++++++++++++----- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/ui/src/api/types.ts b/ui/src/api/types.ts index 154ea67cf..dd9b4d781 100644 --- a/ui/src/api/types.ts +++ b/ui/src/api/types.ts @@ -547,7 +547,7 @@ export interface SourceInfo { } } eest?: { - github_repo: string + github_repo?: string github_release?: string fixtures_url?: string genesis_url?: string @@ -556,6 +556,10 @@ export interface SourceInfo { genesis_artifact_name?: string fixtures_artifact_run_id?: string genesis_artifact_run_id?: string + local_fixtures_dir?: string + local_genesis_dir?: string + local_fixtures_tarball?: string + local_genesis_tarball?: string } } diff --git a/ui/src/components/suite-detail/SuiteSource.tsx b/ui/src/components/suite-detail/SuiteSource.tsx index 6ff2314eb..4e3aa3282 100644 --- a/ui/src/components/suite-detail/SuiteSource.tsx +++ b/ui/src/components/suite-detail/SuiteSource.tsx @@ -96,7 +96,11 @@ function GitHubIcon({ className }: { className?: string }) { ) } -function getGitHubUrl(repo: string, sha?: string, directory?: string): string { +function getGitHubUrl(repo?: string, sha?: string, directory?: string): string { + if (!repo) { + return '' + } + let baseUrl = repo if (repo.startsWith('git@github.com:')) { baseUrl = repo.replace('git@github.com:', 'https://github.com/').replace(/\.git$/, '') @@ -130,7 +134,14 @@ function SourceTypeBadge({ source }: { source: SourceInfo }) { if (source.eest) { const hasArtifacts = source.eest.fixtures_artifact_name || source.eest.genesis_artifact_name - return {hasArtifacts ? 'EEST Artifact' : 'EEST Release'} + const isLocal = + !source.eest.github_repo && + (source.eest.local_fixtures_dir || + source.eest.local_genesis_dir || + source.eest.local_fixtures_tarball || + source.eest.local_genesis_tarball) + const label = isLocal ? 'EEST Local' : hasArtifacts ? 'EEST Artifact' : 'EEST Release' + return {label} } return null @@ -290,15 +301,32 @@ export function SuiteSource({ title, source }: SuiteSourceProps) { : 'View on GitHub' const hasArtifacts = eest.fixtures_artifact_name || eest.genesis_artifact_name || eest.fixtures_artifact_run_id || eest.genesis_artifact_run_id + // Local-directory / tarball sources (e.g. stateful fixtures built locally) + // have no GitHub repo, release, or artifact — only show the GitHub link when + // there is actually a repo to point at. + const localFields: Array<{ label: string; value: string }> = [ + { label: 'Local Fixtures Dir', value: eest.local_fixtures_dir ?? '' }, + { label: 'Local Genesis Dir', value: eest.local_genesis_dir ?? '' }, + { label: 'Local Fixtures Tarball', value: eest.local_fixtures_tarball ?? '' }, + { label: 'Local Genesis Tarball', value: eest.local_genesis_tarball ?? '' }, + ].filter((f) => f.value) return ( {title}} collapsible>
-
-
Repository
-
{eest.github_repo}
-
+ {eest.github_repo && ( +
+
Repository
+
{eest.github_repo}
+
+ )} + {localFields.map((f) => ( +
+
{f.label}
+
{f.value}
+
+ ))} {eest.github_release && (
Release
@@ -381,15 +409,17 @@ export function SuiteSource({ title, source }: SuiteSourceProps) {
)} - - - {githubLinkLabel} - + {eest.github_repo && ( + + + {githubLinkLabel} + + )} ) From 60edcf6e6daa8e45e44aa9ae555f887277ce0720 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 16 Jun 2026 15:41:42 +0200 Subject: [PATCH 04/83] feat(build): match run log format and make fill progress readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make `benchmarkoor build` output consistent with `benchmarkoor run` and easier to follow during the long fixture-generation phase: - Apply the run command's `🔵` log formatter, and stream each container's stdout/stderr directly in run's `🟣 TS LABEL | name | line` client-log style (CLIE for the filler EL client, BULD for the state-actor and fill-stateful build containers) instead of wrapping them through logrus. - Reset ANSI state at the end of every streamed line so an unreset bold/color sequence (e.g. pytest's session header) can't bleed into later lines. - Pass -v to fill-stateful so each test node id and outcome prints as it is built, rather than bare pytest progress dots. - Print a separate build summary per builder (state-actor vs eest-payloads). - Trim the per-line logger to the target name; source_dir/output_dir/fork/ image are logged once instead of suffixed onto every line. --- cmd/benchmarkoor/build.go | 55 ++++++++++++++++++++++--------- pkg/builder/eest_payloads.go | 29 +++++++++++----- pkg/builder/eest_payloads_test.go | 2 +- pkg/builder/state_actor.go | 16 ++++----- pkg/builder/util.go | 48 ++++++++++++++------------- 5 files changed, 94 insertions(+), 56 deletions(-) diff --git a/cmd/benchmarkoor/build.go b/cmd/benchmarkoor/build.go index 1401abadd..3c29a8171 100644 --- a/cmd/benchmarkoor/build.go +++ b/cmd/benchmarkoor/build.go @@ -48,6 +48,10 @@ func init() { } func runBuild(_ *cobra.Command, _ []string) error { + // Match the `benchmarkoor run` log format (🔵 prefix); container output is + // streamed in the same 🟣 client-log style for a consistent look. + log.SetFormatter(&consistentFormatter{prefix: "🔵"}) + if len(cfgFiles) == 0 { return fmt.Errorf("config file is required (use --config)") } @@ -173,6 +177,7 @@ func newContainerManager(runtime string) (docker.ContainerManager, error) { // buildResult captures the outcome of a single target build. type buildResult struct { + builder string name string client string outputDir string @@ -206,6 +211,7 @@ func runBuilders(ctx context.Context, builders []builder.Builder) error { skipped, buildErr := sel.builder.Build(ctx, sel.info.Name, builder.BuildOptions{Force: buildForce}) results = append(results, buildResult{ + builder: sel.builder.Name(), name: sel.info.Name, client: sel.info.Client, outputDir: sel.info.OutputDir, @@ -221,31 +227,50 @@ func runBuilders(ctx context.Context, builders []builder.Builder) error { return summarise(results) } -// summarise logs the per-target outcome and returns an error if any failed. +// summarise logs the per-target outcome grouped under each builder and returns +// an error if any target failed. Builders are emitted in first-seen +// (declaration) order so state_actor appears before eest_payloads. func summarise(results []buildResult) error { var failed []string - log.Info("Build summary:") + var order []string + + byBuilder := make(map[string][]buildResult, 2) for _, r := range results { - var status string + if _, seen := byBuilder[r.builder]; !seen { + order = append(order, r.builder) + } - switch { - case r.err != nil: - status = "ERR " + byBuilder[r.builder] = append(byBuilder[r.builder], r) + if r.err != nil { failed = append(failed, r.name) - case r.skipped: - status = "SKIP" - default: - status = "OK " } + } - log.WithFields(logrus.Fields{ - "target": r.name, - "client": r.client, - "output_dir": r.outputDir, - }).Infof(" %s %s", status, r.name) + for _, b := range order { + log.Infof("Build summary [%s]:", b) + + for _, r := range byBuilder[b] { + var status string + + switch { + case r.err != nil: + status = "ERR " + case r.skipped: + status = "SKIP" + default: + status = "OK " + } + + log.WithFields(logrus.Fields{ + "builder": r.builder, + "target": r.name, + "client": r.client, + "output_dir": r.outputDir, + }).Infof(" %s %s", status, r.name) + } } if len(failed) > 0 { diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index d0b8bbf35..ee6ef7573 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -102,13 +102,10 @@ func (b *EESTPayloadsBuilder) Build(ctx context.Context, name string, opts Build resolved := b.cfg.ResolveTarget(idx) target := &resolved - log := b.log.WithFields(logrus.Fields{ - "target": target.EffectiveName(), - "filler_client": target.FillerClient, - "source_dir": target.SourceDir, - "output_dir": target.OutputDir, - "fork": target.Fork, - }) + // Keep only the target on the per-line logger; source_dir/output_dir/fork + // are reference details logged once below rather than suffixed onto every + // orchestration and streamed-client log line. + log := b.log.WithField("target", target.EffectiveName()) force := opts.Force || target.Force @@ -177,6 +174,14 @@ func (b *EESTPayloadsBuilder) checkInputs(t *config.EESTPayloadTarget) error { // run performs the orchestration: temp JWT, network, datadir copy, filler // boot, fill-stateful, and teardown. func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t *config.EESTPayloadTarget) error { + // Record the build details once; the per-line logger carries only target. + log.WithFields(logrus.Fields{ + "filler_client": t.FillerClient, + "source_dir": t.SourceDir, + "output_dir": t.OutputDir, + "fork": t.Fork, + }).Info("Generating EEST payloads") + spec, err := b.registry.Get(client.ClientType(t.FillerClient)) if err != nil { return fmt.Errorf("resolving filler client %q: %w", t.FillerClient, err) @@ -321,7 +326,9 @@ func (b *EESTPayloadsBuilder) startFiller( } go func() { - w := logWriter(log.WithField("filler", t.FillerClient), logrus.InfoLevel) + // Stream filler-client output in the same "🟣 … CLIE | |" + // format `benchmarkoor run` uses for client logs. + w := containerStream("CLIE", t.FillerClient) if streamErr := b.mgr.StreamLogs(streamCtx, id, w, w); streamErr != nil { log.WithError(streamErr).Debug("Filler log streaming stopped") } @@ -411,7 +418,7 @@ func (b *EESTPayloadsBuilder) runFill( } tail := newTailBuffer(64 * 1024) - out := io.MultiWriter(logWriter(log, logrus.InfoLevel), tail) + out := io.MultiWriter(containerStream("BULD", "fill-stateful"), tail) log.WithField("argv", args).Info("Running fill-stateful") @@ -488,6 +495,10 @@ func buildFillArgs( // wipes it) before we get here, so fill-stateful just mkdirs into it. args := append([]string{}, prefix...) args = append(args, + // -v makes the underlying pytest print each test node id and its + // outcome as it is built, so the fill progress is visible instead of + // bare progress dots. + "-v", fmt.Sprintf("--rpc-endpoint=http://%s:%d", fillerIP, spec.RPCPort()), fmt.Sprintf("--engine-endpoint=http://%s:%d", fillerIP, spec.EnginePort()), "--engine-jwt-secret-file="+fillJWTPath, diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go index 0c25a0ac3..53742ef90 100644 --- a/pkg/builder/eest_payloads_test.go +++ b/pkg/builder/eest_payloads_test.go @@ -31,7 +31,7 @@ func TestBuildFillArgs(t *testing.T) { Tests: []string{"tests/benchmark/compute"}, }, wantContain: []string{ - "uv", "run", "fill-stateful", + "uv", "run", "fill-stateful", "-v", "--rpc-endpoint=http://10.0.0.5:8545", "--engine-endpoint=http://10.0.0.5:8551", "--engine-jwt-secret-file=" + fillJWTPath, diff --git a/pkg/builder/state_actor.go b/pkg/builder/state_actor.go index db675917c..905ea8c01 100644 --- a/pkg/builder/state_actor.go +++ b/pkg/builder/state_actor.go @@ -86,12 +86,10 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp return false, fmt.Errorf("no image configured for client %q", target.Client) } - log := b.log.WithFields(logrus.Fields{ - "target": target.EffectiveName(), - "client": target.Client, - "output_dir": target.OutputDir, - "image": image, - }) + // Keep only the target on the per-line logger; client/output_dir are already + // in build.go's "Building target" header and image is logged on the + // "Running state-actor" line, so they don't repeat on every streamed line. + log := b.log.WithField("target", target.EffectiveName()) // The CLI `--force` flag and per-target `force: true` both bypass the // skip-on-populated check, wipe the existing output_dir, and forward @@ -166,10 +164,10 @@ func (b *StateActorBuilder) Build(ctx context.Context, name string, opts BuildOp // exit yields a useful error without spamming the whole log. tail := newTailBuffer(64 * 1024) - stdout := io.MultiWriter(logWriter(log, logrus.InfoLevel), tail) - stderr := io.MultiWriter(logWriter(log, logrus.InfoLevel), tail) + stdout := io.MultiWriter(containerStream("BULD", "state-actor"), tail) + stderr := io.MultiWriter(containerStream("BULD", "state-actor"), tail) - log.WithField("argv", args).Info("Running state-actor") + log.WithFields(logrus.Fields{"image": image, "argv": args}).Info("Running state-actor") if err := b.mgr.RunInitContainer(ctx, spec, stdout, stderr); err != nil { return false, fmt.Errorf("running state-actor: %w (output tail: %s)", diff --git a/pkg/builder/util.go b/pkg/builder/util.go index 993bdb272..5445f6b99 100644 --- a/pkg/builder/util.go +++ b/pkg/builder/util.go @@ -7,8 +7,9 @@ import ( "fmt" "io" "os" + "time" - "github.com/sirupsen/logrus" + "github.com/ethpandaops/benchmarkoor/pkg/config" ) // isPopulated reports whether dir exists and contains at least one @@ -64,21 +65,30 @@ func randSuffix() (string, error) { return hex.EncodeToString(b[:]), nil } -// logWriter returns an io.Writer that forwards each line written to it to -// the supplied logger at the given level. Used to stream container -// stdout/stderr without writing the bytes directly to the global -// stdout/stderr. -func logWriter(log logrus.FieldLogger, level logrus.Level) io.Writer { - return &lineLogger{log: log, level: level} +// containerStream returns an io.Writer that prefixes each line of streamed +// container output with "🟣 $TS $label | $name | " and writes it directly to +// stdout. This matches the client-log format `benchmarkoor run` uses (see +// pkg/runner clientLogPrefix) so build output looks consistent and carries a +// clear leading tag identifying the source. label is a short tag (e.g. "CLIE" +// for an EL client, "BULD" for a build-tool container). +func containerStream(label, name string) io.Writer { + return &containerStreamWriter{label: label, name: name, w: os.Stdout} } -type lineLogger struct { - log logrus.FieldLogger - level logrus.Level +// ansiReset clears any ANSI color/style left set by a streamed line. Tools like +// pytest emit a bold/color sequence (e.g. the test-session header) without a +// trailing reset, which would otherwise bleed into the next line — including our +// prefix — making everything after it bold. +const ansiReset = "\x1b[0m" + +type containerStreamWriter struct { + label string + name string + w io.Writer buf bytes.Buffer } -func (w *lineLogger) Write(p []byte) (int, error) { +func (w *containerStreamWriter) Write(p []byte) (int, error) { w.buf.Write(p) for { @@ -88,17 +98,11 @@ func (w *lineLogger) Write(p []byte) (int, error) { } line := w.buf.Next(i + 1) - msg := string(bytes.TrimRight(line, "\r\n")) - - switch w.level { - case logrus.ErrorLevel: - w.log.Error(msg) - case logrus.WarnLevel: - w.log.Warn(msg) - case logrus.DebugLevel: - w.log.Debug(msg) - default: - w.log.Info(msg) + ts := time.Now().UTC().Format(config.LogTimestampFormat) + msg := bytes.TrimRight(line, "\r\n") + + if _, err := fmt.Fprintf(w.w, "🟣 %s %s | %s | %s%s\n", ts, w.label, w.name, msg, ansiReset); err != nil { + return len(p), err } } From 922f6a019cc7b6d35d26e6ba685c18717cd11c9e Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 16 Jun 2026 16:37:16 +0200 Subject: [PATCH 05/83] feat(client): boot state-actor snapshots (besu/reth genesis-trust flags) state-actor direct-writes synthetic state (and the genesis state root) into the client DB and emits a chainspec with an empty alloc, so a client booting that snapshot recomputes a different genesis from the chainspec and aborts. geth boots datadir-only (no genesis) and is unaffected; besu and reth need a flag to trust the DB-resident genesis instead: - besu: --genesis-state-hash-cache-enabled=true (else "Supplied genesis block does not match chain data stored"). - reth: --debug.skip-genesis-validation (else "genesis hash in the storage does not match the specified chainspec"); the flag's own help names state-actor as the intended use case. With these, besu replays the stateful fixtures cleanly (with a bootstrap FCU and a stable besu image). reth boots but currently diverges on execution state root (an upstream state-actor-reth snapshot inconsistency). --- pkg/client/besu.go | 8 ++++++++ pkg/client/reth.go | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/pkg/client/besu.go b/pkg/client/besu.go index 4f8dd2b08..366c3242e 100644 --- a/pkg/client/besu.go +++ b/pkg/client/besu.go @@ -23,6 +23,14 @@ func (s *besuSpec) DefaultCommand() []string { // Data directory - should always point to /data "--data-path=/data", "--data-storage-format=BONSAI", + // Trust the genesis state hash stored in the datadir instead of + // recomputing it from the chainspec alloc. Required to boot a + // state-actor snapshot: state-actor writes synthetic state (and the + // genesis state root) directly into RocksDB and emits an empty + // chainspec alloc, so the default recompute path would otherwise fail + // with "Supplied genesis block does not match chain data stored". Has + // no effect on a normal genesis-initialised datadir. + "--genesis-state-hash-cache-enabled=true", // Peering / Syncing / TXPool "--p2p-enabled=false", "--sync-mode=FULL", diff --git a/pkg/client/reth.go b/pkg/client/reth.go index 38b3f7a19..4ce92fd15 100644 --- a/pkg/client/reth.go +++ b/pkg/client/reth.go @@ -38,6 +38,13 @@ func (s *rethSpec) DefaultCommand() []string { "--authrpc.addr=0.0.0.0", "--authrpc.port=8551", "--engine.disable-precompile-cache", + // Trust the genesis stored in the datadir instead of recomputing it from + // the chainspec alloc. Required to boot a state-actor snapshot: state-actor + // direct-writes synthetic state (and the genesis) into reth's DB and emits a + // chainspec whose alloc can't reproduce it, so reth would otherwise abort with + // "genesis hash in the storage does not match the specified chainspec". + // (Besu's equivalent is --genesis-state-hash-cache-enabled.) + "--debug.skip-genesis-validation", // Others "--full", } From 7ed44110a6ea92f985d7ec284583ff2a71460051 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 14:20:46 +0200 Subject: [PATCH 06/83] feat(builder): clone EEST repo at build time into a slim fill image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fill image previously baked the execution-specs checkout (and its uv venv) at docker build time, so changing the EEST version meant rebuilding the image. Instead, clone the repo at benchmarkoor build time and mount it into the fill container, with the version in config. - pkg/gitrepo: extract the clone-into-cache logic from executor.GitSource into a shared package (CloneOrUpdate + HeadSHA); GitSource now delegates to it, so the test-source and EEST clones share one cache implementation. - builder.eest_payloads: clone eest_repo@eest_ref (defaults: execution-specs / forks/amsterdam) into an on-disk cache and bind-mount the checkout at /eest. uv builds the venv into that user-owned checkout on first use (cached across runs); no UV_NO_SYNC needed. - Dockerfile.eest-filler: slimmed to just the uv/python/git toolchain — no repo or venv baked in. - config: add eest_repo / eest_ref with resolvers + docs. --- Dockerfile.eest-filler | 47 ++++------- config.example.yaml | 5 ++ docs/configuration.md | 2 + pkg/builder/eest_payloads.go | 96 +++++++++++++++-------- pkg/config/config.go | 43 ++++++++++- pkg/config/config_test.go | 16 ++++ pkg/executor/source.go | 146 +---------------------------------- pkg/executor/source_test.go | 28 ------- pkg/gitrepo/gitrepo.go | 139 +++++++++++++++++++++++++++++++++ pkg/gitrepo/gitrepo_test.go | 90 +++++++++++++++++++++ 10 files changed, 374 insertions(+), 238 deletions(-) create mode 100644 pkg/gitrepo/gitrepo.go create mode 100644 pkg/gitrepo/gitrepo_test.go diff --git a/Dockerfile.eest-filler b/Dockerfile.eest-filler index d9538cc67..99792c7e5 100644 --- a/Dockerfile.eest-filler +++ b/Dockerfile.eest-filler @@ -1,11 +1,15 @@ # Dockerfile.eest-filler # -# Builds the image referenced by `builder.eest_payloads.fill_image`: it carries -# the EEST `fill-stateful` command (execution-specs PR #2637) plus `uv`. +# Builds the toolchain image referenced by `builder.eest_payloads.fill_image`: +# it carries only `uv` + python + the build toolchain — NOT the execution-specs +# repo. benchmarkoor clones execution-specs at build time (at the configured +# `eest_ref`, caching it on disk) and mounts the checkout into this container at +# /eest, then runs `uv run fill-stateful`. uv builds the venv into the mounted +# checkout on first use (cached across runs). This keeps the EEST version in +# config and changeable without rebuilding the image. # # fill-stateful drives a *live* EL client over testing_buildBlockV1, so — unlike -# the t8n-based `fill` — it needs NO evmone/eels transition-tool binary. The -# image is just `uv` + the execution-specs checkout. +# the t8n-based `fill` — it needs NO evmone/eels transition-tool binary. # # Build: # docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . @@ -15,16 +19,8 @@ # builder: # eest_payloads: # fill_image: ghcr.io/your-org/eest-fill-stateful:latest -# -# benchmarkoor invokes it as: `uv run fill-stateful ` -# (the default builder.eest_payloads.fill_command), with the test paths resolved -# relative to the execution-specs checkout (WORKDIR below). +# eest_ref: forks/amsterdam # optional; defaults to forks/amsterdam -# --- Source selection ------------------------------------------------------- -# fill-stateful is merged upstream on the forks/amsterdam branch. Override the -# ref to pin a specific tag/commit, e.g. --build-arg EEST_REF=. -ARG EEST_REPO=https://github.com/ethereum/execution-specs.git -ARG EEST_REF=forks/amsterdam ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim-bookworm @@ -32,8 +28,9 @@ FROM python:${PYTHON_VERSION}-slim-bookworm # uv: copy the static binary from the official image (recommended pattern). COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ -# Build deps: git to fetch the source; build-essential/headers for any EEST -# dependencies without prebuilt wheels. +# Runtime deps: git (fill-stateful reads its commit hash from the checkout, and +# uv resolves git deps); build-essential/headers for any EEST dependency without +# a prebuilt wheel; ca-certificates for HTTPS to PyPI. RUN apt-get update \ && apt-get install -y --no-install-recommends \ git \ @@ -41,23 +38,11 @@ RUN apt-get update \ build-essential \ && rm -rf /var/lib/apt/lists/* -ARG EEST_REPO -ARG EEST_REF - -WORKDIR /eest - -# Clone and check out the requested ref (branch, tag, or commit SHA). `uv run -# fill-stateful` and the relative test paths (e.g. tests/benchmark/stateful) -# both resolve against this directory. -RUN git clone "${EEST_REPO}" . \ - && git checkout "${EEST_REF}" - -# Pre-create the virtualenv at build time so runs don't sync on every invocation. ENV UV_NO_PROGRESS=1 -RUN uv sync -# Record the exact source revision for traceability. -RUN git rev-parse HEAD > /eest/.fill-stateful-rev +# The execution-specs checkout is bind-mounted here by benchmarkoor at run time; +# the relative test paths (e.g. tests/benchmark/stateful) resolve against it. +WORKDIR /eest # Standalone default; benchmarkoor overrides this with the full fill-stateful argv. -CMD ["uv", "run", "fill-stateful", "--help"] +CMD ["uv", "--version"] diff --git a/config.example.yaml b/config.example.yaml index 1520023e7..0929070fa 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -280,6 +280,11 @@ runner: # # container_runtime: docker # defaults to runner.container_runtime, then docker # # jwt: # Engine API secret shared with the filler (default: built-in) # # fill_command: [uv, run, fill-stateful] # argv prefix inside fill_image (default) +# # EEST checkout cloned at build time and mounted into the fill container at +# # /eest (fill_image is just the uv/python toolchain). Cached on disk; re-cloned +# # only when the ref changes. Both default as shown. +# # eest_repo: https://github.com/ethereum/execution-specs.git +# # eest_ref: forks/amsterdam # branch, tag, or commit to check out # config: # shared per-target defaults; targets override when set # filler_image: ethpandaops/geth:master # fork: Osaka diff --git a/docs/configuration.md b/docs/configuration.md index e3e46b5fc..263ee76a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1596,6 +1596,8 @@ builder: | `container_runtime` | string | runner's runtime, then `docker` | Container runtime for the filler + fill containers. | | `jwt` | string | benchmarkoor's `DefaultJWT` | Engine API JWT secret; shared between the filler client and `fill-stateful`. | | `fill_command` | []string | `[uv, run, fill-stateful]` | argv prefix invoked inside `fill_image` before the `fill-stateful` flags. Override if your image exposes the command differently. | +| `eest_repo` | string | `https://github.com/ethereum/execution-specs.git` | execution-specs repo cloned for filling. | +| `eest_ref` | string | `forks/amsterdam` | Branch, tag, or commit of `eest_repo`. benchmarkoor always clones the repo at this ref into an on-disk cache at build time and mounts the checkout into the fill container at `/eest` (the `fill_image` carries only the uv/python toolchain, not the repo), so the EEST version is config-driven and changeable without rebuilding the image. The clone is cached and re-fetched only when the ref changes; `uv` builds the venv into the mounted checkout on first use (cached across runs). | | `config` | object | – | Shared defaults for the per-target parameters. See below. | | `targets` | []object | – | Required when invoking `benchmarkoor build`. See below. | diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index ee6ef7573..4bad1af9a 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strconv" "strings" "time" @@ -13,6 +14,7 @@ import ( "github.com/ethpandaops/benchmarkoor/pkg/config" "github.com/ethpandaops/benchmarkoor/pkg/datadir" "github.com/ethpandaops/benchmarkoor/pkg/docker" + "github.com/ethpandaops/benchmarkoor/pkg/gitrepo" "github.com/sirupsen/logrus" ) @@ -36,6 +38,9 @@ const ( fillJWTPath = "/jwt/jwtsecret" fillOutputPath = "/out" fillStubsPath = "/stubs.json" + // fillRepoPath is the fill image's WORKDIR (the execution-specs checkout); + // a config-selected EEST repo clone is mounted here when configured. + fillRepoPath = "/eest" // minerGasLimit is the huge gas limit the filler geth is started with so // benchmark blocks of any size can be built (mirrors the fill-stateful docs). @@ -46,13 +51,18 @@ const ( // it boots a filler EL client on a writable copy of a pre-populated snapshot // datadir, runs fill-stateful against the live client, then tears it down. type EESTPayloadsBuilder struct { - log logrus.FieldLogger - cfg *config.EESTPayloadsConfig - runtime string - mgr docker.ContainerManager - registry client.Registry + log logrus.FieldLogger + cfg *config.EESTPayloadsConfig + runtime string + mgr docker.ContainerManager + registry client.Registry + repoCache string } +// eestRepoCacheDir is where EEST repo clones are cached between builds (so a +// recurring build of the same ref doesn't re-clone). Persists across runs. +var eestRepoCacheDir = filepath.Join(os.TempDir(), "benchmarkoor-eest-repos") + // NewEESTPayloadsBuilder constructs a builder bound to a specific container // manager. The caller is expected to have Start()'d the manager and to // Stop() it after the last Build() call. @@ -63,11 +73,12 @@ func NewEESTPayloadsBuilder( mgr docker.ContainerManager, ) *EESTPayloadsBuilder { return &EESTPayloadsBuilder{ - log: log.WithField("component", "builder.eest_payloads"), - cfg: cfg, - runtime: runtime, - mgr: mgr, - registry: client.NewRegistry(), + log: log.WithField("component", "builder.eest_payloads"), + cfg: cfg, + runtime: runtime, + mgr: mgr, + registry: client.NewRegistry(), + repoCache: eestRepoCacheDir, } } @@ -182,6 +193,22 @@ func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t "fork": t.Fork, }).Info("Generating EEST payloads") + // Clone the EEST repo at the configured ref into the cache and mount it into + // the fill container at /eest. The fill image carries only the uv/python + // toolchain (no repo), so this is always done; the EEST version is + // config-driven (eest_repo / eest_ref). + repo, ref := b.cfg.ResolveEESTRepo(), b.cfg.ResolveEESTRef() + + eestRepoPath, err := gitrepo.CloneOrUpdate(ctx, log, repo, ref, b.repoCache) + if err != nil { + return fmt.Errorf("cloning EEST repo %s@%s: %w", repo, ref, err) + } + + sha, _ := gitrepo.HeadSHA(ctx, eestRepoPath) + log.WithFields(logrus.Fields{ + "repo": repo, "ref": ref, "commit": sha, "path": eestRepoPath, + }).Info("Using cloned EEST repo for fill") + spec, err := b.registry.Get(client.ClientType(t.FillerClient)) if err != nil { return fmt.Errorf("resolving filler client %q: %w", t.FillerClient, err) @@ -251,7 +278,7 @@ func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t "snapshot_block": snapshotHash, }).Info("Filler client ready; running fill-stateful") - return b.runFill(ctx, log, t, fillerIP, spec, jwtPath, snapshotHash) + return b.runFill(ctx, log, t, fillerIP, spec, jwtPath, snapshotHash, eestRepoPath) } // startFiller boots the filler EL client and returns its container ID and IP. @@ -364,7 +391,7 @@ func (b *EESTPayloadsBuilder) runFill( t *config.EESTPayloadTarget, fillerIP string, spec client.Spec, - jwtPath, snapshotHash string, + jwtPath, snapshotHash, eestRepoPath string, ) error { if err := b.mgr.PullImage(ctx, b.cfg.FillImage, b.cfg.PullPolicy); err != nil { return fmt.Errorf("pulling fill image %q: %w", b.cfg.FillImage, err) @@ -383,6 +410,27 @@ func (b *EESTPayloadsBuilder) runFill( }) } + // Run as the invoking host user so fixtures written to /out are owned by + // that user. The uv/pytest fill image keeps writable state under /tmp. + env := map[string]string{ + "HOME": "/tmp", + "UV_CACHE_DIR": "/tmp/uv-cache", + "PYTEST_ADDOPTS": "-o cache_dir=/tmp/.pytest_cache", + // fill-stateful reads its commit hash from the /eest git checkout; as a + // non-root user git can refuse with "dubious ownership". Inject + // safe.directory=* via git's env-based config so it trusts the repo. + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "safe.directory", + "GIT_CONFIG_VALUE_0": "*", + } + + // Mount the host-cloned, user-owned EEST checkout at /eest. uv builds the + // venv into this writable dir on first use (cached across runs), so we do + // NOT skip the sync — the toolchain image carries no prebuilt venv. + mounts = append(mounts, docker.Mount{ + Source: eestRepoPath, Target: fillRepoPath, Type: "bind", + }) + suffix, err := randSuffix() if err != nil { return fmt.Errorf("generating container name suffix: %w", err) @@ -394,27 +442,9 @@ func (b *EESTPayloadsBuilder) runFill( Command: args, Mounts: mounts, NetworkName: eestBuildNetwork, - // Run as the invoking host user so the fixtures written to /out are - // owned by that user rather than root. The default fill image is - // uv/pytest based with a root-owned /eest checkout and venv, so - // redirect every writable path it touches to /tmp (world-writable) and - // skip the runtime venv re-sync (the venv is already built into the - // image) so it runs cleanly as a non-root UID. - User: currentUserSpec(), - Env: map[string]string{ - "HOME": "/tmp", - "UV_CACHE_DIR": "/tmp/uv-cache", - "UV_NO_SYNC": "1", - "PYTEST_ADDOPTS": "-o cache_dir=/tmp/.pytest_cache", - // fill-stateful reads its commit hash from the root-owned /eest git - // checkout; as a non-root user git refuses with "dubious ownership". - // Inject safe.directory=* via git's env-based config so it trusts the - // repo regardless of ownership (* avoids hardcoding the repo path). - "GIT_CONFIG_COUNT": "1", - "GIT_CONFIG_KEY_0": "safe.directory", - "GIT_CONFIG_VALUE_0": "*", - }, - Labels: b.labels(t), + User: currentUserSpec(), + Env: env, + Labels: b.labels(t), } tail := newTailBuffer(64 * 1024) diff --git a/pkg/config/config.go b/pkg/config/config.go index e4170ae5d..81a28c357 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -298,9 +298,46 @@ type EESTPayloadsConfig struct { JWT string `yaml:"jwt,omitempty" mapstructure:"jwt"` // FillCommand is the argv prefix invoked inside FillImage before the // fill-stateful flags. Defaults to ["uv", "run", "fill-stateful"]. - FillCommand []string `yaml:"fill_command,omitempty" mapstructure:"fill_command"` - Config *EESTPayloadDefaults `yaml:"config,omitempty" mapstructure:"config"` - Targets []EESTPayloadTarget `yaml:"targets,omitempty" mapstructure:"targets"` + FillCommand []string `yaml:"fill_command,omitempty" mapstructure:"fill_command"` + // EESTRepo / EESTRef select the execution-specs checkout used for filling. + // benchmarkoor always clones the repo at this ref into an on-disk cache at + // build time and mounts it into the fill container at /eest (the fill image + // carries only the uv/python toolchain, not the repo). This lets the EEST + // version live in config and change without rebuilding the image. Both + // default when unset: EESTRepo to the execution-specs URL, EESTRef to + // DefaultEESTRef. + EESTRepo string `yaml:"eest_repo,omitempty" mapstructure:"eest_repo"` + EESTRef string `yaml:"eest_ref,omitempty" mapstructure:"eest_ref"` + Config *EESTPayloadDefaults `yaml:"config,omitempty" mapstructure:"config"` + Targets []EESTPayloadTarget `yaml:"targets,omitempty" mapstructure:"targets"` +} + +const ( + // DefaultEESTRepo is the execution-specs repository cloned for fill-stateful + // when builder.eest_payloads.eest_repo is unset. + DefaultEESTRepo = "https://github.com/ethereum/execution-specs.git" + // DefaultEESTRef is the execution-specs ref cloned for fill-stateful when + // builder.eest_payloads.eest_ref is unset (where fill-stateful currently lives). + DefaultEESTRef = "forks/amsterdam" +) + +// ResolveEESTRepo returns the configured EEST repo URL, defaulting to +// DefaultEESTRepo. +func (e *EESTPayloadsConfig) ResolveEESTRepo() string { + if e.EESTRepo != "" { + return e.EESTRepo + } + + return DefaultEESTRepo +} + +// ResolveEESTRef returns the configured EEST ref, defaulting to DefaultEESTRef. +func (e *EESTPayloadsConfig) ResolveEESTRef() string { + if e.EESTRef != "" { + return e.EESTRef + } + + return DefaultEESTRef } // DefaultFillCommand is the argv prefix used to invoke fill-stateful inside diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 5ec0b2599..ae7f7a422 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3712,6 +3712,22 @@ func TestValidateEESTPayloads(t *testing.T) { wantErr: true, errSubstr: "fill_image", }, + { + name: "eest_repo without eest_ref is valid (ref defaults)", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + EESTRepo: "https://github.com/ethereum/execution-specs.git", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, + { + name: "eest_ref alone is valid (repo defaults)", + ep: &EESTPayloadsConfig{ + FillImage: "fill:latest", + EESTRef: "v1.2.3", + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, { name: "invalid container_runtime", ep: &EESTPayloadsConfig{ diff --git a/pkg/executor/source.go b/pkg/executor/source.go index 781ebfbea..d000b3893 100644 --- a/pkg/executor/source.go +++ b/pkg/executor/source.go @@ -6,13 +6,13 @@ import ( "encoding/hex" "fmt" "os" - "os/exec" "path/filepath" "sort" "strings" "github.com/ethpandaops/benchmarkoor/pkg/config" "github.com/ethpandaops/benchmarkoor/pkg/eest" + "github.com/ethpandaops/benchmarkoor/pkg/gitrepo" "github.com/sirupsen/logrus" ) @@ -199,77 +199,9 @@ func (s *GitSource) Prepare(ctx context.Context) (*PreparedSource, error) { return s.discoverTests() } -// prepareRepo clones or updates the git repository. +// prepareRepo clones or updates the git repository into the on-disk cache. func (s *GitSource) prepareRepo(ctx context.Context) (string, error) { - repoHash := hashRepoURL(s.cfg.Repo) - localPath := filepath.Join(s.cacheDir, repoHash) - - log := s.log.WithFields(logrus.Fields{ - "repo": s.cfg.Repo, - "version": s.cfg.Version, - "path": localPath, - }) - - if _, err := os.Stat(localPath); os.IsNotExist(err) { - log.Info("Cloning repository") - - if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil { - return "", fmt.Errorf("creating cache directory: %w", err) - } - - if looksLikeCommitHash(s.cfg.Version) { - // Commit hashes can't be used with --branch, so we init + fetch instead. - if err := s.cloneByCommitHash(ctx, localPath); err != nil { - return "", err - } - } else { - // Shallow clone with specific branch/tag. - cmd := exec.CommandContext(ctx, "git", "clone", - "--depth=1", - "--branch", s.cfg.Version, - "--single-branch", - s.cfg.Repo, localPath) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("cloning repository: %w", err) - } - } - } else { - // For commit hashes, skip fetch if HEAD already matches. - if looksLikeCommitHash(s.cfg.Version) { - headHash, err := s.getHeadHash(ctx, localPath) - if err == nil && strings.HasPrefix(headHash, s.cfg.Version) { - log.Info("Cached repository already at requested version") - - return localPath, nil - } - } - - log.Info("Updating cached repository") - - // Fetch the specific version. - cmd := exec.CommandContext(ctx, "git", "-C", localPath, "fetch", - "--depth=1", "origin", s.cfg.Version) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("fetching version: %w", err) - } - - // Checkout FETCH_HEAD. - cmd = exec.CommandContext(ctx, "git", "-C", localPath, "checkout", "FETCH_HEAD") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("checking out version: %w", err) - } - } - - return localPath, nil + return gitrepo.CloneOrUpdate(ctx, s.log, s.cfg.Repo, s.cfg.Version, s.cacheDir) } // discoverTests discovers all tests from the git source. @@ -307,78 +239,6 @@ func (s *GitSource) GetSourceInfo() (*SuiteSource, error) { return &SuiteSource{Git: git}, nil } -// cloneByCommitHash initializes a repo and fetches a specific commit hash. -func (s *GitSource) cloneByCommitHash(ctx context.Context, localPath string) error { - // git init - cmd := exec.CommandContext(ctx, "git", "init", localPath) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("initializing repository: %w", err) - } - - // git remote add origin - cmd = exec.CommandContext(ctx, "git", "-C", localPath, - "remote", "add", "origin", s.cfg.Repo) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("adding remote: %w", err) - } - - // git fetch --depth=1 origin - cmd = exec.CommandContext(ctx, "git", "-C", localPath, - "fetch", "--depth=1", "origin", s.cfg.Version) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("fetching commit %s: %w", s.cfg.Version, err) - } - - // git checkout FETCH_HEAD - cmd = exec.CommandContext(ctx, "git", "-C", localPath, - "checkout", "FETCH_HEAD") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("checking out commit %s: %w", s.cfg.Version, err) - } - - return nil -} - -// getHeadHash returns the current HEAD commit hash for the repository at repoPath. -func (s *GitSource) getHeadHash(ctx context.Context, repoPath string) (string, error) { - cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", "HEAD") - - out, err := cmd.Output() - if err != nil { - return "", err - } - - return strings.TrimSpace(string(out)), nil -} - -// looksLikeCommitHash returns true if s looks like a git commit hash -// (7-40 lowercase/uppercase hex characters). -func looksLikeCommitHash(s string) bool { - if len(s) < 7 || len(s) > 40 { - return false - } - - for _, c := range s { - if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { - return false - } - } - - return true -} - // hashRepoURL creates a hash of the repository URL for caching. func hashRepoURL(url string) string { hash := sha256.Sum256([]byte(url)) diff --git a/pkg/executor/source_test.go b/pkg/executor/source_test.go index 30ab54a83..783b46610 100644 --- a/pkg/executor/source_test.go +++ b/pkg/executor/source_test.go @@ -59,31 +59,3 @@ func TestDiscoverTestsFromConfig_PreRunStepsNotFiltered(t *testing.T) { assert.Len(t, result.Tests, 1, "only bn128 test should match filter") assert.Contains(t, result.Tests[0].Name, "bn128") } - -func TestLooksLikeCommitHash(t *testing.T) { - tests := []struct { - name string - input string - expected bool - }{ - {name: "full sha1", input: "e5011aa5f75d7a1722481f25408347fadfb7fd3c", expected: true}, - {name: "short hash 7 chars", input: "e5011aa", expected: true}, - {name: "short hash 8 chars", input: "e5011aa5", expected: true}, - {name: "uppercase hex", input: "E5011AA5F75D7A17", expected: true}, - {name: "mixed case hex", input: "e5011AA5f75d", expected: true}, - {name: "branch name", input: "main", expected: false}, - {name: "branch with slash", input: "feature/foo", expected: false}, - {name: "tag semver", input: "v1.0.0", expected: false}, - {name: "too short 6 chars", input: "e5011a", expected: false}, - {name: "too long 41 chars", input: "e5011aa5f75d7a1722481f25408347fadfb7fd3c0", expected: false}, - {name: "empty string", input: "", expected: false}, - {name: "hex with non-hex char", input: "e5011gg", expected: false}, - {name: "7 char all digits", input: "1234567", expected: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, looksLikeCommitHash(tt.input)) - }) - } -} diff --git a/pkg/gitrepo/gitrepo.go b/pkg/gitrepo/gitrepo.go new file mode 100644 index 000000000..c00278ca0 --- /dev/null +++ b/pkg/gitrepo/gitrepo.go @@ -0,0 +1,139 @@ +// Package gitrepo clones git repositories into an on-disk cache and keeps the +// cached checkout at a requested branch, tag, or commit. It is the shared +// implementation behind the test-source git clones and the EEST fill-repo +// clone, so recurring clones of the same repo are avoided. +package gitrepo + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" +) + +// CloneOrUpdate ensures repo is cloned under cacheDir and checked out at +// version (a branch, tag, or commit hash), reusing an existing cached clone +// when present. It returns the local path of the checkout. The cache key is +// derived from the repo URL, so different versions of the same repo share one +// directory (the checkout is moved to the requested version). +func CloneOrUpdate(ctx context.Context, log logrus.FieldLogger, repo, version, cacheDir string) (string, error) { + localPath := filepath.Join(cacheDir, hashRepoURL(repo)) + + log = log.WithFields(logrus.Fields{ + "repo": repo, + "version": version, + "path": localPath, + }) + + if _, err := os.Stat(localPath); os.IsNotExist(err) { + log.Info("Cloning repository") + + if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { + return "", fmt.Errorf("creating cache directory: %w", err) + } + + if looksLikeCommitHash(version) { + // Commit hashes can't be used with --branch, so init + fetch. + if err := cloneByCommitHash(ctx, repo, version, localPath); err != nil { + return "", err + } + } else if err := gitRun(ctx, "git", "clone", + "--depth=1", "--branch", version, "--single-branch", repo, localPath); err != nil { + return "", fmt.Errorf("cloning repository: %w", err) + } + + return localPath, nil + } + + // Cached clone exists. For commit hashes, skip the fetch when HEAD already + // matches the requested commit. + if looksLikeCommitHash(version) { + if sha, err := HeadSHA(ctx, localPath); err == nil && strings.HasPrefix(sha, version) { + log.Info("Cached repository already at requested version") + + return localPath, nil + } + } + + log.Info("Updating cached repository") + + if err := gitRun(ctx, "git", "-C", localPath, "fetch", "--depth=1", "origin", version); err != nil { + return "", fmt.Errorf("fetching version: %w", err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "checkout", "FETCH_HEAD"); err != nil { + return "", fmt.Errorf("checking out version: %w", err) + } + + return localPath, nil +} + +// HeadSHA returns the current HEAD commit hash for the repository at repoPath. +func HeadSHA(ctx context.Context, repoPath string) (string, error) { + out, err := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", "HEAD").Output() + if err != nil { + return "", fmt.Errorf("getting commit SHA: %w", err) + } + + return strings.TrimSpace(string(out)), nil +} + +// cloneByCommitHash initializes a repo and fetches a specific commit hash, then +// checks it out (a commit hash cannot be passed to `git clone --branch`). +func cloneByCommitHash(ctx context.Context, repo, version, localPath string) error { + if err := gitRun(ctx, "git", "init", localPath); err != nil { + return fmt.Errorf("initializing repository: %w", err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "remote", "add", "origin", repo); err != nil { + return fmt.Errorf("adding remote: %w", err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "fetch", "--depth=1", "origin", version); err != nil { + return fmt.Errorf("fetching commit %s: %w", version, err) + } + + if err := gitRun(ctx, "git", "-C", localPath, "checkout", "FETCH_HEAD"); err != nil { + return fmt.Errorf("checking out commit %s: %w", version, err) + } + + return nil +} + +// gitRun runs a git command, forwarding its output to the process stdio. +func gitRun(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} + +// looksLikeCommitHash returns true if s looks like a git commit hash +// (7-40 hex characters). +func looksLikeCommitHash(s string) bool { + if len(s) < 7 || len(s) > 40 { + return false + } + + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { + return false + } + } + + return true +} + +// hashRepoURL creates a short stable hash of the repository URL for caching. +func hashRepoURL(url string) string { + hash := sha256.Sum256([]byte(url)) + + return hex.EncodeToString(hash[:8]) +} diff --git a/pkg/gitrepo/gitrepo_test.go b/pkg/gitrepo/gitrepo_test.go new file mode 100644 index 000000000..22f223360 --- /dev/null +++ b/pkg/gitrepo/gitrepo_test.go @@ -0,0 +1,90 @@ +package gitrepo + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLooksLikeCommitHash(t *testing.T) { + tests := []struct { + name string + input string + expected bool + }{ + {name: "full sha1", input: "e5011aa5f75d7a1722481f25408347fadfb7fd3c", expected: true}, + {name: "short hash 7 chars", input: "e5011aa", expected: true}, + {name: "uppercase hex", input: "E5011AA5F75D7A17", expected: true}, + {name: "branch name", input: "main", expected: false}, + {name: "branch with slash", input: "feature/foo", expected: false}, + {name: "tag semver", input: "v1.0.0", expected: false}, + {name: "too short 6 chars", input: "e5011a", expected: false}, + {name: "empty string", input: "", expected: false}, + {name: "hex with non-hex char", input: "e5011gg", expected: false}, + {name: "7 char all digits", input: "1234567", expected: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, looksLikeCommitHash(tt.input)) + }) + } +} + +func TestHashRepoURLStable(t *testing.T) { + a := hashRepoURL("https://example.com/repo.git") + b := hashRepoURL("https://example.com/repo.git") + c := hashRepoURL("https://example.com/other.git") + assert.Equal(t, a, b, "same URL must hash the same") + assert.NotEqual(t, a, c, "different URLs must differ") + assert.Len(t, a, 16, "8-byte hex") +} + +// git runs a git command in dir, failing the test on error. +func git(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t") + out, err := cmd.CombinedOutput() + require.NoError(t, err, "git %v: %s", args, out) +} + +func TestCloneOrUpdate(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + // Build a tiny local upstream repo with a branch. + upstream := t.TempDir() + git(t, upstream, "init", "-b", "main") + require.NoError(t, os.WriteFile(filepath.Join(upstream, "README"), []byte("hello"), 0o644)) + git(t, upstream, "add", ".") + git(t, upstream, "commit", "-m", "init") + + cache := t.TempDir() + ctx := context.Background() + log := logrus.New() + + // First call clones. + path, err := CloneOrUpdate(ctx, log, upstream, "main", cache) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(path, "README")) + assert.Equal(t, cache, filepath.Dir(path), "checkout lives directly under the cache dir") + + sha, err := HeadSHA(ctx, path) + require.NoError(t, err) + assert.Len(t, sha, 40) + + // Second call reuses the same cached path. + path2, err := CloneOrUpdate(ctx, log, upstream, "main", cache) + require.NoError(t, err) + assert.Equal(t, path, path2) +} From be636d8a7d14eb4b60ab0c862e25436be092833a Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 14:51:42 +0200 Subject: [PATCH 07/83] refactor(config): one shared cache dir (global.directories.cachedir) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace runner.directories.tmp_cachedir with global.directories.cachedir so a single on-disk cache is shared by both commands. It was misnamed (it defaults to ~/.cache/benchmarkoor, not a temp dir) and lived under `runner`, so the build command couldn't use it — the eest builder hard-coded /tmp instead. - config: add global.directories.cachedir + Config.ResolveCacheDir (default ~/.cache/benchmarkoor); remove runner.directories.tmp_cachedir. tmp_datadir (genuinely ephemeral scratch) stays under runner. - run: resolve the cache dir once and reuse it for executor sources, cpufreq, and the pre-run log buffer; drop the local getExecutorCacheDir helper. - runner: rename internal Config.TmpCacheDir -> CacheDir. - build: resolve the shared cache dir and pass it to the eest builder, which now caches the EEST repo under /eest-repos instead of /tmp. BREAKING: configs using runner.directories.tmp_cachedir must move to global.directories.cachedir (env: BENCHMARKOOR_GLOBAL_DIRECTORIES_CACHEDIR). --- cmd/benchmarkoor/build.go | 10 ++++++- cmd/benchmarkoor/run.go | 47 ++++++++----------------------- config.example.yaml | 8 ++++-- docs/configuration.md | 9 ++++-- pkg/builder/eest_payloads.go | 13 ++++----- pkg/builder/eest_payloads_test.go | 6 ++-- pkg/config/config.go | 33 ++++++++++++++++++---- pkg/config/config_test.go | 6 ++-- pkg/runner/lifecycle.go | 2 +- pkg/runner/runner.go | 2 +- 10 files changed, 74 insertions(+), 62 deletions(-) diff --git a/cmd/benchmarkoor/build.go b/cmd/benchmarkoor/build.go index 3c29a8171..f66151e8c 100644 --- a/cmd/benchmarkoor/build.go +++ b/cmd/benchmarkoor/build.go @@ -159,7 +159,15 @@ func buildBuilders(ctx context.Context, cfg *config.Config) ([]builder.Builder, return nil, nil, err } - builders = append(builders, builder.NewEESTPayloadsBuilder(log, cfg.Builder.EESTPayloads, runtime, mgr)) + cacheDir, err := cfg.ResolveCacheDir() + if err != nil { + stop() + + return nil, nil, err + } + + builders = append(builders, + builder.NewEESTPayloadsBuilder(log, cfg.Builder.EESTPayloads, runtime, mgr, cacheDir)) } return builders, stop, nil diff --git a/cmd/benchmarkoor/run.go b/cmd/benchmarkoor/run.go index 3ce952ace..7a64dab45 100644 --- a/cmd/benchmarkoor/run.go +++ b/cmd/benchmarkoor/run.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "os/signal" - "path/filepath" "strings" "syscall" "time" @@ -93,10 +92,15 @@ func runBenchmark(cmd *cobra.Command, args []string) error { } } - if dir := cfg.Runner.Directories.TmpCacheDir; dir != "" { - if err := fsutil.MkdirAll(dir, 0755, resultsOwner); err != nil { - return fmt.Errorf("creating tmp_cachedir %q: %w", dir, err) - } + // Resolve the shared cache dir (global.directories.cachedir, default + // ~/.cache/benchmarkoor) and ensure it exists. + cacheDir, err := cfg.ResolveCacheDir() + if err != nil { + return err + } + + if err := fsutil.MkdirAll(cacheDir, 0755, resultsOwner); err != nil { + return fmt.Errorf("creating cachedir %q: %w", cacheDir, err) } // Use consistent log format when client logs go to stdout. @@ -108,7 +112,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error { // instance's benchmarkoor.log, capturing logs from before RunInstance. // Created after the formatter is set so the buffered output matches the // per-instance log format. - preRunLogBuffer, err := runner.NewBufferHook(log.Formatter, cfg.Runner.Directories.TmpCacheDir) + preRunLogBuffer, err := runner.NewBufferHook(log.Formatter, cacheDir) if err != nil { return fmt.Errorf("creating pre-run log buffer: %w", err) } @@ -231,16 +235,6 @@ func runBenchmark(cmd *cobra.Command, args []string) error { var exec executor.Executor if cfg.Runner.Benchmark.Tests.Source.IsConfigured() { - cacheDir := cfg.Runner.Directories.TmpCacheDir - if cacheDir == "" { - var err error - - cacheDir, err = getExecutorCacheDir() - if err != nil { - return fmt.Errorf("getting cache directory: %w", err) - } - } - // Pass suite metadata to executor only when labels are present. var suiteMetadata *config.MetadataConfig if len(cfg.Runner.Benchmark.Tests.Metadata.Labels) > 0 { @@ -276,15 +270,6 @@ func runBenchmark(cmd *cobra.Command, args []string) error { // Create CPU frequency manager if CPU frequency settings are configured. var cpufreqMgr cpufreq.Manager if needsCPUFreqManager(cfg) { - cacheDir := cfg.Runner.Directories.TmpCacheDir - if cacheDir == "" { - var err error - cacheDir, err = getExecutorCacheDir() - if err != nil { - return fmt.Errorf("getting cache directory: %w", err) - } - } - cpufreqMgr = cpufreq.NewManager(log, cacheDir, cfg.GetCPUSysfsPath()) if err := cpufreqMgr.Start(ctx); err != nil { return fmt.Errorf("starting cpufreq manager: %w", err) @@ -328,7 +313,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error { GenesisURLs: cfg.Runner.Client.Config.Genesis, DataDirs: cfg.Runner.Client.DataDirs, TmpDataDir: cfg.Runner.Directories.TmpDataDir, - TmpCacheDir: cfg.Runner.Directories.TmpCacheDir, + CacheDir: cacheDir, TestFilter: cfg.Runner.Benchmark.Tests.Filter, FullConfig: cfg, StopAfterPrerun: stopAfterPrerun, @@ -391,16 +376,6 @@ func runBenchmark(cmd *cobra.Command, args []string) error { return nil } -// getExecutorCacheDir returns the cache directory for the executor. -func getExecutorCacheDir() (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("getting home directory: %w", err) - } - - return filepath.Join(homeDir, ".cache", "benchmarkoor"), nil -} - // needsCPUFreqManager returns true if any instance has CPU frequency settings configured. func needsCPUFreqManager(cfg *config.Config) bool { // Check global resource limits. diff --git a/config.example.yaml b/config.example.yaml index 0929070fa..216f57dde 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -6,6 +6,11 @@ global: log_level: ${LOG_LEVEL:-info} + # Optional directory configurations shared by the build and run commands. + # directories: + # # On-disk cache: executor git/archive clones (run) and the EEST repo clone + # # (build). Defaults to ~/.cache/benchmarkoor. + # cachedir: ${TMP_DIR:-/tmp}/benchmarkoor-cache runner: # Container runtime: "docker" (default) or "podman". @@ -24,8 +29,7 @@ runner: # directories: # # Directory for temporary datadir copies (defaults to system temp). # tmp_datadir: ${TMP_DIR:-/tmp}/benchmarkoor - # # Directory for executor cache - git clones, etc (defaults to ~/.cache/benchmarkoor). - # tmp_cachedir: ${TMP_DIR:-/tmp}/benchmarkoor-cache + # # (The shared cache dir lives under global.directories.cachedir.) # Optional: Override path to drop_caches file (default: /proc/sys/vm/drop_caches). # Useful when running in containers where the file is mounted at a different path. # drop_caches_path: /proc/sys/vm/drop_caches diff --git a/docs/configuration.md b/docs/configuration.md index 263ee76a4..49c072890 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -83,6 +83,8 @@ The `global` section contains application-wide settings. ```yaml global: log_level: info + directories: + cachedir: ~/.cache/benchmarkoor ``` ### Options @@ -90,6 +92,7 @@ global: | Option | Type | Default | Description | |--------|------|---------|-------------| | `log_level` | string | `info` | Logging level: `debug`, `info`, `warn`, `error` | +| `directories.cachedir` | string | `~/.cache/benchmarkoor` | On-disk cache shared by both commands: executor git/archive clones (`run`) and the EEST repo clone (`build`). | ## Runner Settings @@ -104,7 +107,6 @@ runner: run_timeout: 4h directories: tmp_datadir: /tmp/benchmarkoor - tmp_cachedir: /tmp/benchmarkoor-cache drop_caches_path: /proc/sys/vm/drop_caches cpu_sysfs_path: /sys/devices/system/cpu ``` @@ -118,8 +120,7 @@ runner: | `container_network` | string | `benchmarkoor` | Container network name | | `cleanup_on_start` | bool | `false` | Remove leftover containers/networks on startup | | `run_timeout` | string | - | Global timeout for the entire run covering all instances, setup, and teardown. Uses Go duration format (e.g., `4h`, `30m`). See [Runner Run Timeout](#runner-run-timeout) | -| `directories.tmp_datadir` | string | system temp | Directory for temporary datadir copies | -| `directories.tmp_cachedir` | string | `~/.cache/benchmarkoor` | Directory for executor cache (git clones, etc.) | +| `directories.tmp_datadir` | string | system temp | Directory for temporary datadir copies. (The shared cache dir is `global.directories.cachedir`.) | | `drop_caches_path` | string | `/proc/sys/vm/drop_caches` | Path to Linux drop_caches file (for containerized environments) | | `cpu_sysfs_path` | string | `/sys/devices/system/cpu` | Base path for CPU sysfs files (for containerized environments where `/sys` is read-only and the host path is bind-mounted elsewhere, e.g., `/host_sys_cpu`) | | `metadata.labels` | map[string]string | - | Arbitrary key-value labels attached to the run (see [Metadata Labels](#metadata-labels)) | @@ -1572,6 +1573,8 @@ builder: container_runtime: docker # docker | podman (default: inherits runner.container_runtime, then docker) # jwt: # Engine API secret, shared with the filler (default: benchmarkoor's DefaultJWT) # fill_command: [uv, run, fill-stateful] # argv prefix inside fill_image (this is the default) + # eest_repo: https://github.com/ethereum/execution-specs.git # cloned + mounted at /eest (default) + # eest_ref: forks/amsterdam # branch, tag, or commit to check out (default: forks/amsterdam) config: # shared per-target defaults; targets override when set filler_image: ethpandaops/geth:master fork: Osaka diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index 4bad1af9a..f8d35e6e0 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -59,18 +59,17 @@ type EESTPayloadsBuilder struct { repoCache string } -// eestRepoCacheDir is where EEST repo clones are cached between builds (so a -// recurring build of the same ref doesn't re-clone). Persists across runs. -var eestRepoCacheDir = filepath.Join(os.TempDir(), "benchmarkoor-eest-repos") - // NewEESTPayloadsBuilder constructs a builder bound to a specific container -// manager. The caller is expected to have Start()'d the manager and to -// Stop() it after the last Build() call. +// manager. cacheDir is the shared on-disk cache (global.directories.cachedir); +// EEST repo clones are cached under /eest-repos so recurring builds +// of the same ref don't re-clone. The caller is expected to have Start()'d the +// manager and to Stop() it after the last Build() call. func NewEESTPayloadsBuilder( log logrus.FieldLogger, cfg *config.EESTPayloadsConfig, runtime string, mgr docker.ContainerManager, + cacheDir string, ) *EESTPayloadsBuilder { return &EESTPayloadsBuilder{ log: log.WithField("component", "builder.eest_payloads"), @@ -78,7 +77,7 @@ func NewEESTPayloadsBuilder( runtime: runtime, mgr: mgr, registry: client.NewRegistry(), - repoCache: eestRepoCacheDir, + repoCache: filepath.Join(cacheDir, "eest-repos"), } } diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go index 53742ef90..73ca31050 100644 --- a/pkg/builder/eest_payloads_test.go +++ b/pkg/builder/eest_payloads_test.go @@ -157,7 +157,7 @@ func TestEESTPayloadsBuilder_Targets(t *testing.T) { }, } - b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}) + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}, t.TempDir()) got := b.Targets() require.Len(t, got, 2) @@ -171,7 +171,7 @@ func TestEESTPayloadsBuilder_BuildUnknownTarget(t *testing.T) { Targets: []config.EESTPayloadTarget{{FillerClient: "geth", OutputDir: "/srv/g"}}, } - b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}) + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}, t.TempDir()) _, err := b.Build(context.Background(), "nope", BuildOptions{}) require.Error(t, err) @@ -192,7 +192,7 @@ func TestEESTPayloadsBuilder_BuildSkipsPopulatedDir(t *testing.T) { }}, } - b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}) + b := NewEESTPayloadsBuilder(noopLogger(), cfg, "docker", &fakeMgr{}, t.TempDir()) skipped, err := b.Build(context.Background(), "compute", BuildOptions{}) require.NoError(t, err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 81a28c357..4b9952efe 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -588,17 +588,39 @@ type MetadataConfig struct { // GlobalConfig contains global application settings. type GlobalConfig struct { - LogLevel string `yaml:"log_level" mapstructure:"log_level"` + LogLevel string `yaml:"log_level" mapstructure:"log_level"` + Directories GlobalDirectoriesConfig `yaml:"directories,omitempty" mapstructure:"directories"` } -// DirectoriesConfig contains directory path configurations. +// GlobalDirectoriesConfig contains directory paths shared across the build and +// run commands. +type GlobalDirectoriesConfig struct { + // CacheDir is the on-disk cache shared by both commands: executor git/archive + // clones (run) and the EEST repo clone (build). If empty, defaults to + // ~/.cache/benchmarkoor. + CacheDir string `yaml:"cachedir,omitempty" mapstructure:"cachedir"` +} + +// DirectoriesConfig contains runner-specific directory path configurations. type DirectoriesConfig struct { // TmpDataDir is the directory for temporary datadir copies. // If empty, uses the system default temp directory. TmpDataDir string `yaml:"tmp_datadir,omitempty" mapstructure:"tmp_datadir"` - // TmpCacheDir is the directory for executor cache (git clones, etc). - // If empty, uses ~/.cache/benchmarkoor. - TmpCacheDir string `yaml:"tmp_cachedir,omitempty" mapstructure:"tmp_cachedir"` +} + +// ResolveCacheDir returns the configured global cache directory, defaulting to +// ~/.cache/benchmarkoor when unset. +func (c *Config) ResolveCacheDir() (string, error) { + if c.Global.Directories.CacheDir != "" { + return c.Global.Directories.CacheDir, nil + } + + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("getting home directory for cache dir: %w", err) + } + + return filepath.Join(home, ".cache", "benchmarkoor"), nil } // BenchmarkConfig contains benchmark-specific settings. @@ -1300,6 +1322,7 @@ func bindEnvKeys(v *viper.Viper) { keys := []string{ // Global settings "global.log_level", + "global.directories.cachedir", // Runner settings "runner.container_runtime", "runner.client_logs_to_stdout", diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index ae7f7a422..23a8c328c 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -106,12 +106,12 @@ runner: }, }, { - name: "nested field override - directories.tmp_cachedir", + name: "nested field override - global.directories.cachedir", envVars: map[string]string{ - "BENCHMARKOOR_RUNNER_DIRECTORIES_TMP_CACHEDIR": "/cache/custom", + "BENCHMARKOOR_GLOBAL_DIRECTORIES_CACHEDIR": "/cache/custom", }, validate: func(t *testing.T, cfg *Config) { - assert.Equal(t, "/cache/custom", cfg.Runner.Directories.TmpCacheDir) + assert.Equal(t, "/cache/custom", cfg.Global.Directories.CacheDir) }, }, { diff --git a/pkg/runner/lifecycle.go b/pkg/runner/lifecycle.go index af3dbe2a1..dbdfee614 100644 --- a/pkg/runner/lifecycle.go +++ b/pkg/runner/lifecycle.go @@ -201,7 +201,7 @@ func (r *runner) runContainerLifecycle( // Create temp files for genesis and JWT. tempDir, err := os.MkdirTemp( - r.cfg.TmpCacheDir, "benchmarkoor-"+instance.ID+"-", + r.cfg.CacheDir, "benchmarkoor-"+instance.ID+"-", ) if err != nil { return fmt.Errorf("creating temp directory: %w", err) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index f42f9568c..1ac1fa74d 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -60,7 +60,7 @@ type Config struct { GenesisURLs map[string]string DataDirs map[string]*config.DataDirConfig TmpDataDir string // Directory for temporary datadir copies (empty = system default) - TmpCacheDir string // Directory for temporary cache files (empty = system default) + CacheDir string // Shared on-disk cache dir (git/archive clones, etc.) ReadyTimeout time.Duration TestFilter string FullConfig *config.Config // Full config for resolving per-instance settings From c78085cd71ba1a8c8cb336c5473a7dab9db367ff Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 15:00:06 +0200 Subject: [PATCH 08/83] docs(examples): add state-actor + EEST stateful end-to-end config A worked example covering all three stages: build datadirs with state-actor, generate stateful EEST fixtures (config-driven eest_repo/eest_ref), and replay them per client. Includes the per-client boot specifics found while bringing this up (besu hyperledger image, reth pinned digest + skip-genesis-validation, nethermind sync flags, the genesis chainspec map), parametrised via ${STATE_DIR_PREFIX} / ${EEST_FIXTURES_DIR}. --- .../config.state-actor-eest.yaml | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 examples/configuration/config.state-actor-eest.yaml diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml new file mode 100644 index 000000000..f82a85bb0 --- /dev/null +++ b/examples/configuration/config.state-actor-eest.yaml @@ -0,0 +1,117 @@ +# Building data directories and EEST payloads +# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.yaml --force +# +# Running benchmark using geth: +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.yaml --limit-instance-id=geth +# +global: + log_level: info + +builder: + ## Stage 1: Building datadirs using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + reth: ghcr.io/ethereum/state-actor-reth:main + besu: ghcr.io/ethereum/state-actor-besu:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + pull_policy: always + config: + seed: 1234 + fork: osaka + chain: 1337 + gas_limit: 300000000 # 300M + target_size: 256MB + #target_size: 20GB + spec: | + entities: + - kind: eoa + name: bloated-eoa-2g + balance: "1000000000000000000" + nonce: 0 + code: "0xef01003333333333333333333333333333333333333333" + approximate_size_bytes: 2_000_000_000 + targets: + - client: geth + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + - client: reth + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth + - client: nethermind + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + - client: besu + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + + ## Stage 2: Building EEST stateful fixtures + eest_payloads: + fill_image: skylenet/eest-uv:latest # can also be built locally from Dockerfile.eest-filler + pull_policy: if-not-present + eest_repo: https://github.com/ethereum/execution-specs.git + eest_ref: forks/amsterdam + config: + filler_image: ethpandaops/geth:master + fork: osaka + gas_benchmark_values: [10] # Millions of gas to parametrise against + datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) + targets: + - name: payload-generator-geth + filler_client: geth + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures} + tests: + - tests/benchmark/compute # pytest paths inside the fill image + filter: bn128 # Quick subset for fast end-to-end iteration: only the bn128 compute tests. + +## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads +runner: + client_logs_to_stdout: true + cleanup_on_start: false + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + genesis: + nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + #geth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json + besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + + datadirs: + geth: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + method: copy + reth: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth + method: copy + nethermind: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + method: copy + besu: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + method: copy + + instances: + - id: geth + client: geth + image: ethpandaops/geth:master + - id: reth + client: reth + #image: ghcr.io/paradigmxyz/reth:latest + image: ghcr.io/paradigmxyz/reth:nightly@sha256:e528857e5e9ebc2c6cb99f28436e70ded38ca905629f00afc98d186e27d206e0 + - id: nethermind + client: nethermind + extra_args: + - --Sync.NetworkingEnabled=false + - --Sync.PivotNumber=0 + - --Sync.SynchronizationEnabled=false + #image: ethpandaops/nethermind:master + image: nethermind/nethermind:1.37.0 + - id: besu + client: besu + image: hyperledger/besu:latest From ee4d3d053dc85f7e76f73f96d08464fa6a41a8bc Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 15:15:58 +0200 Subject: [PATCH 09/83] fix(builder): resolve temp symlinks so container mounts work on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builder containers bind-mount host files written under os.TempDir(). On macOS $TMPDIR is /var/folders/… where /var is a symlink to /private/var; Docker Desktop shares /private but not the /var alias, so a single-file mount at the unresolved path silently fails to appear in the container — e.g. the geth filler aborts with "open /tmp/config.toml: no such file or directory". Add mountTempDir() (EvalSymlinks of os.TempDir()) and use it for every mount-source temp file: the eest filler's JWT, geth config.toml, and datadir copy, plus the state-actor inline-spec file. No-op on Linux; on macOS it yields /private/var/folders/… which Docker Desktop shares. (OrbStack/colima share the whole FS and aren't affected either way.) --- pkg/builder/eest_payloads.go | 6 +++--- pkg/builder/state_actor.go | 2 +- pkg/builder/util.go | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index f8d35e6e0..aa1601019 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -234,7 +234,7 @@ func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t prepared, err := provider.Prepare(ctx, &datadir.ProviderConfig{ SourceDir: t.SourceDir, InstanceID: "eest-fill-" + t.EffectiveName(), - TmpDir: os.TempDir(), + TmpDir: mountTempDir(), }) if err != nil { return fmt.Errorf("preparing datadir copy: %w", err) @@ -587,7 +587,7 @@ func buildFillArgs( // writeTempJWT writes the JWT secret to a temp file readable by the // container UID (0644) and returns its path plus a cleanup callback. func writeTempJWT(secret string) (string, func(), error) { - f, err := os.CreateTemp("", "benchmarkoor-eest-jwt-*") + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-eest-jwt-*") if err != nil { return "", nil, fmt.Errorf("creating temp jwt file: %w", err) } @@ -632,7 +632,7 @@ func writeTempConfigFiles(files map[string]string) ([]docker.Mount, func(), erro } for target, content := range files { - f, err := os.CreateTemp("", "benchmarkoor-eest-config-*") + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-eest-config-*") if err != nil { cleanup() diff --git a/pkg/builder/state_actor.go b/pkg/builder/state_actor.go index 905ea8c01..c83885b9b 100644 --- a/pkg/builder/state_actor.go +++ b/pkg/builder/state_actor.go @@ -238,7 +238,7 @@ func (b *StateActorBuilder) resolveSpecPath() (string, func(), error) { return abs, nil, nil case config.StateActorSpecInline: - f, err := os.CreateTemp("", "benchmarkoor-state-actor-spec-*.yaml") + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-state-actor-spec-*.yaml") if err != nil { return "", nil, fmt.Errorf("creating temp spec file: %w", err) } diff --git a/pkg/builder/util.go b/pkg/builder/util.go index 5445f6b99..9914b2c9b 100644 --- a/pkg/builder/util.go +++ b/pkg/builder/util.go @@ -7,11 +7,26 @@ import ( "fmt" "io" "os" + "path/filepath" "time" "github.com/ethpandaops/benchmarkoor/pkg/config" ) +// mountTempDir returns the system temp dir with symlinks resolved. It is the +// base for host files bind-mounted into builder containers. On macOS $TMPDIR is +// /var/folders/… where /var is a symlink to /private/var; Docker Desktop shares +// /private but not the /var alias, so bind-mounting a file at the unresolved +// path silently fails to appear in the container ("no such file or directory"). +// Resolving to /private/var/… makes the mount work; on Linux it's a no-op. +func mountTempDir() string { + if resolved, err := filepath.EvalSymlinks(os.TempDir()); err == nil { + return resolved + } + + return os.TempDir() +} + // isPopulated reports whether dir exists and contains at least one // entry. A missing dir returns (false, nil). func isPopulated(dir string) (bool, error) { From be482caf1a41fde4b96372b994c9942984853cec Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 15:25:05 +0200 Subject: [PATCH 10/83] fix: mac toml mount --- pkg/builder/eest_payloads.go | 60 ++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index aa1601019..53625d482 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -250,11 +250,17 @@ func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t streamCtx, streamCancel := context.WithCancel(ctx) defer streamCancel() - fillerID, fillerIP, err := b.startFiller(ctx, streamCtx, log, t, spec, prepared.MountPath, jwtPath) + fillerID, fillerIP, configCleanup, err := b.startFiller(ctx, streamCtx, log, t, spec, prepared.MountPath, jwtPath) if err != nil { return err } + // Order (defers run LIFO): stop the container first, then remove the temp + // config file it bind-mounted. The host file must outlive the container — + // removing it earlier makes /tmp/config.toml vanish inside the container + // (Docker Desktop syncs the bind mount by path), which geth reports as + // "open /tmp/config.toml: no such file or directory". + defer configCleanup() defer b.stopFiller(log, fillerID) log.Info("Waiting for filler client RPC to become ready") @@ -281,15 +287,31 @@ func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t } // startFiller boots the filler EL client and returns its container ID and IP. +// +// On success it returns a cleanup func that removes the temp config file; the +// caller must defer it for the lifetime of the container (the bind-mounted host +// file has to outlive geth's startup read of /tmp/config.toml). On error the +// cleanup is run here and a no-op is returned. func (b *EESTPayloadsBuilder) startFiller( ctx, streamCtx context.Context, log logrus.FieldLogger, t *config.EESTPayloadTarget, spec client.Spec, dataMount, jwtPath string, -) (string, string, error) { - if err := b.mgr.PullImage(ctx, t.FillerImage, b.cfg.PullPolicy); err != nil { - return "", "", fmt.Errorf("pulling filler image %q: %w", t.FillerImage, err) +) (id string, ip string, cleanup func(), err error) { + configCleanup := func() {} + + // Until the container is successfully handed off to the caller, any error + // return must remove the temp config file itself — the caller only defers + // the cleanup once startFiller succeeds. + defer func() { + if err != nil { + configCleanup() + } + }() + + if err = b.mgr.PullImage(ctx, t.FillerImage, b.cfg.PullPolicy); err != nil { + return "", "", nil, fmt.Errorf("pulling filler image %q: %w", t.FillerImage, err) } mounts := []docker.Mount{ @@ -297,20 +319,18 @@ func (b *EESTPayloadsBuilder) startFiller( {Source: jwtPath, Target: spec.JWTPath(), Type: "bind", ReadOnly: true}, } - configCleanup := func() {} - if files := spec.DefaultConfigFiles(); len(files) > 0 { - configMounts, cleanup, err := writeTempConfigFiles(files) - if err != nil { - return "", "", err + configMounts, cfgCleanup, cfgErr := writeTempConfigFiles(files) + if cfgErr != nil { + err = cfgErr + + return "", "", nil, cfgErr } mounts = append(mounts, configMounts...) - configCleanup = cleanup + configCleanup = cfgCleanup } - defer configCleanup() - if t.GenesisFile != "" { mounts = append(mounts, docker.Mount{ Source: t.GenesisFile, Target: spec.GenesisPath(), Type: "bind", ReadOnly: true, @@ -319,7 +339,7 @@ func (b *EESTPayloadsBuilder) startFiller( suffix, err := randSuffix() if err != nil { - return "", "", fmt.Errorf("generating container name suffix: %w", err) + return "", "", nil, fmt.Errorf("generating container name suffix: %w", err) } cmd := fillerGethCommand(t, spec) @@ -340,15 +360,15 @@ func (b *EESTPayloadsBuilder) startFiller( log.WithField("argv", cmd).Info("Starting filler client") - id, err := b.mgr.CreateContainer(ctx, containerSpec) + id, err = b.mgr.CreateContainer(ctx, containerSpec) if err != nil { - return "", "", fmt.Errorf("creating filler container: %w", err) + return "", "", nil, fmt.Errorf("creating filler container: %w", err) } - if err := b.mgr.StartContainer(ctx, id); err != nil { + if err = b.mgr.StartContainer(ctx, id); err != nil { _ = b.mgr.RemoveContainer(context.Background(), id) - return "", "", fmt.Errorf("starting filler container: %w", err) + return "", "", nil, fmt.Errorf("starting filler container: %w", err) } go func() { @@ -360,14 +380,14 @@ func (b *EESTPayloadsBuilder) startFiller( } }() - ip, err := b.mgr.GetContainerIP(ctx, id, eestBuildNetwork) + ip, err = b.mgr.GetContainerIP(ctx, id, eestBuildNetwork) if err != nil { _ = b.mgr.RemoveContainer(context.Background(), id) - return "", "", fmt.Errorf("getting filler container IP: %w", err) + return "", "", nil, fmt.Errorf("getting filler container IP: %w", err) } - return id, ip, nil + return id, ip, configCleanup, nil } // stopFiller stops and removes the filler container. It uses a background From 12552ddc2f8bcf1dd047fe6b19abe5d3e5227ecf Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 16:01:43 +0200 Subject: [PATCH 11/83] feat(builder): build fill image from fill_dockerfile when set Let builder.eest_payloads.fill_dockerfile point at a Dockerfile that benchmarkoor builds with the container runtime at build time, instead of requiring a pre-built fill_image. The built image is tagged fill_image when set, else benchmarkoor-eest-fill:local; one of fill_image / fill_dockerfile is required. - config: add fill_dockerfile + BuildsFillImage/ResolveFillImageTag; require one of the two and reject a missing Dockerfile at config time. - builder: ensureFillImage builds (once per run, via ` build`) or pulls, then uses the resulting tag for the fill container. - example: config.state-actor-eest.yaml now builds Dockerfile.eest-filler, so `benchmarkoor build` is self-contained (no manual docker build). - docs + config.example updated. --- config.example.yaml | 4 ++ docs/configuration.md | 6 +- .../config.state-actor-eest.yaml | 4 +- pkg/builder/eest_payloads.go | 67 ++++++++++++++++++- pkg/config/config.go | 43 ++++++++++-- pkg/config/config_test.go | 23 ++++++- 6 files changed, 135 insertions(+), 12 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 216f57dde..0ec1ce07a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -280,6 +280,10 @@ runner: # # docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . # eest_payloads: # fill_image: ghcr.io/your-org/eest-fill-stateful:latest +# # Instead of a pre-built fill_image, point at a Dockerfile and benchmarkoor +# # builds it (with the container runtime) at build time, tagging it fill_image +# # if set or a default local tag. One of fill_image / fill_dockerfile is required. +# # fill_dockerfile: Dockerfile.eest-filler # # pull_policy: always # always | if-not-present | never # # container_runtime: docker # defaults to runner.container_runtime, then docker # # jwt: # Engine API secret shared with the filler (default: built-in) diff --git a/docs/configuration.md b/docs/configuration.md index 49c072890..aa9eb8fb4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1569,6 +1569,7 @@ builder: builder: eest_payloads: fill_image: ghcr.io/your-org/eest-fill-stateful:latest # image carrying `uv run fill-stateful` + # fill_dockerfile: Dockerfile.eest-filler # build the fill image instead of pulling one pull_policy: always # always | if-not-present | never (default: always) container_runtime: docker # docker | podman (default: inherits runner.container_runtime, then docker) # jwt: # Engine API secret, shared with the filler (default: benchmarkoor's DefaultJWT) @@ -1594,8 +1595,9 @@ builder: | Option | Type | Default | Description | |---|---|---|---| -| `fill_image` | string | – | **Required.** Container image carrying the `fill-stateful` command (`uv` + execution-specs). | -| `pull_policy` | string | `always` | One of `always`, `if-not-present`, `never`. Applies to both the fill image and the filler image. | +| `fill_image` | string | – | Pre-built container image carrying the uv/python toolchain that runs `fill-stateful`. Required unless `fill_dockerfile` is set. | +| `fill_dockerfile` | string | – | Path to a Dockerfile (e.g. `Dockerfile.eest-filler`) that benchmarkoor builds with the container runtime at build time, instead of pulling a pre-built image. Tagged `fill_image` when set, else `benchmarkoor-eest-fill:local`. Requires the runtime's `build` CLI (docker/podman) on the host. One of `fill_image` / `fill_dockerfile` is required. | +| `pull_policy` | string | `always` | One of `always`, `if-not-present`, `never`. Applies to both the fill image and the filler image (ignored for a locally built `fill_dockerfile`). | | `container_runtime` | string | runner's runtime, then `docker` | Container runtime for the filler + fill containers. | | `jwt` | string | benchmarkoor's `DefaultJWT` | Engine API JWT secret; shared between the filler client and `fill-stateful`. | | `fill_command` | []string | `[uv, run, fill-stateful]` | argv prefix invoked inside `fill_image` before the `fill-stateful` flags. Override if your image exposes the command differently. | diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index f82a85bb0..e36567433 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -43,7 +43,9 @@ builder: ## Stage 2: Building EEST stateful fixtures eest_payloads: - fill_image: skylenet/eest-uv:latest # can also be built locally from Dockerfile.eest-filler + # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead + # of pulling a pre-built one. Alternatively set fill_image: . + fill_dockerfile: Dockerfile.eest-filler pull_policy: if-not-present eest_repo: https://github.com/ethereum/execution-specs.git eest_ref: forks/amsterdam diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index 53625d482..839a6e5c6 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -5,9 +5,11 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "strconv" "strings" + "sync" "time" "github.com/ethpandaops/benchmarkoor/pkg/client" @@ -57,6 +59,13 @@ type EESTPayloadsBuilder struct { mgr docker.ContainerManager registry client.Registry repoCache string + + // fillImage is resolved once per builder: built from fill_dockerfile or the + // configured fill_image. Guarded by fillImageOnce so a multi-target build + // only builds the image a single time. + fillImageOnce sync.Once + fillImage string + fillImageErr error } // NewEESTPayloadsBuilder constructs a builder bound to a specific container @@ -404,6 +413,57 @@ func (b *EESTPayloadsBuilder) stopFiller(log logrus.FieldLogger, id string) { } // runFill runs the fill-stateful container against the live filler client. +// ensureFillImage returns the fill image reference, building it from +// fill_dockerfile (once, regardless of target count) when configured, otherwise +// pulling the configured fill_image. +func (b *EESTPayloadsBuilder) ensureFillImage(ctx context.Context, log logrus.FieldLogger) (string, error) { + if !b.cfg.BuildsFillImage() { + if err := b.mgr.PullImage(ctx, b.cfg.FillImage, b.cfg.PullPolicy); err != nil { + return "", fmt.Errorf("pulling fill image %q: %w", b.cfg.FillImage, err) + } + + return b.cfg.FillImage, nil + } + + b.fillImageOnce.Do(func() { + tag := b.cfg.ResolveFillImageTag() + if err := b.buildFillImage(ctx, log, b.cfg.FillDockerfile, tag); err != nil { + b.fillImageErr = err + + return + } + + b.fillImage = tag + }) + + return b.fillImage, b.fillImageErr +} + +// buildFillImage builds the fill image from dockerfile, tagging it tag, by +// shelling out to the configured container runtime's `build`. The build +// context is the Dockerfile's directory. +func (b *EESTPayloadsBuilder) buildFillImage(ctx context.Context, log logrus.FieldLogger, dockerfile, tag string) error { + contextDir := filepath.Dir(dockerfile) + + log.WithFields(logrus.Fields{ + "dockerfile": dockerfile, + "tag": tag, + "runtime": b.runtime, + }).Info("Building fill image") + + cmd := exec.CommandContext(ctx, b.runtime, "build", "-f", dockerfile, "-t", tag, contextDir) + + w := containerStream("BULD", "fill-image-build") + cmd.Stdout = w + cmd.Stderr = w + + if err := cmd.Run(); err != nil { + return fmt.Errorf("building fill image from %q: %w", dockerfile, err) + } + + return nil +} + func (b *EESTPayloadsBuilder) runFill( ctx context.Context, log logrus.FieldLogger, @@ -412,8 +472,9 @@ func (b *EESTPayloadsBuilder) runFill( spec client.Spec, jwtPath, snapshotHash, eestRepoPath string, ) error { - if err := b.mgr.PullImage(ctx, b.cfg.FillImage, b.cfg.PullPolicy); err != nil { - return fmt.Errorf("pulling fill image %q: %w", b.cfg.FillImage, err) + fillImage, err := b.ensureFillImage(ctx, log) + if err != nil { + return err } args := buildFillArgs(b.cfg.ResolveFillCommand(), t, fillerIP, spec, snapshotHash) @@ -457,7 +518,7 @@ func (b *EESTPayloadsBuilder) runFill( containerSpec := &docker.ContainerSpec{ Name: fmt.Sprintf("benchmarkoor-build-eest-fill-%s-%s", t.FillerClient, suffix), - Image: b.cfg.FillImage, + Image: fillImage, Command: args, Mounts: mounts, NetworkName: eestBuildNetwork, diff --git a/pkg/config/config.go b/pkg/config/config.go index 4b9952efe..247811c2c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -294,8 +294,14 @@ var stateActorValidPullPolicies = map[string]bool{ type EESTPayloadsConfig struct { ContainerRuntime string `yaml:"container_runtime,omitempty" mapstructure:"container_runtime"` FillImage string `yaml:"fill_image,omitempty" mapstructure:"fill_image"` - PullPolicy string `yaml:"pull_policy,omitempty" mapstructure:"pull_policy"` - JWT string `yaml:"jwt,omitempty" mapstructure:"jwt"` + // FillDockerfile, when set, makes benchmarkoor build the fill image from the + // given Dockerfile at build time (using the container runtime), instead of + // pulling a pre-built image. The built image is tagged FillImage when set, + // otherwise DefaultFillImageTag. Mutually informative with FillImage: at + // least one of the two must be provided. + FillDockerfile string `yaml:"fill_dockerfile,omitempty" mapstructure:"fill_dockerfile"` + PullPolicy string `yaml:"pull_policy,omitempty" mapstructure:"pull_policy"` + JWT string `yaml:"jwt,omitempty" mapstructure:"jwt"` // FillCommand is the argv prefix invoked inside FillImage before the // fill-stateful flags. Defaults to ["uv", "run", "fill-stateful"]. FillCommand []string `yaml:"fill_command,omitempty" mapstructure:"fill_command"` @@ -319,8 +325,27 @@ const ( // DefaultEESTRef is the execution-specs ref cloned for fill-stateful when // builder.eest_payloads.eest_ref is unset (where fill-stateful currently lives). DefaultEESTRef = "forks/amsterdam" + // DefaultFillImageTag is the tag applied to a fill image built from + // fill_dockerfile when no explicit fill_image tag is given. + DefaultFillImageTag = "benchmarkoor-eest-fill:local" ) +// BuildsFillImage reports whether benchmarkoor should build the fill image from +// FillDockerfile (rather than pulling a pre-built FillImage). +func (e *EESTPayloadsConfig) BuildsFillImage() bool { + return e.FillDockerfile != "" +} + +// ResolveFillImageTag returns the image reference for the fill container: the +// configured FillImage, or DefaultFillImageTag when only a Dockerfile is set. +func (e *EESTPayloadsConfig) ResolveFillImageTag() string { + if e.FillImage != "" { + return e.FillImage + } + + return DefaultFillImageTag +} + // ResolveEESTRepo returns the configured EEST repo URL, defaulting to // DefaultEESTRepo. func (e *EESTPayloadsConfig) ResolveEESTRepo() string { @@ -1979,13 +2004,21 @@ func (c *Config) validateEESTPayloads() error { ) } - if ep.FillImage == "" { + if ep.FillImage == "" && ep.FillDockerfile == "" { return fmt.Errorf( - "builder.eest_payloads.fill_image is required " + - "(the container image carrying the fill-stateful command)", + "builder.eest_payloads: one of fill_image (a pre-built image) or " + + "fill_dockerfile (built by benchmarkoor) is required", ) } + // Reject a missing Dockerfile at config time so typos surface early. + // Relative paths are resolved against the working directory. + if ep.FillDockerfile != "" { + if _, err := os.Stat(ep.FillDockerfile); err != nil { + return fmt.Errorf("builder.eest_payloads.fill_dockerfile: %w", err) + } + } + seenOutputs := make(map[string]int, len(ep.Targets)) seenNames := make(map[string]int, len(ep.Targets)) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 23a8c328c..ba07c6aed 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3687,6 +3687,11 @@ func TestValidateEESTPayloads(t *testing.T) { } } + dockerfile := filepath.Join(dirA, "Dockerfile.eest-filler") + if err := os.WriteFile(dockerfile, []byte("FROM scratch\n"), 0o644); err != nil { + t.Fatal(err) + } + tests := []struct { name string ep *EESTPayloadsConfig @@ -3705,13 +3710,29 @@ func TestValidateEESTPayloads(t *testing.T) { }, }, { - name: "missing fill_image", + name: "neither fill_image nor fill_dockerfile", ep: &EESTPayloadsConfig{ Targets: []EESTPayloadTarget{base(dirA)}, }, wantErr: true, errSubstr: "fill_image", }, + { + name: "fill_dockerfile only is valid", + ep: &EESTPayloadsConfig{ + FillDockerfile: dockerfile, + Targets: []EESTPayloadTarget{base(dirA)}, + }, + }, + { + name: "fill_dockerfile not found", + ep: &EESTPayloadsConfig{ + FillDockerfile: filepath.Join(dirB, "missing", "Dockerfile"), + Targets: []EESTPayloadTarget{base(dirA)}, + }, + wantErr: true, + errSubstr: "fill_dockerfile", + }, { name: "eest_repo without eest_ref is valid (ref defaults)", ep: &EESTPayloadsConfig{ From 4d78d2587ea4dfa6ff5710a130fc9f4f0db7a3f4 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 16:08:47 +0200 Subject: [PATCH 12/83] fix(builder): install clang/libclang in fill image for native EEST deps uv run fill-stateful syncs ethereum-execution[optimized], which pulls in rust-pyspec-glue. On arm64 there is no prebuilt wheel, so its native mdbx-sys crate is compiled from source; the build script runs bindgen, which dlopens libclang. The fill image only carried build-essential, so the build failed with "Unable to find libclang ... set the LIBCLANG_PATH environment variable". Add clang, libclang-dev and pkg-config to the image and pin LIBCLANG_PATH=/usr/lib/llvm-14/lib (bookworm's default LLVM major) so bindgen/clang-sys locates libclang deterministically. Cargo is already bootstrapped by uv at build time, so only the clang toolchain was missing. --- Dockerfile.eest-filler | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Dockerfile.eest-filler b/Dockerfile.eest-filler index 99792c7e5..dee864909 100644 --- a/Dockerfile.eest-filler +++ b/Dockerfile.eest-filler @@ -29,15 +29,27 @@ FROM python:${PYTHON_VERSION}-slim-bookworm COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ # Runtime deps: git (fill-stateful reads its commit hash from the checkout, and -# uv resolves git deps); build-essential/headers for any EEST dependency without -# a prebuilt wheel; ca-certificates for HTTPS to PyPI. +# uv resolves git deps); ca-certificates for HTTPS to PyPI; build-essential plus +# clang/libclang-dev/pkg-config for EEST dependencies without a prebuilt wheel. +# In particular `ethereum-execution[optimized]` pulls in rust-pyspec-glue, whose +# native mdbx-sys crate has no arm64 wheel and is compiled from source: its build +# script runs bindgen, which dlopens libclang. Cargo itself is bootstrapped by uv +# at build time, so only the clang toolchain has to be present in the image. RUN apt-get update \ && apt-get install -y --no-install-recommends \ git \ ca-certificates \ build-essential \ + clang \ + libclang-dev \ + pkg-config \ && rm -rf /var/lib/apt/lists/* +# Point bindgen/clang-sys straight at the bookworm-default LLVM 14 libclang so it +# never falls back to its "couldn't find any valid shared libraries" failure. The +# base image is pinned to bookworm (above), so the LLVM major version is stable. +ENV LIBCLANG_PATH=/usr/lib/llvm-14/lib + ENV UV_NO_PROGRESS=1 # The execution-specs checkout is bind-mounted here by benchmarkoor at run time; From 212adf66cbadcfcc8da39388ca0b34a4421d43b3 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 17 Jun 2026 16:44:34 +0200 Subject: [PATCH 13/83] fix(stats): stop Docker stats collection from inflating MGas/s and run time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On macOS there is no cgroup, so the executor falls back to the Docker Stats API. Two problems compounded there: 1. Correctness: executeRPC read the post-request stats snapshot between http.Do() returning (headers only) and the response body read, i.e. INSIDE the measured timing window. The Docker one-shot ContainerStats(stream=false) call blocks ~1-2s on the daemon's collection cycle, so that latency landed in `duration = bodyReadComplete - wroteRequest` — the value MGas/s is derived from. Every engine_newPayload measured ~2.0s and MGas/s came out ~200x low (e.g. 4.98 MGas/s for a 10M-gas block). 2. Performance: issuing a blocking ~2s one-shot stats request twice per RPC (before + after) made whole runs crawl (~2.5h). Fixes: - Move the after-request stats read out of the timing window via a new collectResourceDelta helper; the body read, bodyReadComplete and duration are captured first, so stats-backend latency can never be attributed to the RPC. - Rewrite dockerReader to stream: one long-lived ContainerStats(stream=true) connection feeds a cached latest sample, so ReadStats() is non-blocking. Includes lifecycle control (context cancel + Close wait), reconnect-on-error and copy-on-read. Add docker_reader_test.go (passes under -race). Caveat: the Docker stats stream refreshes ~1/s, so per-RPC resource deltas for sub-second calls on macOS are coarse (often 0). Linux still uses the fine-grained cgroup reader. The key properties — accurate MGas/s and fast runs — are restored. --- pkg/executor/executor.go | 62 +++++++---- pkg/stats/docker_reader.go | 182 +++++++++++++++++++++++++++----- pkg/stats/docker_reader_test.go | 107 +++++++++++++++++++ 3 files changed, 304 insertions(+), 47 deletions(-) create mode 100644 pkg/stats/docker_reader_test.go diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index f6a253b47..4c305f8e4 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -1296,29 +1296,12 @@ func (e *executor) executeRPC( start := time.Now() resp, err := http.DefaultClient.Do(req) - // Read stats AFTER the request completes and compute delta. - // This captures resource usage during server processing, not during body read. - var delta *ResourceDelta - if e.statsReader != nil && beforeStats != nil { - if afterStats, readErr := e.statsReader.ReadStats(); readErr == nil { - statsDelta := stats.ComputeDelta(beforeStats, afterStats) - if statsDelta != nil { - delta = &ResourceDelta{ - MemoryDelta: statsDelta.MemoryDelta, - MemoryAbsBytes: afterStats.Memory, - CPUDeltaUsec: statsDelta.CPUDeltaUsec, - DiskReadBytes: statsDelta.DiskReadBytes, - DiskWriteBytes: statsDelta.DiskWriteBytes, - DiskReadOps: statsDelta.DiskReadOps, - DiskWriteOps: statsDelta.DiskWriteOps, - } - } - } - } - if err != nil { fullDuration := time.Since(start).Nanoseconds() + // Collect the delta after timing is captured (see collectResourceDelta). + delta := e.collectResourceDelta(beforeStats) + return "", 0, fullDuration, delta, fmt.Errorf("executing request: %w", err) } @@ -1335,6 +1318,13 @@ func (e *executor) executeRPC( duration = bodyReadComplete.Sub(wroteRequest).Nanoseconds() } + // Read the AFTER stats only now that the timing window is closed. The stats + // backend can block — the Docker Stats API on macOS takes ~1-2s per read — + // and any time spent there must NOT count toward the measured RPC duration, + // which feeds MGas/s. Reading it earlier (between Do() and the body read) + // inflated every newPayload to ~2s and crushed MGas/s. + delta := e.collectResourceDelta(beforeStats) + if err != nil { return "", duration, fullDuration, delta, fmt.Errorf("reading response: %w", err) } @@ -1342,6 +1332,38 @@ func (e *executor) executeRPC( return strings.TrimSpace(string(body)), duration, fullDuration, delta, nil } +// collectResourceDelta reads the post-request stats snapshot and diffs it +// against before, returning nil when stats collection is disabled/unavailable. +// +// It MUST be called OUTSIDE the request-timing window: the stats backend can +// block (notably the Docker Stats API on macOS), and that latency must never be +// attributed to the RPC under measurement. +func (e *executor) collectResourceDelta(before *stats.Stats) *ResourceDelta { + if e.statsReader == nil || before == nil { + return nil + } + + afterStats, err := e.statsReader.ReadStats() + if err != nil { + return nil + } + + statsDelta := stats.ComputeDelta(before, afterStats) + if statsDelta == nil { + return nil + } + + return &ResourceDelta{ + MemoryDelta: statsDelta.MemoryDelta, + MemoryAbsBytes: afterStats.Memory, + CPUDeltaUsec: statsDelta.CPUDeltaUsec, + DiskReadBytes: statsDelta.DiskReadBytes, + DiskWriteBytes: statsDelta.DiskWriteBytes, + DiskReadOps: statsDelta.DiskReadOps, + DiskWriteOps: statsDelta.DiskWriteOps, + } +} + // rpcRequest is used to parse the method from a JSON-RPC request. type rpcRequest struct { Method string `json:"method"` diff --git a/pkg/stats/docker_reader.go b/pkg/stats/docker_reader.go index 7551719a7..06f4c8856 100644 --- a/pkg/stats/docker_reader.go +++ b/pkg/stats/docker_reader.go @@ -4,23 +4,60 @@ import ( "context" "encoding/json" "fmt" + "io" + "sync" + "time" "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" "github.com/sirupsen/logrus" ) -// dockerReader implements Reader using Docker Stats API. +const ( + // firstSampleTimeout bounds how long newDockerReader waits for the first + // streamed stats sample before returning. The stream usually delivers a + // sample within ~1s; this is a one-time wait per reader, not per ReadStats. + firstSampleTimeout = 5 * time.Second + + // streamRetryDelay is the backoff between stats-stream reconnect attempts + // (e.g. after a transient daemon error or a container restart). + streamRetryDelay = 1 * time.Second + + // closeTimeout bounds how long Close waits for the streaming goroutine to + // exit after cancellation. + closeTimeout = 2 * time.Second +) + +// dockerReader implements Reader using the Docker Stats API. +// +// A one-shot stats request (ContainerStats with stream=false) blocks on the +// daemon's next collection cycle — ~1-2s on Docker Desktop/OrbStack for macOS. +// Issuing one per ReadStats made benchmark runs crawl and, when read inside a +// timing window, inflated the measured RPC duration. Instead this reader opens a +// single streaming stats connection in the background and caches the most recent +// sample; ReadStats returns that cached snapshot without touching the daemon, so +// it is effectively instant. The trade-off is granularity: samples refresh at +// the daemon's cadence (~1/s), so per-RPC deltas for sub-second calls are coarse +// — acceptable on the macOS fallback path, where the cgroup reader is unavailable. type dockerReader struct { log logrus.FieldLogger - client *client.Client containerID string + + cancel context.CancelFunc + done chan struct{} + + mu sync.RWMutex + latest *Stats + + readyOnce sync.Once + ready chan struct{} // closed once the first sample has been cached } // Ensure interface compliance. var _ Reader = (*dockerReader)(nil) -// newDockerReader creates a new Docker Stats API reader. +// newDockerReader creates a streaming Docker Stats API reader and waits briefly +// for the first sample so the initial ReadStats has data. func newDockerReader( log logrus.FieldLogger, dockerClient *client.Client, @@ -30,11 +67,25 @@ func newDockerReader( return nil, fmt.Errorf("docker client is nil") } - return &dockerReader{ + ctx, cancel := context.WithCancel(context.Background()) + + r := &dockerReader{ log: log.WithField("reader", "docker"), - client: dockerClient, containerID: containerID, - }, nil + cancel: cancel, + done: make(chan struct{}), + ready: make(chan struct{}), + } + + go r.stream(ctx, dockerClient) + + select { + case <-r.ready: + case <-time.After(firstSampleTimeout): + r.log.Warn("Timed out waiting for first Docker stats sample; metrics may be delayed") + } + + return r, nil } // Type returns the reader implementation type. @@ -42,43 +93,120 @@ func (r *dockerReader) Type() string { return "docker" } -// Close releases any resources held by the reader. +// Close cancels the streaming goroutine and waits briefly for it to exit. func (r *dockerReader) Close() error { + r.cancel() + + select { + case <-r.done: + case <-time.After(closeTimeout): + } + return nil } -// ReadStats returns current resource metrics using Docker Stats API. +// ReadStats returns the most recent cached sample. It never blocks on the Docker +// daemon. Returns an error only if no sample has been collected yet. func (r *dockerReader) ReadStats() (*Stats, error) { - // Use one-shot stats (stream=false) for lower overhead. - ctx := context.Background() + r.mu.RLock() + defer r.mu.RUnlock() + + if r.latest == nil { + return nil, fmt.Errorf("no docker stats sample available yet") + } + + // Return a copy so callers cannot mutate the cached snapshot. + snapshot := *r.latest + + return &snapshot, nil +} + +// stream maintains a long-lived ContainerStats connection, decoding samples into +// the cache until the context is cancelled. It reconnects on transient errors +// and container restarts. +func (r *dockerReader) stream(ctx context.Context, dockerClient *client.Client) { + defer close(r.done) + + for ctx.Err() == nil { + resp, err := dockerClient.ContainerStats(ctx, r.containerID, true) + if err != nil { + if ctx.Err() != nil { + return + } + + r.log.WithError(err).Debug("Docker stats stream failed; retrying") + + if !sleepCtx(ctx, streamRetryDelay) { + return + } + + continue + } + + r.consume(ctx, resp.Body) + _ = resp.Body.Close() + + if ctx.Err() != nil { + return + } - statsResp, err := r.client.ContainerStats(ctx, r.containerID, false) - if err != nil { - return nil, fmt.Errorf("getting container stats: %w", err) + // The stream ended (e.g. container stop/restart); back off and retry. + if !sleepCtx(ctx, streamRetryDelay) { + return + } } - defer func() { _ = statsResp.Body.Close() }() +} + +// consume decodes stats samples from a single stream connection until it ends or +// the context is cancelled, updating the cache for each sample. +func (r *dockerReader) consume(ctx context.Context, body io.Reader) { + dec := json.NewDecoder(body) + + for ctx.Err() == nil { + var ds container.StatsResponse + if err := dec.Decode(&ds); err != nil { + return + } - var dockerStats container.StatsResponse - if err := json.NewDecoder(statsResp.Body).Decode(&dockerStats); err != nil { - return nil, fmt.Errorf("decoding stats response: %w", err) + r.update(&ds) } +} - stats := &Stats{ +// update caches a freshly decoded sample and signals readiness on the first one. +func (r *dockerReader) update(ds *container.StatsResponse) { + snapshot := &Stats{ // Memory usage in bytes. - Memory: dockerStats.MemoryStats.Usage, - // CPU usage: Docker reports in nanoseconds, convert to microseconds. - CPUUsage: dockerStats.CPUStats.CPUUsage.TotalUsage / 1000, + Memory: ds.MemoryStats.Usage, + // CPU usage: Docker reports nanoseconds, convert to microseconds. + CPUUsage: ds.CPUStats.CPUUsage.TotalUsage / 1000, } - // Sum disk I/O from BlkioStats. - stats.DiskRead, stats.DiskWrite = r.extractBlkioBytes(&dockerStats) - stats.DiskReadOps, stats.DiskWriteOps = r.extractBlkioOps(&dockerStats) + snapshot.DiskRead, snapshot.DiskWrite = extractBlkioBytes(ds) + snapshot.DiskReadOps, snapshot.DiskWriteOps = extractBlkioOps(ds) + + r.mu.Lock() + r.latest = snapshot + r.mu.Unlock() - return stats, nil + r.readyOnce.Do(func() { close(r.ready) }) +} + +// sleepCtx sleeps for d or until ctx is cancelled. It returns false if the +// context was cancelled (caller should stop). +func sleepCtx(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } } // extractBlkioBytes extracts read/write bytes from BlkioStats. -func (r *dockerReader) extractBlkioBytes(stats *container.StatsResponse) (readBytes, writeBytes uint64) { +func extractBlkioBytes(stats *container.StatsResponse) (readBytes, writeBytes uint64) { for _, entry := range stats.BlkioStats.IoServiceBytesRecursive { switch entry.Op { case "Read", "read": @@ -92,7 +220,7 @@ func (r *dockerReader) extractBlkioBytes(stats *container.StatsResponse) (readBy } // extractBlkioOps extracts read/write I/O operations from BlkioStats. -func (r *dockerReader) extractBlkioOps(stats *container.StatsResponse) (readOps, writeOps uint64) { +func extractBlkioOps(stats *container.StatsResponse) (readOps, writeOps uint64) { for _, entry := range stats.BlkioStats.IoServicedRecursive { switch entry.Op { case "Read", "read": diff --git a/pkg/stats/docker_reader_test.go b/pkg/stats/docker_reader_test.go new file mode 100644 index 000000000..e4b8c04a4 --- /dev/null +++ b/pkg/stats/docker_reader_test.go @@ -0,0 +1,107 @@ +package stats + +import ( + "sync" + "testing" + + "github.com/docker/docker/api/types/container" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestReader builds a dockerReader without starting the streaming goroutine, +// so the cache logic can be exercised without a Docker daemon. +func newTestReader() *dockerReader { + return &dockerReader{ + log: logrus.New(), + ready: make(chan struct{}), + } +} + +func TestDockerReader_ReadStatsBeforeFirstSample(t *testing.T) { + r := newTestReader() + + _, err := r.ReadStats() + require.Error(t, err, "ReadStats must error until the first sample is cached") +} + +func TestDockerReader_UpdateCachesSampleAndSignalsReady(t *testing.T) { + r := newTestReader() + + ds := &container.StatsResponse{} + ds.MemoryStats.Usage = 2048 + ds.CPUStats.CPUUsage.TotalUsage = 5000 // nanoseconds + ds.BlkioStats.IoServiceBytesRecursive = []container.BlkioStatEntry{ + {Op: "Read", Value: 100}, + {Op: "Write", Value: 200}, + } + ds.BlkioStats.IoServicedRecursive = []container.BlkioStatEntry{ + {Op: "Read", Value: 3}, + {Op: "Write", Value: 4}, + } + + r.update(ds) + + // readyOnce should have closed the ready channel. + select { + case <-r.ready: + default: + t.Fatal("ready channel was not closed after first update") + } + + got, err := r.ReadStats() + require.NoError(t, err) + assert.Equal(t, uint64(2048), got.Memory) + assert.Equal(t, uint64(5), got.CPUUsage, "nanoseconds should convert to microseconds") + assert.Equal(t, uint64(100), got.DiskRead) + assert.Equal(t, uint64(200), got.DiskWrite) + assert.Equal(t, uint64(3), got.DiskReadOps) + assert.Equal(t, uint64(4), got.DiskWriteOps) +} + +func TestDockerReader_ReadStatsReturnsCopy(t *testing.T) { + r := newTestReader() + + ds := &container.StatsResponse{} + ds.MemoryStats.Usage = 1000 + r.update(ds) + + first, err := r.ReadStats() + require.NoError(t, err) + + // Mutating the returned snapshot must not affect the cached sample. + first.Memory = 9999 + + second, err := r.ReadStats() + require.NoError(t, err) + assert.Equal(t, uint64(1000), second.Memory, "cached sample must be isolated from callers") +} + +func TestDockerReader_UpdateReadConcurrent(t *testing.T) { + r := newTestReader() + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for i := range 1000 { + ds := &container.StatsResponse{} + ds.MemoryStats.Usage = uint64(i) + r.update(ds) + } + }() + + go func() { + defer wg.Done() + + for range 1000 { + _, _ = r.ReadStats() + } + }() + + wg.Wait() +} From 4b6e2881454b3cdb3ccd9b2adb6d9547e1cdb7dc Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 18 Jun 2026 09:38:19 +0200 Subject: [PATCH 14/83] fix(examples): enable bootstrap_fcu and pin besu so besu passes besu booted from a state-actor snapshot stays in PoS initial-sync and answers SYNCING to every payload until it receives a bootstrap forkchoiceUpdated. - add runner.client.config.bootstrap_fcu.enabled (required for besu/reth/nethermind; geth is unaffected). - pin besu to hyperledger/besu:26.6.0: :latest resolves to 26.6.1, which regressed and won't accept the bootstrap FCU on a snapshot. 26.6.0 works. geth and besu now both pass 36/36 (osaka bn128) on the example. --- examples/configuration/config.state-actor-eest.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index e36567433..def44c078 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -78,6 +78,14 @@ runner: fixtures_subdir: blockchain_tests_stateful_engine client: config: + # Booting from a snapshot datadir leaves the Engine API forkchoice head + # un-established, so clients answer SYNCING to every newPayload. A bootstrap + # FCU to the snapshot head makes them canonical and ready to validate the + # replayed payloads (required for besu/reth/nethermind; geth is fine without). + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s genesis: nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json #geth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json @@ -116,4 +124,6 @@ runner: image: nethermind/nethermind:1.37.0 - id: besu client: besu - image: hyperledger/besu:latest + # Pinned: besu 26.6.1 regressed — it won't accept the bootstrap FCU on a + # snapshot (stays in PoS initial-sync → SYNCING). 26.6.0 works (36/36). + image: hyperledger/besu:26.6.0 From dfcc84868845e5a0ebbb471cf8ebeacebff41cb1 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 18 Jun 2026 11:10:39 +0200 Subject: [PATCH 15/83] fix(examples): point nethermind at the state-actor datadir via Init.BaseDbPath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmarkoor mounts the datadir at /data and passes nethermind --datadir=/data, but nethermind's Init.BaseDbPath then defaults to /data/nethermind_db//. So nethermind opened a fresh empty state DB there and ignored the state-actor snapshot written directly under /data/{state,blocks,headers,blockInfos}. It then built genesis from the empty-alloc chainspec (whose baked-in stateRoot makes the genesis hash match, masking the problem), leaving the state DB empty — so StateReader.HasStateForBlock(genesis) is false, NewPayloadHandler.ShouldProcessBlock returns false, and every replayed engine_newPayload answers SYNCING (the client beacon-syncs instead of executing). Passing --Init.BaseDbPath=/data makes nethermind read the snapshot DBs directly ("Detected HalfPath key scheme"). nethermind now passes 36/36 on the example, matching geth and besu. No state-actor change is required (state-actor's own oracle already sets BaseDbPath to the datadir). Scoped to the example's nethermind extra_args so the global client DefaultCommand still uses --datadir. reth remains unresolved (separate execution-state divergence, not a path issue). --- examples/configuration/config.state-actor-eest.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index def44c078..1db398e92 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -117,10 +117,15 @@ runner: - id: nethermind client: nethermind extra_args: - - --Sync.NetworkingEnabled=false - - --Sync.PivotNumber=0 - - --Sync.SynchronizationEnabled=false - #image: ethpandaops/nethermind:master + # benchmarkoor mounts the datadir at /data and the nethermind client + # passes --datadir=/data, but nethermind's BaseDbPath then defaults to + # /data/nethermind_db// — so it reads a fresh EMPTY state db and + # ignores the state-actor snapshot written directly under /data/{state, + # blocks,headers,blockInfos}. It then builds genesis from the empty-alloc + # chainspec, so HasStateForBlock(genesis) is false and every replayed + # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind + # read the snapshot directly (it then logs "Detected HalfPath key scheme"). + - --Init.BaseDbPath=/data image: nethermind/nethermind:1.37.0 - id: besu client: besu From 48fb7af13a63e1c1e25945cd23432714eff273a4 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 18 Jun 2026 13:32:57 +0200 Subject: [PATCH 16/83] feat(state-actor): support ethrex as a builder target benchmarkoor's client registry and state-actor both already support ethrex, but the state_actor builder's config validation rejected it as a target client, so it couldn't be used end-to-end. - allow "ethrex" in stateActorSupportedClients (pkg/config) + update the validation error message. - add ethrex to the state-actor EEST example: builder image ghcr.io/ethereum/state-actor-ethrex:main, a target + datadir, the ethrex-genesis.json sidecar under runner.client.config.genesis, and an instance (ghcr.io/lambdaclass/ethrex:latest) with --skip-genesis-validation (ethrex >= v16.0.0 is required for that flag, needed because the snapshot commits a synthetic state root the empty-alloc genesis can't reproduce). Like reth, ethrex has no RPC rollback, so the replay reorgs via the engine API. - document ethrex in docs/configuration.md. Verified: ethrex passes 36/36 on the example, matching geth, besu, nethermind and reth. --- docs/configuration.md | 4 ++-- .../configuration/config.state-actor-eest.yaml | 17 ++++++++++++++++- pkg/config/config.go | 3 ++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index aa9eb8fb4..335e92942 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1459,7 +1459,7 @@ The fields below mirror `builder.state_actor.config`; any field set here overrid | Option | Type | Default | Applies to | Description | |---|---|---|---|---| | `name` | string | `client` | all | Human-readable name. Used by `--target` to filter. Must be unique across targets; defaults to the `client` field when omitted. | -| `client` | string | – | all | One of `geth`, `reth`, `besu`, `nethermind`. State-actor does not support `erigon` or `nimbus`. | +| `client` | string | – | all | One of `geth`, `reth`, `besu`, `nethermind`, `ethrex`. State-actor does not support `erigon` or `nimbus`. | | `output_dir` | string | – | all | Absolute host path. If the directory already contains entries, that target is **skipped** (no error) — pass `--force` (CLI) or set `force: true` here to wipe and rebuild. For geth, state-actor writes into `/geth/chaindata`. | | `target_size` | string | from `config` | all | Advisory size budget for auto-generated state, e.g. `5GB`, `500MB` (base-1024). Required for the target when no spec is configured; when a spec is configured (top-level or default), `target_size` is optional and acts as a headroom budget that state-actor fills past the spec's projected cost. | | `force` | bool | `false` | all | Per-target override of the CLI `--force` flag: wipes `output_dir` before building so state-actor sees a clean directory. Useful when most targets should skip-if-built but specific ones should always rebuild. | @@ -1473,7 +1473,7 @@ The fields below mirror `builder.state_actor.config`; any field set here overrid | `binary_trie` | bool | from `config`, then `false` | geth | EIP-7864 binary trie. Set `false` to opt out of a global default. Rejected (after resolution) for non-geth. | | `group_depth` | int | from `config`, then `8` (state-actor) | geth + binary_trie | Binary-trie serialisation unit. Range 1..8. Requires effective `binary_trie=true`. | -State-actor itself only writes the genesis block; subsequent blocks come from running a client against the produced datadir. See [state-actor RUNBOOK.md](https://github.com/ethereum/state-actor/blob/main/docs/RUNBOOK.md) for the per-client boot recipes (e.g. geth needs `--db.engine=pebble`; reth needs `--debug.skip-genesis-validation`; besu needs `--data-storage-format=BONSAI`). +State-actor itself only writes the genesis block; subsequent blocks come from running a client against the produced datadir. See [state-actor RUNBOOK.md](https://github.com/ethereum/state-actor/blob/main/docs/RUNBOOK.md) for the per-client boot recipes (e.g. geth needs `--db.engine=pebble`; reth needs `--debug.skip-genesis-validation`; besu needs `--data-storage-format=BONSAI`; ethrex needs `--skip-genesis-validation` and ≥ v16.0.0). ### Running diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index 1db398e92..6e10bcd71 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -15,6 +15,7 @@ builder: reth: ghcr.io/ethereum/state-actor-reth:main besu: ghcr.io/ethereum/state-actor-besu:main nethermind: ghcr.io/ethereum/state-actor-nethermind:main + ethrex: ghcr.io/ethereum/state-actor-ethrex:main pull_policy: always config: seed: 1234 @@ -40,6 +41,8 @@ builder: output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind - client: besu output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + - client: ethrex + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex ## Stage 2: Building EEST stateful fixtures eest_payloads: @@ -91,6 +94,7 @@ runner: #geth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json datadirs: geth: @@ -105,6 +109,9 @@ runner: besu: source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu method: copy + ethrex: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex + method: copy instances: - id: geth @@ -126,9 +133,17 @@ runner: # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind # read the snapshot directly (it then logs "Detected HalfPath key scheme"). - --Init.BaseDbPath=/data - image: nethermind/nethermind:1.37.0 - id: besu client: besu # Pinned: besu 26.6.1 regressed — it won't accept the bootstrap FCU on a # snapshot (stays in PoS initial-sync → SYNCING). 26.6.0 works (36/36). image: hyperledger/besu:26.6.0 + - id: ethrex + client: ethrex + # ethrex >= v16.0.0 is required: it's the first release with + # --skip-genesis-validation (lambdaclass/ethrex#6783), needed because the + # state-actor snapshot commits a synthetic state root that the emitted + # ethrex-genesis.json (empty alloc) can't reproduce. + image: ghcr.io/lambdaclass/ethrex:latest + extra_args: + - --skip-genesis-validation diff --git a/pkg/config/config.go b/pkg/config/config.go index 247811c2c..64288316d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -267,6 +267,7 @@ var stateActorSupportedClients = map[string]struct{}{ "reth": {}, "besu": {}, "nethermind": {}, + "ethrex": {}, } // stateActorValidPullPolicies mirrors the pull-policy vocabulary used by @@ -1889,7 +1890,7 @@ func (c *Config) validateStateActor() error { if _, ok := stateActorSupportedClients[t.Client]; !ok { return fmt.Errorf( "%s.client: %q is not supported by state-actor "+ - "(must be geth, reth, besu, or nethermind)", + "(must be geth, reth, besu, nethermind, or ethrex)", prefix, t.Client, ) } From d728f503f5d154ff185046cde1d3d038cbd4d187 Mon Sep 17 00:00:00 2001 From: CPerezz <37264926+CPerezz@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:36:41 +0200 Subject: [PATCH 17/83] fix(examples): enable p2p on besu so it accepts the bootstrap FCU (#249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem With besu in `config.state-actor-eest.yaml`, the bootstrap `engine_forkchoiceUpdatedV3` is rejected with `SYNCING` — benchmarkoor retries then aborts before any payload runs. geth, reth, and nethermind are unaffected. ## Root cause besu returns `SYNCING` from `forkchoiceUpdated` while `mergeContext.isSyncing()` is true. On an isolated snapshot node that hinges on `SyncState.reachedTerminalDifficulty`, which is set only when besu's **synchronizer** runs (`DefaultSynchronizer`). This config passes `--p2p-enabled=false`, which suppresses the synchronizer, so the flag is never set and besu treats the (post-merge) snapshot head as pre-merge → `SYNCING`. (besu ≤ 26.6.0 masked this: its `isSyncing()` also required `!isInSync()`, and `isInSync()` is true here, so it short-circuited to `VALID`. 26.6.1 returns `SYNCING` on the unset terminal-difficulty flag alone.) ## Fix Enable p2p on the besu instance via `extra_args`: ```yaml - id: besu image: hyperledger/besu:26.6.1 extra_args: - --p2p-enabled=true ``` `--max-peers=0` + `--discovery-enabled=false` (besu client defaults) keep the node isolated — p2p initializes but no real peers connect. Also bumps besu to the latest release (26.6.1). ## Verification - besu **26.6.1** + `--p2p-enabled=true`: bootstrap FCU `VALID`, **36/36** (`benchmark/compute`, bn128). - besu **26.6.0** + `--p2p-enabled=true`: **36/36** (no regression). Also documents the requirement in `docs/configuration.md` (Bootstrap FCU section). --- _typos.toml | 1 + docs/configuration.md | 2 ++ examples/configuration/config.state-actor-eest.yaml | 10 +++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/_typos.toml b/_typos.toml index f250b3b86..32b561106 100644 --- a/_typos.toml +++ b/_typos.toml @@ -4,3 +4,4 @@ extend-exclude = ["go.mod"] [default.extend-words] PANC = "PANC" ERRO = "ERRO" +BULD = "BULD" diff --git a/docs/configuration.md b/docs/configuration.md index 335e92942..34260cab8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -945,6 +945,8 @@ When both `retry_new_payloads_syncing_state` and `retry_new_payloads_failed_stat Some clients (e.g., Erigon) may still be performing internal initialization or syncing after their RPC endpoint becomes available. The `bootstrap_fcu` option sends an `engine_forkchoiceUpdatedV3` call in a retry loop after RPC is ready, using the latest block hash from `eth_getBlockByNumber("latest")`. The client accepting the FCU with `VALID` status confirms it has finished syncing and is ready for test execution. +> **Besu** accepts the bootstrap FCU on an isolated snapshot node only with `--p2p-enabled=true`: its synchronizer must run to register the post-merge head as in-sync, otherwise besu answers `SYNCING` to every FCU. Set `extra_args: [--p2p-enabled=true]` on the besu instance (`--max-peers=0` + `--discovery-enabled=false` keep it isolated, with zero real peers). + **Shorthand** (uses defaults: `max_retries: 30`, `backoff: 1s`): ```yaml diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index 6e10bcd71..cb19e4068 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -135,9 +135,13 @@ runner: - --Init.BaseDbPath=/data - id: besu client: besu - # Pinned: besu 26.6.1 regressed — it won't accept the bootstrap FCU on a - # snapshot (stays in PoS initial-sync → SYNCING). 26.6.0 works (36/36). - image: hyperledger/besu:26.6.0 + # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: + # its synchronizer must run to register the post-merge head as in-sync, + # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false + # (the client defaults) keep it isolated — no real peers. + image: hyperledger/besu:26.6.1 + extra_args: + - --p2p-enabled=true - id: ethrex client: ethrex # ethrex >= v16.0.0 is required: it's the first release with From 233284925f8a9545e0c02a5e5b66dfc069f044d5 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Mon, 22 Jun 2026 20:01:23 +0200 Subject: [PATCH 18/83] =?UTF-8?q?feat(builder):=20use=20distinct=20?= =?UTF-8?q?=F0=9F=9F=A0=20prefix=20for=20BULD=20container=20logs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EL-client and build-tool container output both streamed with the same 🟣 prefix, making them hard to tell apart when the filler client and fill-stateful/state-actor stream side by side. Keep 🟣 for CLIE (matches benchmarkoor run client logs) and give BULD its own 🟠. --- pkg/builder/util.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/builder/util.go b/pkg/builder/util.go index 9914b2c9b..fd1b99a11 100644 --- a/pkg/builder/util.go +++ b/pkg/builder/util.go @@ -81,13 +81,23 @@ func randSuffix() (string, error) { } // containerStream returns an io.Writer that prefixes each line of streamed -// container output with "🟣 $TS $label | $name | " and writes it directly to -// stdout. This matches the client-log format `benchmarkoor run` uses (see -// pkg/runner clientLogPrefix) so build output looks consistent and carries a -// clear leading tag identifying the source. label is a short tag (e.g. "CLIE" -// for an EL client, "BULD" for a build-tool container). +// container output with "$emoji $TS $label | $name | " and writes it directly +// to stdout. EL-client output ("CLIE") uses 🟣 to match the client-log format +// `benchmarkoor run` uses (see pkg/runner clientLogPrefix); build-tool +// containers ("BULD", e.g. state-actor / fill-stateful) use 🟠 so they are easy +// to distinguish from the filler client streaming alongside them. func containerStream(label, name string) io.Writer { - return &containerStreamWriter{label: label, name: name, w: os.Stdout} + return &containerStreamWriter{emoji: streamEmoji(label), label: label, name: name, w: os.Stdout} +} + +// streamEmoji picks the leading emoji for a streamed-container line based on its +// label. Build-tool output gets 🟠; everything else (EL clients) keeps 🟣. +func streamEmoji(label string) string { + if label == "BULD" { + return "🟠" + } + + return "🟣" } // ansiReset clears any ANSI color/style left set by a streamed line. Tools like @@ -97,6 +107,7 @@ func containerStream(label, name string) io.Writer { const ansiReset = "\x1b[0m" type containerStreamWriter struct { + emoji string label string name string w io.Writer @@ -116,7 +127,7 @@ func (w *containerStreamWriter) Write(p []byte) (int, error) { ts := time.Now().UTC().Format(config.LogTimestampFormat) msg := bytes.TrimRight(line, "\r\n") - if _, err := fmt.Fprintf(w.w, "🟣 %s %s | %s | %s%s\n", ts, w.label, w.name, msg, ansiReset); err != nil { + if _, err := fmt.Fprintf(w.w, "%s %s %s | %s | %s%s\n", w.emoji, ts, w.label, w.name, msg, ansiReset); err != nil { return len(p), err } } From 77703caaba7bda1767f84f558bcd808b7817e7aa Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 23 Jun 2026 09:33:38 +0200 Subject: [PATCH 19/83] feat(builder): support erigon as a state_actor client Add erigon to stateActorSupportedClients so the builder accepts it, update the validation error message and supported-clients comment, and wire it through the config.state-actor-eest.yaml example (state_actor image + target, plus runner genesis/datadir/instance). dbPath already handles erigon correctly: only geth needs the /geth/chaindata suffix; erigon takes the datadir root like the other non-geth clients. NOTE: state-actor does not publish a state-actor-erigon image yet, so the build/run path is wired but untested end-to-end against a real datadir. --- .../config.state-actor-eest.yaml | 19 +++++++++++++++++++ pkg/config/config.go | 7 ++++--- pkg/config/config_test.go | 6 +++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index cb19e4068..14c811829 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -16,6 +16,7 @@ builder: besu: ghcr.io/ethereum/state-actor-besu:main nethermind: ghcr.io/ethereum/state-actor-nethermind:main ethrex: ghcr.io/ethereum/state-actor-ethrex:main + erigon: ghcr.io/ethereum/state-actor-erigon:main pull_policy: always config: seed: 1234 @@ -43,6 +44,8 @@ builder: output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu - client: ethrex output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex + - client: erigon + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon ## Stage 2: Building EEST stateful fixtures eest_payloads: @@ -95,6 +98,11 @@ runner: reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json + # erigon consumes a geth-format genesis via its init container. Filename + # follows the per-client state-actor convention (geth-genesis.json, + # ethrex-genesis.json); confirm/adjust once state-actor publishes the + # erigon image. + erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/erigon-genesis.json datadirs: geth: @@ -112,6 +120,9 @@ runner: ethrex: source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex method: copy + erigon: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon + method: copy instances: - id: geth @@ -151,3 +162,11 @@ runner: image: ghcr.io/lambdaclass/ethrex:latest extra_args: - --skip-genesis-validation + - id: erigon + client: erigon + # NOTE: untested end-to-end — state-actor does not publish an erigon image + # yet, so the stage-1 datadir can't be produced. erigon boots a snapshot via + # an init container (`erigon init --datadir=/data /tmp/genesis.json`); like + # besu/nethermind this may need snapshot-boot tuning (extra_args / a + # bootstrap FCU) once a real datadir exists to test against. + image: erigontech/erigon:latest diff --git a/pkg/config/config.go b/pkg/config/config.go index 64288316d..6ae74d03e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -260,14 +260,15 @@ func (s *StateActorConfig) ImageFor(client string) string { } // stateActorSupportedClients lists the clients state-actor itself can -// materialise datadirs for. Erigon and Nimbus are intentionally absent -// (state-actor does not implement writers for them). +// materialise datadirs for. Nimbus is intentionally absent (state-actor +// does not implement a writer for it). var stateActorSupportedClients = map[string]struct{}{ "geth": {}, "reth": {}, "besu": {}, "nethermind": {}, "ethrex": {}, + "erigon": {}, } // stateActorValidPullPolicies mirrors the pull-policy vocabulary used by @@ -1890,7 +1891,7 @@ func (c *Config) validateStateActor() error { if _, ok := stateActorSupportedClients[t.Client]; !ok { return fmt.Errorf( "%s.client: %q is not supported by state-actor "+ - "(must be geth, reth, besu, nethermind, or ethrex)", + "(must be geth, reth, besu, nethermind, ethrex, or erigon)", prefix, t.Client, ) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index ba07c6aed..171bb983f 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3232,6 +3232,8 @@ func TestValidateBuilder(t *testing.T) { "reth": "ghcr.io/ethereum/state-actor-reth:latest", "besu": "ghcr.io/ethereum/state-actor-besu:latest", "nethermind": "ghcr.io/ethereum/state-actor-nethermind:latest", + "ethrex": "ghcr.io/ethereum/state-actor-ethrex:latest", + "erigon": "ghcr.io/ethereum/state-actor-erigon:latest", } // mkCfg builds a Config with just the builder block populated so the @@ -3286,13 +3288,11 @@ func TestValidateBuilder(t *testing.T) { }, }, { - name: "unsupported client erigon", + name: "supported client erigon", sa: &StateActorConfig{ Images: allImages, Targets: []StateActorTarget{{Client: "erigon", OutputDir: dirA, TargetSize: "5GB"}}, }, - wantErr: true, - errSubstr: "erigon", }, { name: "unsupported client nimbus", From 907dc48d0bf5dc0cb62d28fd03a492221b28d9a5 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 23 Jun 2026 09:48:59 +0200 Subject: [PATCH 20/83] fix(examples): run erigon daemon on the bal-devnet-7 build for state-actor snapshots state-actor pins erigon at the bal-devnet-7 commit and bakes that binary into its image; its `erigon init` encodes the MDBX chain config that erigon's way (chainId as a JSON string). Booting the snapshot with stable erigon (erigontech/erigon:latest, 3.4.x) panics with "cannot unmarshal \"1337\" into a *big.Int". Point the erigon instance at ethpandaops/erigon:bal-devnet-7, which reads the config correctly and supports the amsterdam/BAL (EIP-7928) fork these benchmarks target. Verified end-to-end: build (state-actor-erigon:main) + run replay 180 engine_newPayloadV4 / forkchoiceUpdatedV3 calls with no SYNCING/panic. --- .../configuration/config.state-actor-eest.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index 14c811829..9e0c7fe86 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -102,7 +102,8 @@ runner: # follows the per-client state-actor convention (geth-genesis.json, # ethrex-genesis.json); confirm/adjust once state-actor publishes the # erigon image. - erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/erigon-genesis.json + erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/chainspec.json + #erigon: /tmp/benchmarkoor/state-actor/geth/geth-genesis.json datadirs: geth: @@ -164,9 +165,12 @@ runner: - --skip-genesis-validation - id: erigon client: erigon - # NOTE: untested end-to-end — state-actor does not publish an erigon image - # yet, so the stage-1 datadir can't be produced. erigon boots a snapshot via - # an init container (`erigon init --datadir=/data /tmp/genesis.json`); like - # besu/nethermind this may need snapshot-boot tuning (extra_args / a - # bootstrap FCU) once a real datadir exists to test against. - image: erigontech/erigon:latest + # MUST match the erigon that state-actor pins for the snapshot + # (Dockerfile.erigon builds erigon at the bal-devnet-7 commit and notes the + # daemon must use that exact binary). state-actor's `erigon init` encodes the + # MDBX chain config the bal-devnet-7 way (e.g. chainId as a JSON string); + # stable erigon (erigontech/erigon:latest, 3.4.x) panics reading it back with + # "cannot unmarshal \"1337\" into a *big.Int". ethpandaops/erigon:bal-devnet-7 + # reads it correctly and supports the amsterdam/BAL (EIP-7928) fork these + # benchmarks target. + image: ethpandaops/erigon:bal-devnet-7 From 2ab6fcb7eb564a020fdb3688567031b4f5ae85e2 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 23 Jun 2026 11:25:00 +0200 Subject: [PATCH 21/83] feat(builder): plumb besu/nethermind as eest_payloads fillers (gated, geth still the only working one) fill-stateful forces use_testing_build_block=True, so a filler must implement the testing_buildBlockV1 RPC plus debug_setHead (the per-test chain rewind) and return EEST-compatible receipts. Add per-client filler boot commands (fillerCommand dispatcher + besu/nethermind argv) carrying the same snapshot-boot workarounds the runner uses (besu --p2p-enabled, nethermind --Init.BaseDbPath), the testing namespace each needs (besu TESTING, nethermind Testing module), and a non-zero session-tip pin for besu (its eth_maxPriorityFeePerGas is 0 on a fresh snapshot). Move filler_image onto each target so targets can fill with different clients/images. besu and nethermind are accepted by validation and fully plumbed, but both are blocked upstream today, so their example targets are commented out: - besu: testing_buildBlockV1 (TESTING, besu-eth/besu#9838) + debug_setHead + session fees all work, but its self-built block fails its own engine_newPayloadV4 with a World-State-Root mismatch. - nethermind: testing_buildBlockV1 works via the Testing module + special image, but debug_setHead is unimplemented and its receipt trips EEST's strict TransactionReceipt model. geth remains the only filler that produces fixtures end-to-end. --- .../config.state-actor-eest.yaml | 35 ++++- pkg/builder/eest_payloads.go | 129 +++++++++++++++++- pkg/builder/eest_payloads_test.go | 41 ++++++ pkg/config/config.go | 26 +++- pkg/config/config_test.go | 20 ++- 5 files changed, 238 insertions(+), 13 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index 9e0c7fe86..81d06b91f 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -56,18 +56,51 @@ builder: eest_repo: https://github.com/ethereum/execution-specs.git eest_ref: forks/amsterdam config: - filler_image: ethpandaops/geth:master + # filler_image is set per-target below so each target can fill with a + # different client. Shared knobs stay here and are hoisted into every target. fork: osaka gas_benchmark_values: [10] # Millions of gas to parametrise against datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) + # fill-stateful forces testing_buildBlockV1 + debug_setHead, so a filler must + # implement both. geth (ethpandaops/geth) does. besu/nethermind are plumbed + # in benchmarkoor (see fillerCommand in pkg/builder) but blocked upstream, so + # their targets are commented out below — uncomment to experiment once the + # upstream gaps close. targets: - name: payload-generator-geth filler_client: geth + filler_image: ethpandaops/geth:master source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures} tests: - tests/benchmark/compute # pytest paths inside the fill image filter: bn128 # Quick subset for fast end-to-end iteration: only the bn128 compute tests. + # besu: testing_buildBlockV1 (TESTING namespace, besu-eth/besu#9838), + # debug_setHead and the session-fee pin all work, but besu's self-built + # block fails its own engine_newPayloadV4 with a World-State-Root mismatch + # (upstream besu bug in testing_buildBlockV1). Re-enable when fixed. + # - name: payload-generator-besu + # filler_client: besu + # filler_image: hyperledger/besu:26.6.1 # >= 26.6 ships testing_buildBlockV1 + # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-besu + # tests: + # - tests/benchmark/compute + # filter: bn128 + # nethermind: testing_buildBlockV1 works (Testing module in the special + # nethermindeth/nethermind:testing_build_block_with_opcode_tracing image), + # but debug_setHead is unimplemented (the per-test rewind aborts the run) + # and its receipt trips EEST's strict model. Re-enable when fixed. + # - name: payload-generator-nethermind + # filler_client: nethermind + # filler_image: nethermindeth/nethermind:testing_build_block_with_opcode_tracing + # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-nethermind + # tests: + # - tests/benchmark/compute + # filter: bn128 ## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads runner: diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index 839a6e5c6..c68aaf152 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -47,6 +47,13 @@ const ( // minerGasLimit is the huge gas limit the filler geth is started with so // benchmark blocks of any size can be built (mirrors the fill-stateful docs). minerGasLimit = "1000000000000" + + // besuDefaultPriorityFeeWei pins fill-stateful's session tip for the besu + // filler. besu's eth_maxPriorityFeePerGas returns 0 on a freshly-booted + // snapshot (no fee history), which fill-stateful rejects with "requires the + // backend to carry non-zero session fees". geth suggests a non-zero tip on + // its own, so this is only needed for besu. 1 gwei. + besuDefaultPriorityFeeWei = "1000000000" ) // EESTPayloadsBuilder generates stateful EEST benchmark fixtures. Per target @@ -351,7 +358,7 @@ func (b *EESTPayloadsBuilder) startFiller( return "", "", nil, fmt.Errorf("generating container name suffix: %w", err) } - cmd := fillerGethCommand(t, spec) + cmd := fillerCommand(t, spec) containerSpec := &docker.ContainerSpec{ Name: fmt.Sprintf("benchmarkoor-build-eest-filler-%s-%s", t.FillerClient, suffix), @@ -552,11 +559,28 @@ func (b *EESTPayloadsBuilder) labels(t *config.EESTPayloadTarget) map[string]str } } -// fillerGethCommand builds the geth argv for the filler client. Only geth is -// supported today (validated in config), so the namespaces and flags are -// geth-specific: the http API exposes the testing/engine/miner namespaces -// fill-stateful needs, archive gcmode keeps full state, and peering is -// disabled. spec supplies the in-container paths and ports. +// fillerCommand builds the filler EL client's argv, dispatching on the +// configured filler_client. fill-stateful drives whichever client over the +// standard Engine API flow (eth_sendRawTransaction + engine_forkchoiceUpdated / +// getPayload / newPayload), so every client just needs HTTP RPC (with txpool), +// the Engine API (JWT), peering disabled, and the snapshot-boot workarounds the +// runner uses for the same client. config validation guarantees the client is +// one of the cases below. +func fillerCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + switch t.FillerClient { + case "besu": + return fillerBesuCommand(t, spec) + case "nethermind": + return fillerNethermindCommand(t, spec) + default: + return fillerGethCommand(t, spec) + } +} + +// fillerGethCommand builds the geth argv for the filler client: the http API +// exposes the eth/net/web3/txpool/engine namespaces fill-stateful needs, archive +// gcmode keeps full state, and peering is disabled. spec supplies the +// in-container paths and ports. func fillerGethCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { args := []string{ "--config=/tmp/config.toml", @@ -589,6 +613,92 @@ func fillerGethCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { return append(args, t.FillerExtraArgs...) } +// fillerBesuCommand builds the besu argv for the filler client. fill-stateful +// drives every filler over testing_buildBlockV1, which besu exposes behind the +// TESTING JSON-RPC namespace (added in besu-eth/besu#9838, in besu >= 26.6; +// without TESTING in --rpc-http-api besu answers -32604 "Method not enabled"). +// Mirrors the runner besu command (pkg/client/besu.go) otherwise: the ETH/TXPOOL +// namespaces back eth_sendRawTransaction, DEBUG backs the per-test debug_setHead +// rewind, and --p2p-enabled=true lets besu's synchronizer register the snapshot +// head as in-sync — without it besu answers SYNCING to the fill tool's initial +// forkchoice_updated (--max-peers=0 + --discovery-enabled=false keep it +// isolated). The genesis file is required: besu reads chainId from +// --genesis-file at boot, not from the datadir. +func fillerBesuCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + args := []string{ + "--data-path=" + spec.DataDir(), + "--data-storage-format=BONSAI", + // Trust the genesis state hash baked into the state-actor snapshot + // instead of recomputing it from the empty chainspec alloc. + "--genesis-state-hash-cache-enabled=true", + "--sync-mode=FULL", + "--p2p-enabled=true", + "--max-peers=0", + "--discovery-enabled=false", + "--rpc-http-enabled=true", + "--rpc-http-host=0.0.0.0", + "--rpc-http-port=" + strconv.Itoa(spec.RPCPort()), + // TESTING exposes testing_buildBlockV1 (fill-stateful's block builder). + "--rpc-http-api=ETH,NET,WEB3,TXPOOL,DEBUG,ADMIN,MINER,TESTING", + "--rpc-http-cors-origins=*", + "--host-allowlist=*", + "--Xhttp-timeout-seconds=660", + "--engine-rpc-enabled=true", + "--engine-jwt-secret=" + spec.JWTPath(), + "--engine-rpc-port=" + strconv.Itoa(spec.EnginePort()), + "--engine-host-allowlist=*", + "--target-gas-limit=" + minerGasLimit, + } + + if t.GenesisFile != "" { + args = append(args, spec.GenesisFlag()+spec.GenesisPath()) + } + + return append(args, t.FillerExtraArgs...) +} + +// fillerNethermindCommand builds the nethermind argv for the filler client. +// fill-stateful drives every filler over testing_buildBlockV1, so the HTTP RPC +// must expose the Testing module (without it nethermind answers -32604 "Method +// not enabled"); this needs a nethermind build that ships testing_buildBlockV1 +// (e.g. nethermindeth/nethermind:testing_build_block_with_opcode_tracing — set +// as the target's filler_image). Module list, target gas limit and BaseDbPath +// mirror NethermindEth/gas-benchmarks' stateful generator. --Init.BaseDbPath +// points at the datadir so nethermind reads the state-actor snapshot written +// there (its BaseDbPath otherwise defaults to a nested per-network dir holding +// an empty state db). The genesis file is required: nethermind reads the chain +// config from --Init.ChainSpecPath. +func fillerNethermindCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { + args := []string{ + "--datadir=" + spec.DataDir(), + "--Init.BaseDbPath=" + spec.DataDir(), + "--config=none", + "--Network.DiscoveryPort=0", + "--Network.MaxActivePeers=0", + "--Init.DiscoveryEnabled=false", + "--Sync.MaxAttemptsToUpdatePivot=0", + "--Network.ExternalIp=127.0.0.1", + "--JsonRpc.Enabled=true", + "--JsonRpc.Host=0.0.0.0", + "--JsonRpc.Port=" + strconv.Itoa(spec.RPCPort()), + // Testing exposes testing_buildBlockV1 (fill-stateful's block builder). + "--JsonRpc.EnabledModules=Eth,Net,Web3,Admin,Debug,Trace,TxPool,Subscribe,Testing", + "--JsonRpc.EngineEnabledModules=Net,Eth,Subscribe,Web3,Testing,Engine", + "--JsonRpc.Timeout=600000", + "--JsonRpc.JwtSecretFile=" + spec.JWTPath(), + "--JsonRpc.EngineHost=0.0.0.0", + "--JsonRpc.EnginePort=" + strconv.Itoa(spec.EnginePort()), + "--Merge.TerminalTotalDifficulty=0", + "--Blocks.TargetBlockGasLimit=" + minerGasLimit, + } + + if t.GenesisFile != "" { + args = append(args, spec.GenesisFlag()+spec.GenesisPath()) + } + + return append(args, t.FillerExtraArgs...) +} + // buildFillArgs assembles the fill-stateful argv: the configured command // prefix, the live-client endpoints, the run knobs, and the test selection. func buildFillArgs( @@ -652,6 +762,13 @@ func buildFillArgs( args = append(args, "--rpc-seed-key="+t.RPCSeedKey) } + // besu suggests a zero priority fee on a freshly-booted snapshot; pin a + // non-zero tip so fill-stateful's session-fee check passes (see + // besuDefaultPriorityFeeWei). geth derives a non-zero tip itself. + if t.FillerClient == "besu" { + args = append(args, "--default-max-priority-fee-per-gas="+besuDefaultPriorityFeeWei) + } + if t.AddressStubsFile != "" { args = append(args, "--address-stubs="+fillStubsPath) } diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go index 73ca31050..1ba9873f1 100644 --- a/pkg/builder/eest_payloads_test.go +++ b/pkg/builder/eest_payloads_test.go @@ -148,6 +148,47 @@ func TestFillerGethCommand(t *testing.T) { }) } +func TestFillerCommand_Besu(t *testing.T) { + spec := client.NewBesuSpec() + + cmd := fillerCommand(&config.EESTPayloadTarget{ + FillerClient: "besu", + GenesisFile: "/host/besu-chainspec.json", + FillerExtraArgs: []string{"--logging=DEBUG"}, + }, spec) + + assert.Contains(t, cmd, "--data-path=/data") + // TESTING exposes testing_buildBlockV1 (required by fill-stateful). + assert.Contains(t, cmd, "--rpc-http-api=ETH,NET,WEB3,TXPOOL,DEBUG,ADMIN,MINER,TESTING") + assert.Contains(t, cmd, "--engine-jwt-secret=/tmp/jwtsecret") + assert.Contains(t, cmd, "--engine-rpc-port=8551") + assert.Contains(t, cmd, "--rpc-http-port=8545") + // besu boots a snapshot only when its synchronizer can register the head. + assert.Contains(t, cmd, "--p2p-enabled=true") + // besu reads chainId from the genesis file, so it must be passed. + assert.Contains(t, cmd, "--genesis-file=/tmp/genesis.json") + assert.Equal(t, "--logging=DEBUG", cmd[len(cmd)-1], "extra args are appended last") +} + +func TestFillerCommand_Nethermind(t *testing.T) { + spec := client.NewNethermindSpec() + + cmd := fillerCommand(&config.EESTPayloadTarget{ + FillerClient: "nethermind", + GenesisFile: "/host/parity-chainspec.json", + }, spec) + + assert.Contains(t, cmd, "--datadir=/data") + // Point BaseDbPath at the datadir so nethermind reads the state-actor snapshot. + assert.Contains(t, cmd, "--Init.BaseDbPath=/data") + // Testing module exposes testing_buildBlockV1 (required by fill-stateful). + assert.Contains(t, cmd, "--JsonRpc.EnabledModules=Eth,Net,Web3,Admin,Debug,Trace,TxPool,Subscribe,Testing") + assert.Contains(t, cmd, "--JsonRpc.JwtSecretFile=/tmp/jwtsecret") + assert.Contains(t, cmd, "--JsonRpc.EnginePort=8551") + assert.Contains(t, cmd, "--JsonRpc.Port=8545") + assert.Contains(t, cmd, "--Init.ChainSpecPath=/tmp/genesis.json") +} + func TestEESTPayloadsBuilder_Targets(t *testing.T) { cfg := &config.EESTPayloadsConfig{ FillImage: "fill:latest", diff --git a/pkg/config/config.go b/pkg/config/config.go index 6ae74d03e..99daacc85 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -482,11 +482,27 @@ func (t *EESTPayloadTarget) EffectiveName() string { return t.FillerClient } -// eestFillerSupportedClients lists the clients that can act as the -// fill-stateful filler. Only geth implements testing_buildBlockV1 today -// (ethpandaops/geth:master is the production-ready filler image). +// eestFillerSupportedClients lists the clients benchmarkoor knows how to boot as +// the fill-stateful filler (see fillerCommand in pkg/builder). fill-stateful +// forces use_testing_build_block=True, so a filler MUST implement the +// testing_buildBlockV1 RPC and debug_setHead (the per-test chain rewind). +// +// Status as of this writing: +// - geth: fully works (ethpandaops/geth implements both). +// - besu: plumbed but blocked upstream — testing_buildBlockV1 (TESTING +// namespace, besu-eth/besu#9838) + debug_setHead both respond, but +// besu's self-built block fails its own engine_newPayloadV4 with a +// World-State-Root mismatch. Kept as scaffolding. +// - nethermind: plumbed but blocked upstream — testing_buildBlockV1 works +// (Testing module, special image) but debug_setHead is unimplemented and +// its receipt trips EEST's strict model. Kept as scaffolding. +// +// besu/nethermind stay listed so configs can experiment with them once the +// upstream gaps close; their example targets are commented out. var eestFillerSupportedClients = map[string]struct{}{ - "geth": {}, + "geth": {}, + "besu": {}, + "nethermind": {}, } // RunnerConfig contains all run-specific configuration settings. @@ -2031,7 +2047,7 @@ func (c *Config) validateEESTPayloads() error { if _, ok := eestFillerSupportedClients[t.FillerClient]; !ok { return fmt.Errorf( "%s.filler_client: %q cannot act as the fill-stateful filler "+ - "(only geth implements testing_buildBlockV1 today)", + "(supported: geth, besu, nethermind)", prefix, t.FillerClient, ) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 171bb983f..9ba0e72c3 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3768,7 +3768,25 @@ func TestValidateEESTPayloads(t *testing.T) { return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} }(), wantErr: true, - errSubstr: "testing_buildBlockV1", + errSubstr: "cannot act as the fill-stateful filler", + }, + { + name: "besu filler client is supported", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FillerClient = "besu" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), + }, + { + name: "nethermind filler client is supported", + ep: func() *EESTPayloadsConfig { + tgt := base(dirA) + tgt.FillerClient = "nethermind" + + return &EESTPayloadsConfig{FillImage: "fill:latest", Targets: []EESTPayloadTarget{tgt}} + }(), }, { name: "missing tests", From 14e800a8c127ada62f61c7d79aeb90e59d8cde3b Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 23 Jun 2026 19:16:20 +0200 Subject: [PATCH 22/83] feat(builder): schedule a later fork on the filler via fork_activation_genesis Add a fork_activation_genesis field to eest_payloads targets so a snapshot built at one fork (e.g. state-actor's osaka) can fill fixtures for the next fork (e.g. amsterdam). The builder reads the snapshot genesis block's timestamp and boots the geth filler with --override.=, so the fork activates on the first block the filler builds (block N+1) while block 0 stays on the prior fork. --override.genesis can't be used here: state-actor bakes the snapshot state into the DB under an empty-alloc genesis, so geth recomputes an empty-state genesis block and rejects the hash mismatch. The per-fork --override. flag amends only the in-memory chain config, leaving the genesis block untouched. This needs a geth build that registers the flag (e.g. ethpandaops/geth:bal-devnet-7-amsterdam-override for amsterdam). Adds the example config.state-actor-eest.amsterdam.yaml wiring an osaka snapshot to an amsterdam fill. --- .../config.state-actor-eest.amsterdam.yaml | 223 ++++++++++++++++++ pkg/builder/eest_payloads.go | 23 ++ pkg/builder/genesis.go | 90 +++++++ pkg/builder/genesis_test.go | 69 ++++++ pkg/config/config.go | 46 +++- 5 files changed, 442 insertions(+), 9 deletions(-) create mode 100644 examples/configuration/config.state-actor-eest.amsterdam.yaml create mode 100644 pkg/builder/genesis.go create mode 100644 pkg/builder/genesis_test.go diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml new file mode 100644 index 000000000..b08c0f439 --- /dev/null +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -0,0 +1,223 @@ +# Building data directories and EEST payloads — AMSTERDAM fill. +# +# state-actor still generates OSAKA snapshot datadirs (stage 1); the eest_payloads +# stage then fills AMSTERDAM fixtures by booting the filler on the osaka snapshot +# and scheduling amsterdam to activate at the snapshot block's timestamp + 1 (via +# fork_activation_genesis → a geth --override.amsterdam= flag). So block 0 +# stays osaka and the first benchmark block (block 1) is amsterdam. +# +# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.amsterdam.yaml --force +# +# Running benchmark using geth: +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=geth +# +global: + log_level: info + +builder: + ## Stage 1: Building datadirs using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + reth: ghcr.io/ethereum/state-actor-reth:main + besu: ghcr.io/ethereum/state-actor-besu:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + ethrex: ghcr.io/ethereum/state-actor-ethrex:main + erigon: ghcr.io/ethereum/state-actor-erigon:main + pull_policy: always + config: + seed: 1234 + fork: osaka + chain: 1337 + gas_limit: 300000000 # 300M + target_size: 256MB + #target_size: 20GB + spec: | + entities: + - kind: eoa + name: bloated-eoa-2g + balance: "1000000000000000000" + nonce: 0 + code: "0xef01003333333333333333333333333333333333333333" + approximate_size_bytes: 2_000_000_000 + targets: + - client: geth + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + - client: reth + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth + - client: nethermind + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + - client: besu + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + - client: ethrex + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex + - client: erigon + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon + + ## Stage 2: Building EEST stateful fixtures + eest_payloads: + # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead + # of pulling a pre-built one. Alternatively set fill_image: . + fill_dockerfile: Dockerfile.eest-filler + pull_policy: if-not-present + eest_repo: https://github.com/ethereum/execution-specs.git + eest_ref: forks/amsterdam + config: + # filler_image is set per-target below so each target can fill with a + # different client. Shared knobs stay here and are hoisted into every target. + fork: amsterdam # fill amsterdam fixtures (snapshot stays osaka) + gas_benchmark_values: [10] # Millions of gas to parametrise against + datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) + # fill-stateful forces testing_buildBlockV1 + debug_setHead, so a filler must + # implement both. geth (ethpandaops/geth) does. besu/nethermind are plumbed + # in benchmarkoor (see fillerCommand in pkg/builder) but blocked upstream, so + # their targets are commented out below — uncomment to experiment once the + # upstream gaps close. + # + # amsterdam fill: filler_image must be a geth that registers --override.amsterdam + # (bal-devnet-7 + the amsterdam-override patch). fork_activation_genesis reads + # the snapshot block's timestamp and the builder boots the filler with + # --override.amsterdam=. + targets: + - name: payload-generator-geth + filler_client: geth + filler_image: ethpandaops/geth:bal-devnet-7-amsterdam-override + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + # Schedule amsterdam at the osaka snapshot block 0's timestamp + 1. + fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam} + tests: + - tests/benchmark/compute # pytest paths inside the fill image + filter: bn128 # Quick subset for fast end-to-end iteration: only the bn128 compute tests. + # besu: testing_buildBlockV1 (TESTING namespace, besu-eth/besu#9838), + # debug_setHead and the session-fee pin all work, but besu's self-built + # block fails its own engine_newPayloadV4 with a World-State-Root mismatch + # (upstream besu bug in testing_buildBlockV1). Re-enable when fixed. + # - name: payload-generator-besu + # filler_client: besu + # filler_image: hyperledger/besu:26.6.1 # >= 26.6 ships testing_buildBlockV1 + # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-besu + # tests: + # - tests/benchmark/compute + # filter: bn128 + # nethermind: testing_buildBlockV1 works (Testing module in the special + # nethermindeth/nethermind:testing_build_block_with_opcode_tracing image), + # but debug_setHead is unimplemented (the per-test rewind aborts the run) + # and its receipt trips EEST's strict model. Re-enable when fixed. + # - name: payload-generator-nethermind + # filler_client: nethermind + # filler_image: nethermindeth/nethermind:testing_build_block_with_opcode_tracing + # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-nethermind + # tests: + # - tests/benchmark/compute + # filter: bn128 + +## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads +runner: + client_logs_to_stdout: true + cleanup_on_start: false + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + # Booting from a snapshot datadir leaves the Engine API forkchoice head + # un-established, so clients answer SYNCING to every newPayload. A bootstrap + # FCU to the snapshot head makes them canonical and ready to validate the + # replayed payloads (required for besu/reth/nethermind; geth is fine without). + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + genesis: + nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + #geth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json + besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json + # erigon consumes a geth-format genesis via its init container. Filename + # follows the per-client state-actor convention (geth-genesis.json, + # ethrex-genesis.json); confirm/adjust once state-actor publishes the + # erigon image. + erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/chainspec.json + #erigon: /tmp/benchmarkoor/state-actor/geth/geth-genesis.json + + datadirs: + geth: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + method: copy + reth: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth + method: copy + nethermind: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + method: copy + besu: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + method: copy + ethrex: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex + method: copy + erigon: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon + method: copy + + instances: + - id: geth + client: geth + image: ethpandaops/geth:master + - id: reth + client: reth + #image: ghcr.io/paradigmxyz/reth:latest + image: ghcr.io/paradigmxyz/reth:nightly@sha256:e528857e5e9ebc2c6cb99f28436e70ded38ca905629f00afc98d186e27d206e0 + - id: nethermind + client: nethermind + extra_args: + # benchmarkoor mounts the datadir at /data and the nethermind client + # passes --datadir=/data, but nethermind's BaseDbPath then defaults to + # /data/nethermind_db// — so it reads a fresh EMPTY state db and + # ignores the state-actor snapshot written directly under /data/{state, + # blocks,headers,blockInfos}. It then builds genesis from the empty-alloc + # chainspec, so HasStateForBlock(genesis) is false and every replayed + # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind + # read the snapshot directly (it then logs "Detected HalfPath key scheme"). + - --Init.BaseDbPath=/data + - id: besu + client: besu + # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: + # its synchronizer must run to register the post-merge head as in-sync, + # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false + # (the client defaults) keep it isolated — no real peers. + image: hyperledger/besu:26.6.1 + extra_args: + - --p2p-enabled=true + - id: ethrex + client: ethrex + # ethrex >= v16.0.0 is required: it's the first release with + # --skip-genesis-validation (lambdaclass/ethrex#6783), needed because the + # state-actor snapshot commits a synthetic state root that the emitted + # ethrex-genesis.json (empty alloc) can't reproduce. + image: ghcr.io/lambdaclass/ethrex:latest + extra_args: + - --skip-genesis-validation + - id: erigon + client: erigon + # MUST match the erigon that state-actor pins for the snapshot + # (Dockerfile.erigon builds erigon at the bal-devnet-7 commit and notes the + # daemon must use that exact binary). state-actor's `erigon init` encodes the + # MDBX chain config the bal-devnet-7 way (e.g. chainId as a JSON string); + # stable erigon (erigontech/erigon:latest, 3.4.x) panics reading it back with + # "cannot unmarshal \"1337\" into a *big.Int". ethpandaops/erigon:bal-devnet-7 + # reads it correctly and supports the amsterdam/BAL (EIP-7928) fork these + # benchmarks target. + image: ethpandaops/erigon:bal-devnet-7 diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index c68aaf152..24ea6f08a 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -188,6 +188,12 @@ func (b *EESTPayloadsBuilder) checkInputs(t *config.EESTPayloadTarget) error { } } + if t.ForkActivationGenesis != "" { + if _, err := os.Stat(t.ForkActivationGenesis); err != nil { + return fmt.Errorf("fork_activation_genesis: %w", err) + } + } + if t.AddressStubsFile != "" { if _, err := os.Stat(t.AddressStubsFile); err != nil { return fmt.Errorf("address_stubs_file: %w", err) @@ -262,6 +268,23 @@ func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t } }() + // When fork_activation_genesis is set, boot the filler with a + // --override.= flag so the target fork + // activates on the first block it builds (block N+1). See + // forkOverrideActivationFlag for why --override.genesis can't be used. + if t.ForkActivationGenesis != "" { + flag, err := forkOverrideActivationFlag(t.ForkActivationGenesis, t.Fork) + if err != nil { + return fmt.Errorf("scheduling %s activation: %w", t.Fork, err) + } + + t.FillerExtraArgs = append(t.FillerExtraArgs, flag) + + log.WithFields(logrus.Fields{ + "base_genesis": t.ForkActivationGenesis, "fork": t.Fork, "flag": flag, + }).Info("Scheduling fork activation on filler") + } + // Stream the filler's logs for the lifetime of this build. streamCtx, streamCancel := context.WithCancel(ctx) defer streamCancel() diff --git a/pkg/builder/genesis.go b/pkg/builder/genesis.go new file mode 100644 index 000000000..94d3050f1 --- /dev/null +++ b/pkg/builder/genesis.go @@ -0,0 +1,90 @@ +package builder + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" +) + +// forkOverrideActivationFlag returns the geth --override.= flag +// that schedules fork at the snapshot genesis block's timestamp + 1, read from +// baseGenesisPath (the state-actor snapshot's geth-genesis.json). The filler +// boots at the snapshot head (block N, timestamp T) and builds block N+1 at +// timestamp T+1, so fork activates exactly on the first block it builds. +// +// state-actor bakes the snapshot state into the DB under an empty-alloc genesis, +// so --override.genesis can't be used (geth recomputes an empty-state genesis +// and rejects the hash mismatch). The per-fork --override. flag instead +// amends only the in-memory chain config at boot, leaving the genesis block +// untouched. Requires a geth build that registers the flag (e.g. +// ethpandaops/geth:bal-devnet-7-amsterdam-override for amsterdam). +func forkOverrideActivationFlag(baseGenesisPath, fork string) (string, error) { + ts, err := genesisFileTimestamp(baseGenesisPath) + if err != nil { + return "", err + } + + return fmt.Sprintf("--override.%s=%d", strings.ToLower(fork), ts+1), nil +} + +// genesisFileTimestamp reads the genesis block timestamp from the top-level +// "timestamp" field of a geth genesis JSON file. +func genesisFileTimestamp(path string) (uint64, error) { + raw, err := os.ReadFile(path) + if err != nil { + return 0, fmt.Errorf("reading base genesis %q: %w", path, err) + } + + var genesis map[string]any + if err := json.Unmarshal(raw, &genesis); err != nil { + return 0, fmt.Errorf("parsing base genesis %q: %w", path, err) + } + + ts, err := genesisBlockTimestamp(genesis) + if err != nil { + return 0, fmt.Errorf("base genesis %q: %w", path, err) + } + + return ts, nil +} + +// genesisBlockTimestamp reads the genesis block's timestamp from the top-level +// "timestamp" field, accepting a 0x-prefixed hex string (geth's encoding) or a +// JSON number. +func genesisBlockTimestamp(genesis map[string]any) (uint64, error) { + v, ok := genesis["timestamp"] + if !ok { + return 0, fmt.Errorf("missing \"timestamp\" field") + } + + switch t := v.(type) { + case string: + ts, err := parseGenesisUint(t) + if err != nil { + return 0, fmt.Errorf("parsing \"timestamp\" %q: %w", t, err) + } + + return ts, nil + case float64: + return uint64(t), nil + default: + return 0, fmt.Errorf("unexpected \"timestamp\" type %T", v) + } +} + +// parseGenesisUint parses a uint from a genesis-encoded string: 0x-prefixed hex +// or plain decimal. +func parseGenesisUint(s string) (uint64, error) { + s = strings.TrimSpace(s) + if rest, ok := strings.CutPrefix(s, "0x"); ok { + return strconv.ParseUint(rest, 16, 64) + } + + if rest, ok := strings.CutPrefix(s, "0X"); ok { + return strconv.ParseUint(rest, 16, 64) + } + + return strconv.ParseUint(s, 10, 64) +} diff --git a/pkg/builder/genesis_test.go b/pkg/builder/genesis_test.go new file mode 100644 index 000000000..50fb70e5a --- /dev/null +++ b/pkg/builder/genesis_test.go @@ -0,0 +1,69 @@ +package builder + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeGenesisFixture(t *testing.T, timestamp string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "geth-genesis.json") + body := `{"timestamp":"` + timestamp + `","config":{"chainId":1337,"osakaTime":0}}` + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + + return path +} + +func TestForkOverrideActivationFlag(t *testing.T) { + tests := []struct { + name string + timestamp string + fork string + want string + }{ + {name: "zero timestamp", timestamp: "0x0", fork: "amsterdam", want: "--override.amsterdam=1"}, + {name: "hex timestamp", timestamp: "0x10", fork: "amsterdam", want: "--override.amsterdam=17"}, + {name: "fork lowercased", timestamp: "0x0", fork: "Amsterdam", want: "--override.amsterdam=1"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + flag, err := forkOverrideActivationFlag(writeGenesisFixture(t, tc.timestamp), tc.fork) + require.NoError(t, err) + assert.Equal(t, tc.want, flag) + }) + } +} + +func TestForkOverrideActivationFlag_Errors(t *testing.T) { + t.Run("missing file", func(t *testing.T) { + _, err := forkOverrideActivationFlag(filepath.Join(t.TempDir(), "nope.json"), "amsterdam") + require.Error(t, err) + }) + + t.Run("missing timestamp", func(t *testing.T) { + p := filepath.Join(t.TempDir(), "g.json") + require.NoError(t, os.WriteFile(p, []byte(`{"config":{}}`), 0o644)) + _, err := forkOverrideActivationFlag(p, "amsterdam") + require.ErrorContains(t, err, "timestamp") + }) +} + +func TestGenesisFileTimestamp(t *testing.T) { + assert := assert.New(t) + + hexTS, err := genesisFileTimestamp(writeGenesisFixture(t, "0xff")) + require.NoError(t, err) + assert.EqualValues(255, hexTS) + + decPath := filepath.Join(t.TempDir(), "g.json") + require.NoError(t, os.WriteFile(decPath, []byte(`{"timestamp":"42","config":{}}`), 0o644)) + decTS, err := genesisFileTimestamp(decPath) + require.NoError(t, err) + assert.EqualValues(42, decTS) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 99daacc85..520b4aa01 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -397,16 +397,25 @@ type EESTPayloadDefaults struct { } // EESTPayloadTarget is one fixture-generation run. Identity/locator fields -// (Name, FillerClient, SourceDir, OutputDir, GenesisFile, Tests, Filter, -// AddressStubsFile) live exclusively on the target; the remaining fields -// mirror EESTPayloadDefaults and are resolved via ResolveTarget. +// (Name, FillerClient, SourceDir, OutputDir, GenesisFile, +// ForkActivationGenesis, Tests, Filter, AddressStubsFile) live exclusively on +// the target; the remaining fields mirror EESTPayloadDefaults and are resolved +// via ResolveTarget. type EESTPayloadTarget struct { - Name string `yaml:"name,omitempty" mapstructure:"name"` - FillerClient string `yaml:"filler_client" mapstructure:"filler_client"` - SourceDir string `yaml:"source_dir" mapstructure:"source_dir"` - OutputDir string `yaml:"output_dir" mapstructure:"output_dir"` - GenesisFile string `yaml:"genesis_file,omitempty" mapstructure:"genesis_file"` - AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` + Name string `yaml:"name,omitempty" mapstructure:"name"` + FillerClient string `yaml:"filler_client" mapstructure:"filler_client"` + SourceDir string `yaml:"source_dir" mapstructure:"source_dir"` + OutputDir string `yaml:"output_dir" mapstructure:"output_dir"` + GenesisFile string `yaml:"genesis_file,omitempty" mapstructure:"genesis_file"` + // ForkActivationGenesis, when set, is a base genesis JSON (typically the + // state-actor snapshot's geth-genesis.json, generated at the prior fork) + // from which the builder derives the filler's genesis: it schedules Fork's + // activation at the base genesis block's timestamp + 1 (e.g. amsterdamTime) + // and boots the filler with --override.genesis. This fills a fork that + // activates one block after the snapshot. Mutually exclusive with + // GenesisFile; requires Fork. + ForkActivationGenesis string `yaml:"fork_activation_genesis,omitempty" mapstructure:"fork_activation_genesis"` + AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` // Tests are pytest paths inside the fill image, e.g. tests/benchmark/compute. Tests []string `yaml:"tests,omitempty" mapstructure:"tests"` Filter string `yaml:"filter,omitempty" mapstructure:"filter"` @@ -2144,6 +2153,25 @@ func validateEESTPayloadPaths(t *EESTPayloadTarget, prefix string, seenOutputs m return fmt.Errorf("%s.genesis_file must be an absolute path, got %q", prefix, t.GenesisFile) } + if t.ForkActivationGenesis != "" { + if t.GenesisFile != "" { + return fmt.Errorf( + "%s: genesis_file and fork_activation_genesis are mutually exclusive", prefix, + ) + } + + if !filepath.IsAbs(t.ForkActivationGenesis) { + return fmt.Errorf( + "%s.fork_activation_genesis must be an absolute path, got %q", + prefix, t.ForkActivationGenesis, + ) + } + + if t.Fork == "" { + return fmt.Errorf("%s.fork is required when fork_activation_genesis is set", prefix) + } + } + if t.AddressStubsFile != "" && !filepath.IsAbs(t.AddressStubsFile) { return fmt.Errorf( "%s.address_stubs_file must be an absolute path, got %q", prefix, t.AddressStubsFile, From c2a1100a4b13e559353b26e4496cdaf148fc92f5 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 23 Jun 2026 21:19:54 +0200 Subject: [PATCH 23/83] examples: predeploy the create2 deterministic factory for amsterdam fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EEST's fill bootstrap deploys the Arachnid deterministic-deployment proxy via a keyless tx with a fixed 100k gas limit. Amsterdam's state-expansion gas (EIP-7928/8037) makes that deploy cost ~324k, so the keyless method can't be used. Predeploy the proxy at the canonical 0x4e59…956c via state-actor's create2_factory template (explicit address, ordered before the 2 GB bloat entity so it survives target_size truncation); EEST detects the existing bytecode and skips the keyless deploy. --- .../config.state-actor-eest.amsterdam.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml index b08c0f439..39c981667 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -34,6 +34,21 @@ builder: #target_size: 20GB spec: | entities: + # Predeploy the Arachnid deterministic-deployment proxy at the canonical + # 0x4e59…956c. EEST's fill bootstrap deploys it via a keyless tx with a + # fixed 100k gas limit, which amsterdam's state-expansion gas (EIP-7928/ + # 8037, ~324k for this proxy) exceeds — so it must be predeployed. EEST + # detects the existing bytecode and skips the keyless deploy. + # + # MUST come before bloated-eoa-2g: state-actor truncates spec entities + # once the cumulative projected size exceeds target_size, and the 2 GB + # entity below would otherwise drop everything after it (incl. this). + - kind: contract + template: create2_factory + name: deterministic-deployment-proxy + # Explicit canonical address: with a name set, the address would + # otherwise be name-derived instead of the Arachnid default. + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C - kind: eoa name: bloated-eoa-2g balance: "1000000000000000000" From 67f3f738d94dc49666a6f0df87c988d35d8672fa Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 10:59:53 +0200 Subject: [PATCH 24/83] feat(examples): amsterdam eest_payloads fill config for geth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fill amsterdam compute fixtures against a geth filler while the state-actor snapshot stays osaka: - Point at skylenet/execution-specs devnets/bal/7-bench-cap-aware-deploy (cap-aware deploy gas sizing + seed-funding skip). - Use the published skylenet/geth:bal-devnet-7-amsterdam-override image, which registers --override.amsterdam to fork at snapshot block ts + 1. - Pre-fund fill-stateful's seed (0x7e5f…bdf) in the snapshot and pin rpc_seed_key so the withdrawal funding block is skipped and start_block == the snapshot block (stable per-test debug_setHead rewind). - Pre-deploy the Arachnid CREATE2 factory in the snapshot. - Filter out the currently-failing compute tests (blockhash pathdb rewind limit, blobhash blob-tx panic, amsterdam gas-repricing mismatches) so the fill is green. --- .../config.state-actor-eest.amsterdam.yaml | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml index 39c981667..13d465545 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -49,6 +49,17 @@ builder: # Explicit canonical address: with a name set, the address would # otherwise be name-derived instead of the Arachnid default. address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + # Pre-fund fill-stateful's seed account (rpc_seed_key=0x..01 → this + # address). With the seed already funded, fill-stateful skips the + # withdrawal funding block, so start_block == the snapshot block, whose + # persistent state debug_setHead can always rewind to between tests. + # Balance must be >= SEED_FUNDING_WEI (1e27); 1e28 gives headroom. + # Also before bloated-eoa-2g so it survives target_size truncation. + - kind: eoa + name: fill-stateful-seed + address: 0x7e5f4552091a69125d5dfcb7b8c2659029395bdf + balance: "10000000000000000000000000000" + nonce: 0 - kind: eoa name: bloated-eoa-2g balance: "1000000000000000000" @@ -75,12 +86,16 @@ builder: # of pulling a pre-built one. Alternatively set fill_image: . fill_dockerfile: Dockerfile.eest-filler pull_policy: if-not-present - eest_repo: https://github.com/ethereum/execution-specs.git - eest_ref: forks/amsterdam + eest_repo: https://github.com/skylenet/execution-specs.git + eest_ref: devnets/bal/7-bench-cap-aware-deploy config: # filler_image is set per-target below so each target can fill with a # different client. Shared knobs stay here and are hoisted into every target. fork: amsterdam # fill amsterdam fixtures (snapshot stays osaka) + # Pin the seed key so its account (0x7e5f…bdf, privkey 1) can be pre-funded + # in the snapshot above — lets fill-stateful skip the withdrawal funding + # block so start_block == the snapshot block (stable per-test rewind). + rpc_seed_key: "0x0000000000000000000000000000000000000000000000000000000000000001" gas_benchmark_values: [10] # Millions of gas to parametrise against datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) # fill-stateful forces testing_buildBlockV1 + debug_setHead, so a filler must @@ -96,14 +111,35 @@ builder: targets: - name: payload-generator-geth filler_client: geth - filler_image: ethpandaops/geth:bal-devnet-7-amsterdam-override + filler_image: skylenet/geth:bal-devnet-7-amsterdam-override source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth # Schedule amsterdam at the osaka snapshot block 0's timestamp + 1. fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam} tests: - tests/benchmark/compute # pytest paths inside the fill image - filter: bn128 # Quick subset for fast end-to-end iteration: only the bn128 compute tests. + # Exclude the currently-failing compute tests so the fill is green. Reasons: + # blockhash: 256-block chain exceeds geth pathdb rewind (per-test reset) + # blobhash: geth testing_buildBlockV1 panics on blob txs (no sidecar) + # the rest: amsterdam gas-repricing mismatches / sizing / setup issues + # NB: -k is substring-based, so this also drops the passing + # test_creates_collisions and test_storage_access_cold_benchmark. + filter: >- + not test_blockhash + and not test_blobhash + and not test_create + and not test_auth_transaction + and not test_unchunkified_bytecode + and not test_storage_access_cold + and not test_jumpdest_analysis + and not test_state_root_computation + and not test_contract_creation + and not test_contract_calling_many_addresses + and not test_selfdestruct_created + and not test_selfdestruct_initcode + and not test_block_full_data + and not test_prefetch_cold_storage + and not test_ext_account_query_cold # besu: testing_buildBlockV1 (TESTING namespace, besu-eth/besu#9838), # debug_setHead and the session-fee pin all work, but besu's self-built # block fails its own engine_newPayloadV4 with a World-State-Root mismatch From 8fc2ef5b91705c538ce176cf81175ce33136d1ae Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 11:09:08 +0200 Subject: [PATCH 25/83] feat(examples): run amsterdam eest payloads against geth runner Wire the runner's geth instance to execute the filled amsterdam stateful-engine fixtures: - Use skylenet/geth:bal-devnet-7-amsterdam-override (same image used to fill) so geth registers --override.amsterdam. - Add --override.amsterdam=1 so amsterdam activates at snapshot block 0's timestamp + 1; geth boots from the osaka snapshot datadir and would otherwise reject the amsterdam newPayloadV5 blocks. Validated: geth replays the full green set (830 passed, 0 failed). --- .../configuration/config.state-actor-eest.amsterdam.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml index 13d465545..d5da28f54 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -226,7 +226,13 @@ runner: instances: - id: geth client: geth - image: ethpandaops/geth:master + image: skylenet/geth:bal-devnet-7-amsterdam-override + extra_args: + # Activate amsterdam at snapshot block 0's timestamp (0) + 1. The snapshot + # is osaka and geth boots from the datadir (its genesis is commented out + # above), so without this the amsterdam payloads (newPayloadV5) would be + # rejected. Needs the amsterdam-override image; a stock geth ignores it. + - --override.amsterdam=1 - id: reth client: reth #image: ghcr.io/paradigmxyz/reth:latest From e5be9c7c407acd57f12907cda9f2a1b53522bf94 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 13:03:44 +0200 Subject: [PATCH 26/83] feat(runner): automated per-instance genesis fork overrides + amsterdam clients Add per-instance genesis patching so clients that read their fork schedule from the genesis (rather than a CLI flag like geth/erigon's --override.amsterdam) can activate amsterdam on an osaka state-actor snapshot at boot, with no manual chainspec editing: - genesis_fork_override (geth-format): sets config.Time and inherits the blobSchedule entry of the latest preceding fork. Used by besu, reth, ethrex. - genesis_eip_override (parity/nethermind-format): sets the listed EIPs' params.eipTransitionTimestamp. The devnet-specific EIP list lives in config. Used by nethermind. Both rewrite only the config/params object and preserve every other genesis field verbatim (json.Number round-trip), so the genesis block hash is unchanged and stays compatible with the snapshot datadir. Wire all six clients in config.state-actor-eest.amsterdam.yaml from the benchmarkoor-tests *-bal-full reference. Amsterdam is activated three ways: geth/erigon via --override.amsterdam, besu/reth/ethrex via genesis_fork_override, nethermind via genesis_eip_override (devnet-7 EIP set 7708,7778,7843,7928,7954,7976,7981,8024,8037). Verified on the full green compute set (830 tests): geth, besu, ethrex, nethermind, erigon all 830/0; reth has 8 amsterdam warm-storage/tload repricing failures (client/spec discrepancy, not a config issue). --- .../config.state-actor-eest.amsterdam.yaml | 101 +++++++-- pkg/config/config.go | 15 ++ pkg/runner/genesis_override.go | 171 +++++++++++++++ pkg/runner/genesis_override_test.go | 196 ++++++++++++++++++ pkg/runner/lifecycle.go | 53 +++++ 5 files changed, 523 insertions(+), 13 deletions(-) create mode 100644 pkg/runner/genesis_override.go create mode 100644 pkg/runner/genesis_override_test.go diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml index d5da28f54..2f317aa6b 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -8,8 +8,17 @@ # # ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.amsterdam.yaml --force # -# Running benchmark using geth: +# Running the benchmark (one instance per client): # ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=geth +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=besu +# +# Each client activates amsterdam on the osaka snapshot one of three ways: +# - geth, erigon: CLI fork override (--override.amsterdam=) +# - besu, reth, ethrex: genesis_fork_override (benchmarkoor sets Time in +# the geth-format genesis at boot) +# - nethermind: genesis_eip_override (benchmarkoor sets the amsterdam +# EIPs' eipTransitionTimestamp in the parity chainspec) +# Snapshot block 0's timestamp is 0, so amsterdam is scheduled at ts 1. # global: log_level: info @@ -191,9 +200,16 @@ runner: max_retries: 10 backoff: 1s genesis: + # nethermind reads a parity-format chainspec that schedules forks per-EIP, + # so amsterdam is activated via the per-instance genesis_eip_override below + # (not genesis_fork_override). The osaka chainspec is used as-is. nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json #geth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json + # besu/reth/ethrex have no --override.amsterdam flag — they read forks + # from the genesis. benchmarkoor patches it for them at boot via the + # per-instance genesis_fork_override below (sets amsterdamTime + inherits + # the amsterdam blobSchedule), so the osaka chainspec is used as-is. besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json # erigon consumes a geth-format genesis via its init container. Filename @@ -235,10 +251,30 @@ runner: - --override.amsterdam=1 - id: reth client: reth - #image: ghcr.io/paradigmxyz/reth:latest - image: ghcr.io/paradigmxyz/reth:nightly@sha256:e528857e5e9ebc2c6cb99f28436e70ded38ca905629f00afc98d186e27d206e0 + # bal-devnet-7 is the reth line with amsterdam/BAL support. reth reads forks + # from its chainspec, so patch amsterdam in at boot. + image: ethpandaops/reth:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 + extra_args: + # reth-bal-full knobs (mirror benchmarkoor-tests): "full" is the default + # BAL mode, so only the slow-block log threshold is set. + - --engine.slow-block-threshold=0 + # Booting from a snapshot, reth can answer the first newPayload before the + # forkchoice head is established; retry the non-SYNCING failure briefly. + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s - id: nethermind client: nethermind + image: nethermindeth/nethermind:master + # nethermind's parity chainspec schedules forks per-EIP, so activate the + # amsterdam-devnet-7 EIP set at snapshot block 0's ts (0) + 1. benchmarkoor + # patches params.eipTransitionTimestamp into the chainspec at boot. + genesis_eip_override: + timestamp: 1 + eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] extra_args: # benchmarkoor mounts the datadir at /data and the nethermind client # passes --datadir=/data, but nethermind's BaseDbPath then defaults to @@ -249,24 +285,46 @@ runner: # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind # read the snapshot directly (it then logs "Detected HalfPath key scheme"). - --Init.BaseDbPath=/data + # nethermind-bal-full knobs (mirror benchmarkoor-tests). + - --Blocks.ParallelExecution=true + - --Blocks.ParallelExecutionBatchRead=true - id: besu client: besu - # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: - # its synchronizer must run to register the post-merge head as in-sync, - # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false - # (the client defaults) keep it isolated — no real peers. - image: hyperledger/besu:26.6.1 + # bal-devnet-7 is the besu line with amsterdam/BAL (EIP-7928) support. + image: ethpandaops/besu:bal-devnet-7 + # besu reads forks from the genesis, so activate amsterdam at snapshot + # block 0's ts (0) + 1 by patching its chainspec at boot. + genesis_fork_override: + amsterdam: 1 + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" extra_args: + # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: + # its synchronizer must run to register the post-merge head as in-sync, + # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false + # (the client defaults) keep it isolated — no real peers. - --p2p-enabled=true + - --Xplugin-rocksdb-high-spec-enabled=true + # BAL "full" mode — mirrors besu-bal-full in ethpandaops/benchmarkoor-tests. + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=true + - --Xbal-prefetch-batch-size=8 + - --Xbal-processing-timeout=-1 + - --Xbal-state-root-timeout=-1 - id: ethrex client: ethrex - # ethrex >= v16.0.0 is required: it's the first release with - # --skip-genesis-validation (lambdaclass/ethrex#6783), needed because the - # state-actor snapshot commits a synthetic state root that the emitted - # ethrex-genesis.json (empty alloc) can't reproduce. - image: ghcr.io/lambdaclass/ethrex:latest + # bal-devnet-7 is the ethrex line with amsterdam/BAL support. ethrex reads + # forks from its genesis, so patch amsterdam in at boot. + image: ethpandaops/ethrex:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 extra_args: + # --skip-genesis-validation: the state-actor snapshot commits a synthetic + # state root the emitted ethrex-genesis.json (empty alloc) can't reproduce. - --skip-genesis-validation + # ethrex-bal-full knob (mirrors benchmarkoor-tests). + - --no-precompile-cache - id: erigon client: erigon # MUST match the erigon that state-actor pins for the snapshot @@ -278,3 +336,20 @@ runner: # reads it correctly and supports the amsterdam/BAL (EIP-7928) fork these # benchmarks target. image: ethpandaops/erigon:bal-devnet-7 + extra_args: + # erigon supports a geth-style fork override. Snapshot block 0 ts is 0, so + # activate amsterdam at ts 1 (the reference devnet uses its real ts). + - --override.amsterdam=1 + # erigon-bal-full knobs (mirror benchmarkoor-tests): keep execution + # deterministic/online by disabling background merge/prune/maintenance and + # synchronous FCU commit so replayed payloads are validated immediately. + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + # erigon is slow to register the snapshot head as in-sync; give the bootstrap + # FCU a long runway (mirrors the reference erigon-bal-full). + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s diff --git a/pkg/config/config.go b/pkg/config/config.go index 520b4aa01..85d0233fa 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -945,6 +945,19 @@ func (s *SourceConfig) IsConfigured() bool { // DefaultContainerDir is the default container mount path for data directories. const DefaultContainerDir = "/data" +// GenesisEIPOverride activates a set of EIPs at a given timestamp in a +// parity/nethermind-format chainspec (which schedules forks per-EIP rather than +// by fork name). It is the parity-format counterpart of GenesisForkOverride: the +// devnet-specific EIP list lives in config and benchmarkoor patches the +// chainspec at boot. +type GenesisEIPOverride struct { + // Timestamp is the activation time (unix seconds) applied to every listed EIP. + Timestamp uint64 `yaml:"timestamp" mapstructure:"timestamp"` + // EIPs are the EIP numbers to activate, e.g. [7928, 8037]. Each becomes a + // params.eipTransitionTimestamp entry. + EIPs []uint64 `yaml:"eips" mapstructure:"eips"` +} + // DataDirConfig configures a pre-populated data directory for a client. type DataDirConfig struct { SourceDir string `yaml:"source_dir" json:"source_dir" mapstructure:"source_dir"` @@ -1268,6 +1281,8 @@ type ClientInstance struct { Restart string `yaml:"restart,omitempty" mapstructure:"restart"` Environment map[string]string `yaml:"environment,omitempty" mapstructure:"environment"` Genesis string `yaml:"genesis,omitempty" mapstructure:"genesis"` + GenesisForkOverride map[string]uint64 `yaml:"genesis_fork_override,omitempty" mapstructure:"genesis_fork_override"` + GenesisEIPOverride *GenesisEIPOverride `yaml:"genesis_eip_override,omitempty" mapstructure:"genesis_eip_override"` DataDir *DataDirConfig `yaml:"datadir,omitempty" mapstructure:"datadir"` DropMemoryCaches string `yaml:"drop_memory_caches,omitempty" mapstructure:"drop_memory_caches"` RollbackStrategy string `yaml:"rollback_strategy,omitempty" mapstructure:"rollback_strategy"` diff --git a/pkg/runner/genesis_override.go b/pkg/runner/genesis_override.go new file mode 100644 index 000000000..562e8f5b0 --- /dev/null +++ b/pkg/runner/genesis_override.go @@ -0,0 +1,171 @@ +package runner + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/ethpandaops/benchmarkoor/pkg/config" +) + +// blobForkOrder lists the blob-bearing forks in activation order. When a fork +// override adds a fork the genesis blobSchedule doesn't yet cover, the new fork +// inherits the schedule of the latest preceding fork present here — geth-family +// clients reject an active fork that carries no blob schedule. +var blobForkOrder = []string{"cancun", "prague", "osaka", "amsterdam"} + +// applyGenesisForkOverrides patches a geth-format genesis JSON so the given +// forks activate at the given timestamps. It is the genesis-file equivalent of +// geth's --override. flag, for clients that instead read their fork +// schedule from the genesis (besu, reth, ethrex). For each fork it sets +// config.Time and, when a blobSchedule is present but lacks the fork, +// inherits the latest preceding fork's blob parameters. +// +// Only the top-level "config" object is rewritten; every other field round-trips +// verbatim and existing numbers are preserved exactly (so the genesis block hash +// is unchanged). It returns an error if the genesis is not geth-format (has no +// top-level "config" object), since the patch shape is format-specific. +func applyGenesisForkOverrides(genesis []byte, overrides map[string]uint64) ([]byte, error) { + if len(overrides) == 0 { + return genesis, nil + } + + // Decode only the top level so untouched fields round-trip byte-for-byte. + var top map[string]json.RawMessage + if err := json.Unmarshal(genesis, &top); err != nil { + return nil, fmt.Errorf("parsing genesis json: %w", err) + } + + rawConfig, ok := top["config"] + if !ok { + return nil, fmt.Errorf( + "genesis has no \"config\" object; genesis_fork_override only " + + "supports geth-format genesis files", + ) + } + + // UseNumber keeps existing numbers as json.Number so they re-encode exactly + // rather than via lossy float64. + dec := json.NewDecoder(bytes.NewReader(rawConfig)) + dec.UseNumber() + + cfg := make(map[string]any) + if err := dec.Decode(&cfg); err != nil { + return nil, fmt.Errorf("parsing genesis config: %w", err) + } + + for fork, ts := range overrides { + cfg[fork+"Time"] = ts + inheritBlobSchedule(cfg, fork) + } + + patchedConfig, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis config: %w", err) + } + + top["config"] = patchedConfig + + patched, err := json.Marshal(top) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis: %w", err) + } + + return patched, nil +} + +// applyGenesisEIPOverrides patches a parity/nethermind-format chainspec so the +// given EIPs activate at the override timestamp. It is the parity-format +// counterpart of applyGenesisForkOverrides: parity chainspecs schedule forks +// per-EIP (params.eipTransitionTimestamp) rather than by fork name, so the +// devnet-specific EIP list comes from config. +// +// Only the "params" object is rewritten; every other field round-trips verbatim. +// It returns an error if the genesis is not parity-format (has no top-level +// "params" object). +func applyGenesisEIPOverrides(genesis []byte, override *config.GenesisEIPOverride) ([]byte, error) { + if override == nil || len(override.EIPs) == 0 { + return genesis, nil + } + + var top map[string]json.RawMessage + if err := json.Unmarshal(genesis, &top); err != nil { + return nil, fmt.Errorf("parsing genesis json: %w", err) + } + + rawParams, ok := top["params"] + if !ok { + return nil, fmt.Errorf( + "genesis has no \"params\" object; genesis_eip_override only " + + "supports parity/nethermind-format chainspecs", + ) + } + + dec := json.NewDecoder(bytes.NewReader(rawParams)) + dec.UseNumber() + + params := make(map[string]any) + if err := dec.Decode(¶ms); err != nil { + return nil, fmt.Errorf("parsing genesis params: %w", err) + } + + // Parity transition timestamps are hex-encoded strings (e.g. "0x1"). + ts := fmt.Sprintf("0x%x", override.Timestamp) + for _, eip := range override.EIPs { + params[fmt.Sprintf("eip%dTransitionTimestamp", eip)] = ts + } + + patchedParams, err := json.Marshal(params) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis params: %w", err) + } + + top["params"] = patchedParams + + patched, err := json.Marshal(top) + if err != nil { + return nil, fmt.Errorf("encoding patched genesis: %w", err) + } + + return patched, nil +} + +// inheritBlobSchedule ensures cfg.blobSchedule covers fork by copying the +// schedule of the latest preceding fork in blobForkOrder. It is a no-op when the +// genesis has no blobSchedule, the fork already has a schedule, the fork is +// unknown to blobForkOrder, or no preceding schedule exists. +func inheritBlobSchedule(cfg map[string]any, fork string) { + schedule, ok := cfg["blobSchedule"].(map[string]any) + if !ok { + return + } + + if _, exists := schedule[fork]; exists { + return + } + + forkIdx := indexOf(blobForkOrder, fork) + if forkIdx < 0 { + return + } + + // Walk backwards to the nearest preceding fork that has a schedule. + for i := forkIdx - 1; i >= 0; i-- { + if prev, ok := schedule[blobForkOrder[i]]; ok { + schedule[fork] = prev + + return + } + } +} + +// indexOf returns the index of target in list, or -1 if absent. +func indexOf(list []string, target string) int { + for i, v := range list { + if v == target { + return i + } + } + + return -1 +} diff --git a/pkg/runner/genesis_override_test.go b/pkg/runner/genesis_override_test.go new file mode 100644 index 000000000..a4971466f --- /dev/null +++ b/pkg/runner/genesis_override_test.go @@ -0,0 +1,196 @@ +package runner + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ethpandaops/benchmarkoor/pkg/config" +) + +func TestApplyGenesisForkOverrides(t *testing.T) { + t.Run("no overrides returns input unchanged", func(t *testing.T) { + in := []byte(`{"config":{"osakaTime":0}}`) + + out, err := applyGenesisForkOverrides(in, nil) + + require.NoError(t, err) + assert.Equal(t, in, out) + }) + + t.Run("non-geth genesis errors", func(t *testing.T) { + // Parity-format chainspec has params, not config. + in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"}}`) + + _, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "geth-format") + }) + + t.Run("sets fork time and inherits blob schedule", func(t *testing.T) { + in := []byte(`{ + "config": { + "chainId": 1337, + "osakaTime": 0, + "blobSchedule": { + "osaka": {"baseFeeUpdateFraction": 5007716, "max": 12, "target": 9} + } + }, + "gasLimit": "0x11e1a300" + }`) + + out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + cfg := decodeConfig(t, out) + assert.EqualValues(t, 1, cfg["amsterdamTime"]) + + bs, ok := cfg["blobSchedule"].(map[string]any) + require.True(t, ok) + + amsterdam, ok := bs["amsterdam"].(map[string]any) + require.True(t, ok, "amsterdam should inherit osaka's schedule") + assert.EqualValues(t, 12, amsterdam["max"]) + assert.EqualValues(t, 9, amsterdam["target"]) + + // Untouched top-level fields survive. + var top map[string]any + require.NoError(t, json.Unmarshal(out, &top)) + assert.Equal(t, "0x11e1a300", top["gasLimit"]) + }) + + t.Run("inherits latest preceding fork when several exist", func(t *testing.T) { + in := []byte(`{"config":{"blobSchedule":{ + "cancun":{"max":6}, + "prague":{"max":9}, + "osaka":{"max":12} + }}}`) + + out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + bs := decodeConfig(t, out)["blobSchedule"].(map[string]any) + amsterdam := bs["amsterdam"].(map[string]any) + assert.EqualValues(t, 12, amsterdam["max"], "should inherit osaka, the latest") + }) + + t.Run("does not overwrite an existing fork schedule", func(t *testing.T) { + in := []byte(`{"config":{"blobSchedule":{ + "osaka":{"max":12}, + "amsterdam":{"max":99} + }}}`) + + out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + bs := decodeConfig(t, out)["blobSchedule"].(map[string]any) + amsterdam := bs["amsterdam"].(map[string]any) + assert.EqualValues(t, 99, amsterdam["max"]) + }) + + t.Run("no blob schedule only sets the time", func(t *testing.T) { + in := []byte(`{"config":{"osakaTime":0}}`) + + out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + cfg := decodeConfig(t, out) + assert.EqualValues(t, 1, cfg["amsterdamTime"]) + _, hasBlob := cfg["blobSchedule"] + assert.False(t, hasBlob) + }) + + t.Run("preserves large integers without float corruption", func(t *testing.T) { + // A value beyond float64's exact-integer range must round-trip exactly. + in := []byte(`{"config":{"terminalTotalDifficulty":115792089237316195423570985008687907853269984665640564039457584007913129639936}}`) + + out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + require.NoError(t, err) + + assert.Contains(t, string(out), + "115792089237316195423570985008687907853269984665640564039457584007913129639936") + }) +} + +func TestApplyGenesisEIPOverrides(t *testing.T) { + t.Run("nil or empty override returns input unchanged", func(t *testing.T) { + in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"}}`) + + out, err := applyGenesisEIPOverrides(in, nil) + require.NoError(t, err) + assert.Equal(t, in, out) + + out, err = applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{Timestamp: 1}) + require.NoError(t, err) + assert.Equal(t, in, out) + }) + + t.Run("non-parity genesis errors", func(t *testing.T) { + in := []byte(`{"config":{"osakaTime":0}}`) + + _, err := applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{ + Timestamp: 1, EIPs: []uint64{7928}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "parity") + }) + + t.Run("sets eip transition timestamps as hex", func(t *testing.T) { + in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"},"name":"x"}`) + + out, err := applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{ + Timestamp: 1, EIPs: []uint64{7928, 8037}, + }) + require.NoError(t, err) + + params := decodeParams(t, out) + assert.Equal(t, "0x1", params["eip7928TransitionTimestamp"]) + assert.Equal(t, "0x1", params["eip8037TransitionTimestamp"]) + // Pre-existing params survive. + assert.Equal(t, "0x0", params["eip7825TransitionTimestamp"]) + + var top map[string]any + require.NoError(t, json.Unmarshal(out, &top)) + assert.Equal(t, "x", top["name"]) + }) + + t.Run("encodes larger timestamps as hex", func(t *testing.T) { + in := []byte(`{"params":{}}`) + + out, err := applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{ + Timestamp: 1769856767, EIPs: []uint64{7928}, + }) + require.NoError(t, err) + + params := decodeParams(t, out) + assert.Equal(t, "0x697ddeff", params["eip7928TransitionTimestamp"]) + }) +} + +func decodeParams(t *testing.T, genesis []byte) map[string]any { + t.Helper() + + var top map[string]json.RawMessage + require.NoError(t, json.Unmarshal(genesis, &top)) + + var params map[string]any + require.NoError(t, json.Unmarshal(top["params"], ¶ms)) + + return params +} + +func decodeConfig(t *testing.T, genesis []byte) map[string]any { + t.Helper() + + var top map[string]json.RawMessage + require.NoError(t, json.Unmarshal(genesis, &top)) + + var cfg map[string]any + require.NoError(t, json.Unmarshal(top["config"], &cfg)) + + return cfg +} diff --git a/pkg/runner/lifecycle.go b/pkg/runner/lifecycle.go index dbdfee614..8f8d8c489 100644 --- a/pkg/runner/lifecycle.go +++ b/pkg/runner/lifecycle.go @@ -187,6 +187,59 @@ func (r *runner) runContainerLifecycle( log.Info("No genesis configured, skipping genesis setup") } + // Apply per-instance genesis fork-time overrides (the genesis-file + // equivalent of geth's --override. for clients that read forks from + // the genesis, e.g. besu/reth/ethrex). + if len(instance.GenesisForkOverride) > 0 { + if len(genesisContent) == 0 { + return fmt.Errorf( + "instance %s sets genesis_fork_override but has no genesis file", + instance.ID, + ) + } + + patched, overrideErr := applyGenesisForkOverrides( + genesisContent, instance.GenesisForkOverride, + ) + if overrideErr != nil { + return fmt.Errorf( + "applying genesis fork override for %s: %w", instance.ID, overrideErr, + ) + } + + genesisContent = patched + + log.WithField("forks", instance.GenesisForkOverride). + Info("Applied genesis fork-time overrides") + } + + // Apply per-instance genesis EIP overrides (parity/nethermind-format + // chainspecs schedule forks per-EIP rather than by fork name). + if instance.GenesisEIPOverride != nil && len(instance.GenesisEIPOverride.EIPs) > 0 { + if len(genesisContent) == 0 { + return fmt.Errorf( + "instance %s sets genesis_eip_override but has no genesis file", + instance.ID, + ) + } + + patched, overrideErr := applyGenesisEIPOverrides( + genesisContent, instance.GenesisEIPOverride, + ) + if overrideErr != nil { + return fmt.Errorf( + "applying genesis eip override for %s: %w", instance.ID, overrideErr, + ) + } + + genesisContent = patched + + log.WithFields(logrus.Fields{ + "eips": instance.GenesisEIPOverride.EIPs, + "timestamp": instance.GenesisEIPOverride.Timestamp, + }).Info("Applied genesis EIP-time overrides") + } + // Fail if neither genesis nor datadir is configured. if genesisSource == "" && !useDataDir { return fmt.Errorf( From 5c47159cfa578f4820e7e14c3931891198eaf6c6 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 13:08:10 +0200 Subject: [PATCH 27/83] chore(examples): drop bloated-eoa-2g from the amsterdam state-actor spec The 2 GB padding entity (and its target_size truncation-ordering constraints) is not needed for the compute benchmarks. Keep only the functionally-required entities: the CREATE2 factory and the pre-funded fill-stateful seed. --- .../config.state-actor-eest.amsterdam.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml index 2f317aa6b..8dd46671e 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -48,10 +48,6 @@ builder: # fixed 100k gas limit, which amsterdam's state-expansion gas (EIP-7928/ # 8037, ~324k for this proxy) exceeds — so it must be predeployed. EEST # detects the existing bytecode and skips the keyless deploy. - # - # MUST come before bloated-eoa-2g: state-actor truncates spec entities - # once the cumulative projected size exceeds target_size, and the 2 GB - # entity below would otherwise drop everything after it (incl. this). - kind: contract template: create2_factory name: deterministic-deployment-proxy @@ -63,18 +59,11 @@ builder: # withdrawal funding block, so start_block == the snapshot block, whose # persistent state debug_setHead can always rewind to between tests. # Balance must be >= SEED_FUNDING_WEI (1e27); 1e28 gives headroom. - # Also before bloated-eoa-2g so it survives target_size truncation. - kind: eoa name: fill-stateful-seed address: 0x7e5f4552091a69125d5dfcb7b8c2659029395bdf balance: "10000000000000000000000000000" nonce: 0 - - kind: eoa - name: bloated-eoa-2g - balance: "1000000000000000000" - nonce: 0 - code: "0xef01003333333333333333333333333333333333333333" - approximate_size_bytes: 2_000_000_000 targets: - client: geth output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth From 42d47fdee5b525765ffd4a44b2f1cc083992c6a8 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 13:17:47 +0200 Subject: [PATCH 28/83] docs(examples): note erigon can't use genesis_fork_override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit erigon reads its fork schedule from the MDBX datadir, not the genesis file (like geth), so a patched genesis is ignored and amsterdam stays inactive — the --override.amsterdam flag is required. Document this next to the flag so it isn't swapped for genesis_fork_override. --- examples/configuration/config.state-actor-eest.amsterdam.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml index 8dd46671e..496309db6 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -328,6 +328,9 @@ runner: extra_args: # erigon supports a geth-style fork override. Snapshot block 0 ts is 0, so # activate amsterdam at ts 1 (the reference devnet uses its real ts). + # NB: genesis_fork_override does NOT work for erigon — like geth it reads + # the fork schedule from the datadir, not the genesis file, so the flag is + # required (a patched genesis is ignored and amsterdam stays inactive). - --override.amsterdam=1 # erigon-bal-full knobs (mirror benchmarkoor-tests): keep execution # deterministic/online by disabling background merge/prune/maintenance and From 9808138d74b09d6f8b9f8f365e432010228d018b Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 13:19:47 +0200 Subject: [PATCH 29/83] docs(configuration): document genesis_fork_override and genesis_eip_override Add the two per-instance genesis override options to the Client Instances table plus a "Genesis Fork & EIP Overrides" subsection explaining the geth-format vs parity/nethermind formats, blobSchedule inheritance, the geth/erigon caveat (they read forks from the datadir, not the genesis), and the wrong-format error. --- docs/configuration.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 34260cab8..dc6d4a023 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1161,6 +1161,8 @@ runner: | `restart` | string | No | - | Container restart policy | | `environment` | map | No | - | Additional environment variables | | `genesis` | string | No | From `runner.client.config.genesis` | Override genesis file URL | +| `genesis_fork_override` | map | No | - | Activate forks at given timestamps by patching a geth-format genesis at boot. See [Genesis Fork & EIP Overrides](#genesis-fork--eip-overrides) | +| `genesis_eip_override` | object | No | - | Activate EIPs at a timestamp by patching a parity/nethermind chainspec at boot. See [Genesis Fork & EIP Overrides](#genesis-fork--eip-overrides) | | `datadir` | object | No | From `runner.client.datadirs` | Instance-specific data directory config | | `drop_memory_caches` | string | No | From `runner.client.config` | Instance-specific cache drop setting | | `rollback_strategy` | string | No | From `runner.client.config` | Instance-specific rollback strategy | @@ -1175,6 +1177,45 @@ runner: | `bootstrap_fcu` | bool/object | No | From `runner.client.config` | Instance-specific bootstrap FCU setting | | `opcode_extraction` | object | No | From `runner.client.config` | Instance-specific opcode extraction setting (replaces global) | +#### Genesis Fork & EIP Overrides + +These options let an instance activate a fork that is not scheduled in the genesis it boots from — for example, running Amsterdam payloads against an Osaka snapshot. benchmarkoor patches the genesis file in-memory at boot, before mounting it; the source genesis on disk is never modified, and untouched fields (including large integers) round-trip verbatim, so the genesis block hash is unchanged. + +Use these only for clients that read their fork schedule from the genesis file. **geth and erigon do not** — they read the fork schedule from the datadir, so a patched genesis is ignored. For those, use the client's own fork-override flag instead (e.g. `--override.amsterdam=` in `extra_args`). + +**`genesis_fork_override`** — for geth-format genesis files (besu, reth, ethrex). A map of fork name to activation timestamp. For each entry it sets `config.Time`, and if the genesis has a `blobSchedule` that lacks the fork, it inherits the schedule of the latest preceding fork (so the new fork carries a blob schedule, as geth-family clients require). + +```yaml +runner: + instances: + - id: besu + client: besu + genesis: /path/to/osaka-chainspec.json # used as-is + genesis_fork_override: + amsterdam: 1 # sets config.amsterdamTime=1, inherits blobSchedule.amsterdam +``` + +**`genesis_eip_override`** — for parity/nethermind-format chainspecs, which schedule forks per-EIP rather than by fork name. It sets `params.eipTransitionTimestamp` for each listed EIP to the given (hex-encoded) timestamp. The EIP list is devnet-specific, so it lives in config. + +```yaml +runner: + instances: + - id: nethermind + client: nethermind + genesis: /path/to/osaka-parity-chainspec.json # used as-is + genesis_eip_override: + timestamp: 1 + eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] +``` + +| Option | Type | Description | +|--------|------|-------------| +| `genesis_fork_override` | map[string]uint | Fork name → activation timestamp (unix seconds). geth-format genesis only. | +| `genesis_eip_override.timestamp` | uint | Activation timestamp (unix seconds) applied to every listed EIP. | +| `genesis_eip_override.eips` | []uint | EIP numbers to activate, e.g. `[7928, 8037]`. parity/nethermind chainspec only. | + +Applying an override to the wrong genesis format is an error (a geth-format override needs a top-level `config` object; an EIP override needs a top-level `params` object). + ## Resource Limits Resource limits can be configured globally (`runner.client.config.resource_limits`) or per-instance (`runner.instances[].resource_limits`). Instance-level settings override global defaults. From 66b1d4fac133b71799be7582303c34642489b4d4 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 13:36:24 +0200 Subject: [PATCH 30/83] feat(examples): add stateful EIP-7928 (BAL) eest_payloads target Add payload-generator-geth-stateful, filling tests/benchmark/stateful's eip7928_block_level_access_lists into eest-fixtures-amsterdam-stateful. These BAL tests are self-contained (each deploys its own contracts via pre.deploy_contract / pre.fund_eoa against the predeployed CREATE2 factory + funded seed), so no extra state-actor entities (the 23-29 repricing templates) are required. The bloatnet stateful tests are intentionally excluded: test_single_opcode / test_multi_opcode need giant pre-deployed ERC20s/EOAs (50GB+) supplied via --address-stubs, infeasible on this 256MB snapshot. Verified: fill = 8 passed; run = geth/besu/nethermind/ethrex/erigon all 8/8. reth is slow on the heavy SLOAD-loop tests (same as on compute). Document the stateful build outputs and how to run them (point the runner at the -stateful fixtures dir) in the header. --- .../config.state-actor-eest.amsterdam.yaml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.amsterdam.yaml index 496309db6..75a65a4c6 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.amsterdam.yaml @@ -8,10 +8,18 @@ # # ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.amsterdam.yaml --force # -# Running the benchmark (one instance per client): +# Two fixture sets are built: tests/benchmark/compute (target payload-generator-geth, +# output eest-fixtures-amsterdam) and tests/benchmark/stateful EIP-7928/BAL (target +# payload-generator-geth-stateful, output eest-fixtures-amsterdam-stateful). +# +# Running the COMPUTE benchmark (one instance per client): # ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=geth # ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=besu # +# Running the STATEFUL (BAL) benchmark — point the runner at the stateful fixtures: +# EEST_FIXTURES_DIR=/tmp/benchmarkoor/eest-fixtures-amsterdam-stateful \ +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=geth +# # Each client activates amsterdam on the osaka snapshot one of three ways: # - geth, erigon: CLI fork override (--override.amsterdam=) # - besu, reth, ethrex: genesis_fork_override (benchmarkoor sets Time in @@ -138,6 +146,20 @@ builder: and not test_block_full_data and not test_prefetch_cold_storage and not test_ext_account_query_cold + # Stateful EIP-7928 (BAL) benchmarks. Self-contained like the compute tests + # (each deploys its own contracts via pre.deploy_contract / pre.fund_eoa + # against the predeployed CREATE2 factory + funded seed), so no extra + # state-actor entities are needed. The bloatnet stateful tests are NOT built + # here: test_single_opcode / test_multi_opcode need giant pre-deployed ERC20s + # /EOAs (50GB..) supplied via --address-stubs, infeasible on this snapshot. + - name: payload-generator-geth-stateful + filler_client: geth + filler_image: skylenet/geth:bal-devnet-7-amsterdam-override + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam}-stateful + tests: + - tests/benchmark/stateful/eip7928_block_level_access_lists # besu: testing_buildBlockV1 (TESTING namespace, besu-eth/besu#9838), # debug_setHead and the session-fee pin all work, but besu's self-built # block fails its own engine_newPayloadV4 with a World-State-Root mismatch From 8a5a110eda5f4cda5e8809bb79596921ff5704de Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 13:43:06 +0200 Subject: [PATCH 31/83] refactor(examples): split amsterdam config into compute + stateful files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace config.state-actor-eest.amsterdam.yaml with two self-contained configs, each building the same state-actor snapshot but filling/running one fixture set: - config.state-actor-eest.simple.amsterdam.compute.yaml — tests/benchmark /compute (fixtures eest-fixtures-amsterdam), with the green-set filter. - config.state-actor-eest.simple.amsterdam.stateful.yaml — tests/benchmark /stateful EIP-7928/BAL (fixtures eest-fixtures-amsterdam-stateful). The stateful config defaults its fixtures dir to eest-fixtures-amsterdam -stateful for both build output and run input, so running it no longer needs an EEST_FIXTURES_DIR override. Shared state_actor + runner client wiring (genesis overrides, datadirs, instances) is identical in both. Verified: compute geth 830/0; stateful geth 8/8. --- ...-actor-eest.simple.amsterdam.compute.yaml} | 46 +-- ...-actor-eest.simple.amsterdam.stateful.yaml | 300 ++++++++++++++++++ 2 files changed, 314 insertions(+), 32 deletions(-) rename examples/configuration/{config.state-actor-eest.amsterdam.yaml => config.state-actor-eest.simple.amsterdam.compute.yaml} (88%) create mode 100644 examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml diff --git a/examples/configuration/config.state-actor-eest.amsterdam.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml similarity index 88% rename from examples/configuration/config.state-actor-eest.amsterdam.yaml rename to examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml index 75a65a4c6..e42cf050b 100644 --- a/examples/configuration/config.state-actor-eest.amsterdam.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml @@ -1,24 +1,20 @@ -# Building data directories and EEST payloads — AMSTERDAM fill. +# Building data directories and EEST COMPUTE payloads — AMSTERDAM fill. # -# state-actor still generates OSAKA snapshot datadirs (stage 1); the eest_payloads -# stage then fills AMSTERDAM fixtures by booting the filler on the osaka snapshot -# and scheduling amsterdam to activate at the snapshot block's timestamp + 1 (via -# fork_activation_genesis → a geth --override.amsterdam= flag). So block 0 -# stays osaka and the first benchmark block (block 1) is amsterdam. +# state-actor generates OSAKA snapshot datadirs (stage 1); the eest_payloads stage +# fills AMSTERDAM tests/benchmark/compute fixtures by booting the geth filler on the +# osaka snapshot and activating amsterdam at the snapshot block's timestamp + 1 (via +# fork_activation_genesis → a geth --override.amsterdam= flag). So block 0 stays +# osaka and the first benchmark block (block 1) is amsterdam. # -# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.amsterdam.yaml --force +# Stateful (EIP-7928 / BAL) fixtures are built by the sibling config: +# config.state-actor-eest.simple.amsterdam.stateful.yaml +# Both configs build the same state-actor snapshot — build either one first. # -# Two fixture sets are built: tests/benchmark/compute (target payload-generator-geth, -# output eest-fixtures-amsterdam) and tests/benchmark/stateful EIP-7928/BAL (target -# payload-generator-geth-stateful, output eest-fixtures-amsterdam-stateful). +# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml --force # -# Running the COMPUTE benchmark (one instance per client): -# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=geth -# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=besu -# -# Running the STATEFUL (BAL) benchmark — point the runner at the stateful fixtures: -# EEST_FIXTURES_DIR=/tmp/benchmarkoor/eest-fixtures-amsterdam-stateful \ -# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.amsterdam.yaml --limit-instance-id=geth +# Running the benchmark (one instance per client): +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml --limit-instance-id=geth +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml --limit-instance-id=besu # # Each client activates amsterdam on the osaka snapshot one of three ways: # - geth, erigon: CLI fork override (--override.amsterdam=) @@ -86,7 +82,7 @@ builder: - client: erigon output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon - ## Stage 2: Building EEST stateful fixtures + ## Stage 2: Building EEST compute fixtures eest_payloads: # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead # of pulling a pre-built one. Alternatively set fill_image: . @@ -146,20 +142,6 @@ builder: and not test_block_full_data and not test_prefetch_cold_storage and not test_ext_account_query_cold - # Stateful EIP-7928 (BAL) benchmarks. Self-contained like the compute tests - # (each deploys its own contracts via pre.deploy_contract / pre.fund_eoa - # against the predeployed CREATE2 factory + funded seed), so no extra - # state-actor entities are needed. The bloatnet stateful tests are NOT built - # here: test_single_opcode / test_multi_opcode need giant pre-deployed ERC20s - # /EOAs (50GB..) supplied via --address-stubs, infeasible on this snapshot. - - name: payload-generator-geth-stateful - filler_client: geth - filler_image: skylenet/geth:bal-devnet-7-amsterdam-override - source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth - fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json - output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam}-stateful - tests: - - tests/benchmark/stateful/eip7928_block_level_access_lists # besu: testing_buildBlockV1 (TESTING namespace, besu-eth/besu#9838), # debug_setHead and the session-fee pin all work, but besu's self-built # block fails its own engine_newPayloadV4 with a World-State-Root mismatch diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml new file mode 100644 index 000000000..777eba3cb --- /dev/null +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml @@ -0,0 +1,300 @@ +# Building data directories and EEST STATEFUL (EIP-7928 / BAL) payloads — AMSTERDAM fill. +# +# state-actor generates OSAKA snapshot datadirs (stage 1); the eest_payloads stage +# fills AMSTERDAM tests/benchmark/stateful EIP-7928 (BAL) fixtures by booting the geth +# filler on the osaka snapshot and activating amsterdam at the snapshot block's +# timestamp + 1 (via fork_activation_genesis → a geth --override.amsterdam= flag). +# +# These BAL tests are self-contained (each deploys its own contracts via +# pre.deploy_contract / pre.fund_eoa against the predeployed CREATE2 factory + funded +# seed), so no extra state-actor entities are needed. The bloatnet stateful tests are +# NOT built here: test_single_opcode / test_multi_opcode need giant pre-deployed +# ERC20s/EOAs (50GB..) supplied via --address-stubs, infeasible on this snapshot. +# +# Compute fixtures are built by the sibling config: +# config.state-actor-eest.simple.amsterdam.compute.yaml +# Both configs build the same state-actor snapshot — build either one first. +# +# ./bin/benchmarkoor build --config examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml --force +# +# Running the benchmark (one instance per client): +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml --limit-instance-id=geth +# ./bin/benchmarkoor run --config examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml --limit-instance-id=besu +# +# Each client activates amsterdam on the osaka snapshot one of three ways: +# - geth, erigon: CLI fork override (--override.amsterdam=) +# - besu, reth, ethrex: genesis_fork_override (benchmarkoor sets Time in +# the geth-format genesis at boot) +# - nethermind: genesis_eip_override (benchmarkoor sets the amsterdam +# EIPs' eipTransitionTimestamp in the parity chainspec) +# Snapshot block 0's timestamp is 0, so amsterdam is scheduled at ts 1. +# +global: + log_level: info + +builder: + ## Stage 1: Building datadirs using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + reth: ghcr.io/ethereum/state-actor-reth:main + besu: ghcr.io/ethereum/state-actor-besu:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + ethrex: ghcr.io/ethereum/state-actor-ethrex:main + erigon: ghcr.io/ethereum/state-actor-erigon:main + pull_policy: always + config: + seed: 1234 + fork: osaka + chain: 1337 + gas_limit: 300000000 # 300M + target_size: 256MB + #target_size: 20GB + spec: | + entities: + # Predeploy the Arachnid deterministic-deployment proxy at the canonical + # 0x4e59…956c. EEST's fill bootstrap deploys it via a keyless tx with a + # fixed 100k gas limit, which amsterdam's state-expansion gas (EIP-7928/ + # 8037, ~324k for this proxy) exceeds — so it must be predeployed. EEST + # detects the existing bytecode and skips the keyless deploy. + - kind: contract + template: create2_factory + name: deterministic-deployment-proxy + # Explicit canonical address: with a name set, the address would + # otherwise be name-derived instead of the Arachnid default. + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + # Pre-fund fill-stateful's seed account (rpc_seed_key=0x..01 → this + # address). With the seed already funded, fill-stateful skips the + # withdrawal funding block, so start_block == the snapshot block, whose + # persistent state debug_setHead can always rewind to between tests. + # Balance must be >= SEED_FUNDING_WEI (1e27); 1e28 gives headroom. + - kind: eoa + name: fill-stateful-seed + address: 0x7e5f4552091a69125d5dfcb7b8c2659029395bdf + balance: "10000000000000000000000000000" + nonce: 0 + targets: + - client: geth + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + - client: reth + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth + - client: nethermind + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + - client: besu + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + - client: ethrex + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex + - client: erigon + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon + + ## Stage 2: Building EEST stateful (EIP-7928 / BAL) fixtures + eest_payloads: + # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead + # of pulling a pre-built one. Alternatively set fill_image: . + fill_dockerfile: Dockerfile.eest-filler + pull_policy: if-not-present + eest_repo: https://github.com/skylenet/execution-specs.git + eest_ref: devnets/bal/7-bench-cap-aware-deploy + config: + fork: amsterdam # fill amsterdam fixtures (snapshot stays osaka) + # Pin the seed key so its account (0x7e5f…bdf, privkey 1) can be pre-funded + # in the snapshot above — lets fill-stateful skip the withdrawal funding + # block so start_block == the snapshot block (stable per-test rewind). + rpc_seed_key: "0x0000000000000000000000000000000000000000000000000000000000000001" + gas_benchmark_values: [10] # Millions of gas to parametrise against + datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) + # amsterdam fill: filler_image must be a geth that registers --override.amsterdam + # (bal-devnet-7 + the amsterdam-override patch). fork_activation_genesis reads + # the snapshot block's timestamp and the builder boots the filler with + # --override.amsterdam=. + targets: + - name: payload-generator-geth-stateful + filler_client: geth + filler_image: skylenet/geth:bal-devnet-7-amsterdam-override + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + # Schedule amsterdam at the osaka snapshot block 0's timestamp + 1. + fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam-stateful} + tests: + - tests/benchmark/stateful/eip7928_block_level_access_lists + +## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads +runner: + client_logs_to_stdout: true + cleanup_on_start: false + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam-stateful} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + # Booting from a snapshot datadir leaves the Engine API forkchoice head + # un-established, so clients answer SYNCING to every newPayload. A bootstrap + # FCU to the snapshot head makes them canonical and ready to validate the + # replayed payloads (required for besu/reth/nethermind; geth is fine without). + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + genesis: + # nethermind reads a parity-format chainspec that schedules forks per-EIP, + # so amsterdam is activated via the per-instance genesis_eip_override below + # (not genesis_fork_override). The osaka chainspec is used as-is. + nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + #geth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json + # besu/reth/ethrex have no --override.amsterdam flag — they read forks + # from the genesis. benchmarkoor patches it for them at boot via the + # per-instance genesis_fork_override below (sets amsterdamTime + inherits + # the amsterdam blobSchedule), so the osaka chainspec is used as-is. + besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json + # erigon consumes a geth-format genesis via its init container. Filename + # follows the per-client state-actor convention (geth-genesis.json, + # ethrex-genesis.json); confirm/adjust once state-actor publishes the + # erigon image. + erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/chainspec.json + #erigon: /tmp/benchmarkoor/state-actor/geth/geth-genesis.json + + datadirs: + geth: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + method: copy + reth: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth + method: copy + nethermind: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + method: copy + besu: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + method: copy + ethrex: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex + method: copy + erigon: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon + method: copy + + instances: + - id: geth + client: geth + image: skylenet/geth:bal-devnet-7-amsterdam-override + extra_args: + # Activate amsterdam at snapshot block 0's timestamp (0) + 1. The snapshot + # is osaka and geth boots from the datadir (its genesis is commented out + # above), so without this the amsterdam payloads (newPayloadV5) would be + # rejected. Needs the amsterdam-override image; a stock geth ignores it. + - --override.amsterdam=1 + - id: reth + client: reth + # bal-devnet-7 is the reth line with amsterdam/BAL support. reth reads forks + # from its chainspec, so patch amsterdam in at boot. + image: ethpandaops/reth:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 + extra_args: + # reth-bal-full knobs (mirror benchmarkoor-tests): "full" is the default + # BAL mode, so only the slow-block log threshold is set. + - --engine.slow-block-threshold=0 + # Booting from a snapshot, reth can answer the first newPayload before the + # forkchoice head is established; retry the non-SYNCING failure briefly. + retry_new_payloads_failed_state: + enabled: true + max_retries: 10 + backoff: 1s + - id: nethermind + client: nethermind + image: nethermindeth/nethermind:master + # nethermind's parity chainspec schedules forks per-EIP, so activate the + # amsterdam-devnet-7 EIP set at snapshot block 0's ts (0) + 1. benchmarkoor + # patches params.eipTransitionTimestamp into the chainspec at boot. + genesis_eip_override: + timestamp: 1 + eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] + extra_args: + # benchmarkoor mounts the datadir at /data and the nethermind client + # passes --datadir=/data, but nethermind's BaseDbPath then defaults to + # /data/nethermind_db// — so it reads a fresh EMPTY state db and + # ignores the state-actor snapshot written directly under /data/{state, + # blocks,headers,blockInfos}. It then builds genesis from the empty-alloc + # chainspec, so HasStateForBlock(genesis) is false and every replayed + # newPayload answers SYNCING. Pointing BaseDbPath at /data makes nethermind + # read the snapshot directly (it then logs "Detected HalfPath key scheme"). + - --Init.BaseDbPath=/data + # nethermind-bal-full knobs (mirror benchmarkoor-tests). + - --Blocks.ParallelExecution=true + - --Blocks.ParallelExecutionBatchRead=true + - id: besu + client: besu + # bal-devnet-7 is the besu line with amsterdam/BAL (EIP-7928) support. + image: ethpandaops/besu:bal-devnet-7 + # besu reads forks from the genesis, so activate amsterdam at snapshot + # block 0's ts (0) + 1 by patching its chainspec at boot. + genesis_fork_override: + amsterdam: 1 + environment: + BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" + extra_args: + # besu needs --p2p-enabled=true to accept the bootstrap FCU on a snapshot: + # its synchronizer must run to register the post-merge head as in-sync, + # otherwise besu answers SYNCING. --max-peers=0 + --discovery-enabled=false + # (the client defaults) keep it isolated — no real peers. + - --p2p-enabled=true + - --Xplugin-rocksdb-high-spec-enabled=true + # BAL "full" mode — mirrors besu-bal-full in ethpandaops/benchmarkoor-tests. + - --Xbal-perfect-parallelization-enabled=true + - --Xbal-state-root-enabled=true + - --Xbal-prefetch-reading-enabled=true + - --Xbal-prefetch-batch-size=8 + - --Xbal-processing-timeout=-1 + - --Xbal-state-root-timeout=-1 + - id: ethrex + client: ethrex + # bal-devnet-7 is the ethrex line with amsterdam/BAL support. ethrex reads + # forks from its genesis, so patch amsterdam in at boot. + image: ethpandaops/ethrex:bal-devnet-7 + genesis_fork_override: + amsterdam: 1 + extra_args: + # --skip-genesis-validation: the state-actor snapshot commits a synthetic + # state root the emitted ethrex-genesis.json (empty alloc) can't reproduce. + - --skip-genesis-validation + # ethrex-bal-full knob (mirrors benchmarkoor-tests). + - --no-precompile-cache + - id: erigon + client: erigon + # MUST match the erigon that state-actor pins for the snapshot + # (Dockerfile.erigon builds erigon at the bal-devnet-7 commit and notes the + # daemon must use that exact binary). state-actor's `erigon init` encodes the + # MDBX chain config the bal-devnet-7 way (e.g. chainId as a JSON string); + # stable erigon (erigontech/erigon:latest, 3.4.x) panics reading it back with + # "cannot unmarshal \"1337\" into a *big.Int". ethpandaops/erigon:bal-devnet-7 + # reads it correctly and supports the amsterdam/BAL (EIP-7928) fork these + # benchmarks target. + image: ethpandaops/erigon:bal-devnet-7 + extra_args: + # erigon supports a geth-style fork override. Snapshot block 0 ts is 0, so + # activate amsterdam at ts 1 (the reference devnet uses its real ts). + # NB: genesis_fork_override does NOT work for erigon — like geth it reads + # the fork schedule from the datadir, not the genesis file, so the flag is + # required (a patched genesis is ignored and amsterdam stays inactive). + - --override.amsterdam=1 + # erigon-bal-full knobs (mirror benchmarkoor-tests): keep execution + # deterministic/online by disabling background merge/prune/maintenance and + # synchronous FCU commit so replayed payloads are validated immediately. + - --fcu.background.commit=false + - --exec.no-merge=true + - --exec.no-prune=true + - --exec.no-background-maintenance + # erigon is slow to register the snapshot head as in-sync; give the bootstrap + # FCU a long runway (mirrors the reference erigon-bal-full). + bootstrap_fcu: + enabled: true + max_retries: 120 + backoff: 30s From 00f9440971a8024a97022471e0e12708cadf31dc Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 14:23:27 +0200 Subject: [PATCH 32/83] chore(hack): add orbstack.sh to provision a Linux VM for COW datadirs macOS has no overlayfs/zfs/schelk, so large stateful prestates can't use a copy-on-write datadir method and `copy` is infeasible at scale. This script provisions an OrbStack Linux machine with a dedicated dockerd (so benchmarkoor, dockerd, and the overlay mounts share one filesystem namespace), installs Go, builds benchmarkoor natively, and self-checks that overlayfs + the VM dockerd bind-mount works. Validated end-to-end against the eip7928 (BAL) stateful set: build + run with datadir method overlayfs, geth 8/8. --- .hack/orbstack.sh | 101 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 .hack/orbstack.sh diff --git a/.hack/orbstack.sh b/.hack/orbstack.sh new file mode 100755 index 000000000..0fef205de --- /dev/null +++ b/.hack/orbstack.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# Provision an OrbStack Linux machine to run benchmarkoor with a copy-on-write +# datadir method (overlayfs). macOS has no overlayfs/zfs/schelk, so the large +# stateful prestates (e.g. config.state-actor-eest.full.amsterdam.stateful.yaml, +# ~77 GB) can't use anything but `copy` on the host — which is infeasible at that +# size. An OrbStack Linux machine has a real kernel (overlayfs works) and we run +# a DEDICATED dockerd inside it so benchmarkoor, dockerd, and the overlay mounts +# all share one filesystem namespace (the merged dir must be bind-mountable into +# the client containers — the host's shared Docker engine can't see VM-local +# overlay mounts). +# +# Prereqs: OrbStack installed (https://orbstack.dev). The repo is auto-shared +# into the machine at the same path, so no copying is needed. +# +# Usage: +# .hack/orbstack.sh [machine-name] # default: bench +# +# After it finishes, run benchmarkoor INSIDE the machine as root (overlayfs mount +# needs CAP_SYS_ADMIN), e.g.: +# orb -m bench sudo benchmarkoor build \ +# --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --force +# orb -m bench sudo benchmarkoor run \ +# --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --limit-instance-id=geth +set -euo pipefail + +MACHINE="${1:-bench}" +DISTRO="${ORBSTACK_DISTRO:-ubuntu}" +# Build tags mirror the Makefile (avoid btrfs/devicemapper graphdrivers + gpgme). +GO_BUILD_TAGS="exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper,containers_image_openpgp" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if ! command -v orb >/dev/null 2>&1; then + echo "error: 'orb' not found — install OrbStack first (https://orbstack.dev)" >&2 + exit 1 +fi + +echo "==> Repo root: ${REPO_ROOT}" +echo "==> OrbStack machine: ${MACHINE} (${DISTRO})" + +# 1. Create the machine if it doesn't already exist (idempotent). +if orb list 2>/dev/null | awk '{print $1}' | grep -qx "${MACHINE}"; then + echo "==> Machine '${MACHINE}' already exists, reusing it." +else + echo "==> Creating machine '${MACHINE}'..." + orb create "${DISTRO}" "${MACHINE}" +fi + +# 2. Provision inside the machine: dedicated dockerd + Go toolchain, then build +# benchmarkoor natively. Everything below runs as root in the machine. +orb -m "${MACHINE}" sudo bash -s -- "${REPO_ROOT}" "${GO_BUILD_TAGS}" <<'PROVISION' +set -euo pipefail +REPO_ROOT="$1" +GO_BUILD_TAGS="$2" + +echo "==> apt: docker.io + golang-go" +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq docker.io golang-go + +echo "==> Starting the machine's own dockerd" +systemctl enable --now docker +systemctl is-active --quiet docker || { echo "dockerd failed to start" >&2; exit 1; } + +# A dedicated, VM-local docker daemon is the whole point — confirm it. +docker version --format ' dockerd {{.Server.Version}}' >/dev/null + +echo "==> Go: $(go version)" +case "$(go version)" in + *go1.2[3-9]*|*go1.[3-9][0-9]*|*go[2-9].*) : ;; + *) echo "warning: Go < 1.23 detected; benchmarkoor needs >= 1.23. Install a newer Go." >&2 ;; +esac + +echo "==> Building benchmarkoor (native, tags: ${GO_BUILD_TAGS})" +cd "${REPO_ROOT}" +GOCACHE=/root/.cache/go-build GOPATH=/root/go GOFLAGS=-mod=mod \ + go build -tags "${GO_BUILD_TAGS}" -o /usr/local/bin/benchmarkoor ./cmd/benchmarkoor +/usr/local/bin/benchmarkoor --help >/dev/null && echo "==> benchmarkoor installed at /usr/local/bin/benchmarkoor" + +echo "==> Verifying overlayfs works in this machine" +d="$(mktemp -d)"; mkdir -p "$d"/{low,up,work,merged}; echo ok > "$d/low/f" +mount -t overlay overlay -o "lowerdir=$d/low,upperdir=$d/up,workdir=$d/work" "$d/merged" +docker run --rm -v "$d/merged:/m" alpine cat /m/f >/dev/null && echo "==> overlayfs + dockerd bind-mount: OK" +umount "$d/merged"; rm -rf "$d" +PROVISION + +cat < Done. The '${MACHINE}' machine is ready (dedicated dockerd + overlayfs + benchmarkoor). + +Run benchmarkoor inside it as root (the repo is shared at ${REPO_ROOT}): + + orb -m ${MACHINE} sudo benchmarkoor build \\ + --config ${REPO_ROOT}/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --force + + orb -m ${MACHINE} sudo benchmarkoor run \\ + --config ${REPO_ROOT}/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --limit-instance-id=geth + +Tear down with: orb delete ${MACHINE} +EOF From 51a9851820ce1769ec1345b440a771feab7313a6 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 19:24:44 +0200 Subject: [PATCH 33/83] feat(gitrepo): fetch submodules after clone/checkout Some EEST benchmark suites vendor fixtures as git submodules (e.g. tests/benchmark/stateful/bloatnet/depth_benchmarks/.worst_case_miner), and fail at fill time with "run git submodule update --init --recursive" when the submodule isn't present. Run that after every clone/checkout in CloneOrUpdate (shared by the EEST fill-repo clone and git test-sources). It's a no-op for repos without submodules, so all callers are safe. --- pkg/gitrepo/gitrepo.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg/gitrepo/gitrepo.go b/pkg/gitrepo/gitrepo.go index c00278ca0..fc88c75f3 100644 --- a/pkg/gitrepo/gitrepo.go +++ b/pkg/gitrepo/gitrepo.go @@ -48,6 +48,10 @@ func CloneOrUpdate(ctx context.Context, log logrus.FieldLogger, repo, version, c return "", fmt.Errorf("cloning repository: %w", err) } + if err := initSubmodules(ctx, localPath); err != nil { + return "", err + } + return localPath, nil } @@ -57,6 +61,10 @@ func CloneOrUpdate(ctx context.Context, log logrus.FieldLogger, repo, version, c if sha, err := HeadSHA(ctx, localPath); err == nil && strings.HasPrefix(sha, version) { log.Info("Cached repository already at requested version") + if err := initSubmodules(ctx, localPath); err != nil { + return "", err + } + return localPath, nil } } @@ -71,6 +79,10 @@ func CloneOrUpdate(ctx context.Context, log logrus.FieldLogger, repo, version, c return "", fmt.Errorf("checking out version: %w", err) } + if err := initSubmodules(ctx, localPath); err != nil { + return "", err + } + return localPath, nil } @@ -106,6 +118,19 @@ func cloneByCommitHash(ctx context.Context, repo, version, localPath string) err return nil } +// initSubmodules fetches any git submodules declared by the checkout. It is a +// no-op for repos without submodules, so it is safe to call after every +// clone/checkout (some EEST test suites vendor fixtures as submodules and fail +// at fill time with "run git submodule update" if they are absent). +func initSubmodules(ctx context.Context, localPath string) error { + if err := gitRun(ctx, "git", "-C", localPath, + "submodule", "update", "--init", "--recursive"); err != nil { + return fmt.Errorf("updating submodules: %w", err) + } + + return nil +} + // gitRun runs a git command, forwarding its output to the process stdio. func gitRun(ctx context.Context, name string, args ...string) error { cmd := exec.CommandContext(ctx, name, args...) From 203c601b1010455fbd19053a16bc279a758aa49d Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 19:35:30 +0200 Subject: [PATCH 34/83] feat(examples): full bloatnet/repricing amsterdam stateful config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add config.state-actor-eest.full.amsterdam.stateful.yaml + the stubs-repricing-amsterdam.json it references. Builds jochem-brouwer's ~77GB production-minimum repricing prestate via state-actor and fills the tests/benchmark/stateful/bloatnet repricing benchmarks against it. Requires a Linux env with a copy-on-write datadir method (overlayfs) — provision with .hack/orbstack.sh; copy is infeasible at 77GB. geth-only by default (each extra client is another ~77GB snapshot). The filter excludes the tests this snapshot/fork can't satisfy, each documented inline: missing giant ERC20/factory stubs, the 120k-contract state-builder, the deep-MPT tests (block gas limit), an unfunded sender pool, and amsterdam-vs-glamsterdam repricing divergences (param-scoped so passing cases are kept). Validated in an OrbStack VM: 207 filled, 0 failed. --- ...te-actor-eest.full.amsterdam.stateful.yaml | 265 ++++++++++++++++++ .../stubs-repricing-amsterdam.json | 6 + 2 files changed, 271 insertions(+) create mode 100644 examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml create mode 100644 examples/configuration/stubs-repricing-amsterdam.json diff --git a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml new file mode 100644 index 000000000..641682f6f --- /dev/null +++ b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml @@ -0,0 +1,265 @@ +# Building data directories and EEST FULL STATEFUL (bloatnet / repricing) payloads — AMSTERDAM fill. +# +# This is the "full" counterpart of config.state-actor-eest.simple.amsterdam.stateful.yaml: +# a large repricing prestate (jochem-brouwer's production-minimum spec, +# https://gist.github.com/jochem-brouwer/5da19ef1e96edcf645b3dd590b5604bd) that +# backs the tests/benchmark/stateful/bloatnet repricing benchmarks. The geth +# snapshot alone is ~77 GB on disk and takes 30-90 min to build. +# +# ── Run this in a Linux env with a COW datadir method (overlayfs/zfs/schelk) ── +# macOS has none of these, and `copy` is infeasible at 77 GB (copied per fill and +# per client per run). Provision an OrbStack Linux machine with .hack/orbstack.sh +# (dedicated dockerd + overlayfs + a native benchmarkoor build), then run as root +# INSIDE the machine (overlayfs mount needs CAP_SYS_ADMIN): +# +# .hack/orbstack.sh bench +# # one-off: make the address stubs available at the path referenced below +# orb -m bench sudo mkdir -p /tmp/benchmarkoor +# orb -m bench sudo cp $(pwd)/examples/configuration/stubs-repricing-amsterdam.json /tmp/benchmarkoor/ +# orb -m bench sudo benchmarkoor build --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --force +# orb -m bench sudo benchmarkoor run --config $(pwd)/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml --limit-instance-id=geth +# +# Amsterdam is activated exactly as in the other configs: geth/erigon via the +# --override.amsterdam flag, besu/reth/ethrex via genesis_fork_override, +# nethermind via genesis_eip_override. (Per the assume-Amsterdam decision; the +# gist itself notes the repricing actually ships in glamsterdam.) +# +# NB: only geth is enabled by default — each extra client needs its OWN ~77 GB +# snapshot (state-actor builds one per target), so 6 clients ≈ 460 GB. Uncomment +# the others once you have the disk. +# +# STATUS: the overlayfs pipeline is validated end-to-end on the small eip7928 +# stateful set; this full bloatnet variant is a starting point — the exact +# test/stub coverage (which bloatnet tests the gist prestate satisfies) still +# needs iteration in the VM. See the filter note on the geth target. +# +global: + log_level: info + +builder: + ## Stage 1: Building the large repricing datadir using state-actor + state_actor: + images: + geth: ghcr.io/ethereum/state-actor:main + # reth: ghcr.io/ethereum/state-actor-reth:main + # besu: ghcr.io/ethereum/state-actor-besu:main + # nethermind: ghcr.io/ethereum/state-actor-nethermind:main + # ethrex: ghcr.io/ethereum/state-actor-ethrex:main + # erigon: ghcr.io/ethereum/state-actor-erigon:main + pull_policy: always + config: + seed: 42 # gist uses --seed=42 (reproducible addresses) + fork: osaka # snapshot baseline (amsterdam is forked in at fill/run) + chain: 1337 + # gist minimum is 85M (an amsterdam max-size code deploy needs gas_limit + # >= 67M); 300M leaves room for larger benchmark blocks too. + gas_limit: 300000000 + # Must exceed the prestate size (~77 GB) or state-actor truncates entities. + target_size: 200GB + # jochem-brouwer's production-minimum repricing prestate. The big cost is + # entity "storage-pattern-10gb" (335M slots ≈ 73 GB); the 150000-count knobs + # give 300M-gas cold-access headroom (count >= gas/2000). + spec: | + entities: + # Pre-funded seed for fill-stateful — Hardhat/Anvil dev key #0 + # (pkey 0xac0974…ff80). Pinned below via rpc_seed_key. + - kind: eoa + name: fill-stateful-seed + address: 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 + balance: "0x33b2e3c91efc989409c0000" + # Sequential EOAs at 0x1000+ (test_account_access EXISTING_EOA). + - kind: contract + name: sequential-eoas-300m + template: sequential_eoas + address: 0x0000000000000000000000000000000000001000 + parameters: + count: 150000 + balance: "1000000000000000000" + # 10GB storage-pattern target at the bloated_eoa_10GB stub address + # (test_sload_bloated / test_sstore_bloated, existing_slots=True). ~73 GB. + - kind: contract + name: storage-pattern-10gb + template: storage_pattern + address: 0x87a6314da5ac8832f6e7a176c8fb133b19f5be04 + nonce: 1 + balance: "1000000000000000000" + parameters: + final: 335545082 + # Arachnid CREATE2 factory (required by create2_deploys + EEST fill). + - kind: contract + name: create2-factory + template: create2_factory + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + # 150000 CREATE2 contracts with UNIQUE 24KB runtime (cold-contract access + # / test_transaction_types). ~3.7 GB resident code during build. + - kind: contract + name: create2-deploys-300m + template: create2_deploys + parameters: + code_pattern: unique_jumpdest_pre_amsterdam + salt_count: 150000 + # Sender pool for test_ether_transfers_onchain_receivers + # (EOA(key=SENDER_BASE_KEY + i), 150000 funded). + - kind: contract + name: sender-pool-300m + template: sequential_pkey_eoas + parameters: + start_pkey: "0x1111111111111111111111111111111111111111111111111111111111111111" + count: 150000 + balance: "1000000000000000000" + # Bittrex CREATE-preimage chain (test_account_access EXISTING_CONTRACT). + - kind: contract + name: bittrex-create-preimage-300m + template: create_preimage_deploys + parameters: + sender: "0xA3C1E324CA1CE40DB73ED6026C4A177F099B5770" + start_nonce: 2 + count: 150000 + runtime: "0x606060405236156100495763ffffffff7c01000000000000000000000000000000000000000000000000000000006000350416636ea056a98114610052578063c0ee0b8a14610092575b6100505b5b565b005b341561005a57fe5b61007e73ffffffffffffffffffffffffffffffffffffffff60043516602435610104565b604080519115158252519081900360200190f35b341561009a57fe5b604080516020600460443581810135601f810184900484028501840190955284845261005094823573ffffffffffffffffffffffffffffffffffffffff169460248035956064949293919092019181908401838280828437509496506101ef95505050505050565b005b6000805460408051602090810184905281517f3c18d31800000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff878116600483015292519290931692633c18d318926024808301939282900301818787803b151561017b57fe5b6102c65a03f1151561018957fe5b5050506040518051905073ffffffffffffffffffffffffffffffffffffffff1660003660006040516020015260405180838380828437820191505092505050602060405180830381856102c65a03f415156101e057fe5b50506040515190505b92915050565b5b5050505600a165627a7a723058204cdd69fdcf3cf6cbee9677fe380fa5f044048aa9e060ec5619a21ca5a5bd4cd10029" + storage_init: + "0x0": "0xa3c1e324ca1ce40db73ed6026c4a177f099b5770" + targets: + - client: geth + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + # - client: reth # +~77 GB each — enable only with disk to spare + # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth + # - client: nethermind + # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + # - client: besu + # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + # - client: ethrex + # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex + # - client: erigon + # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon + + ## Stage 2: Building EEST bloatnet/repricing fixtures + eest_payloads: + fill_dockerfile: Dockerfile.eest-filler + pull_policy: if-not-present + eest_repo: https://github.com/skylenet/execution-specs.git + eest_ref: devnets/bal/7-bench-cap-aware-deploy + config: + fork: amsterdam + # Hardhat/Anvil dev key #0 → 0xf39Fd6…2266 (the pre-funded seed above). + rpc_seed_key: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + # The bloatnet prestate has 300M-gas headroom; start lower for faster fills. + gas_benchmark_values: [30] + # COW is mandatory at this size — overlayfs (Linux/OrbStack). On the host + # this would be `copy` (infeasible). See .hack/orbstack.sh. + datadir_method: overlayfs + targets: + - name: payload-generator-geth-stateful-full + filler_client: geth + filler_image: skylenet/geth:bal-devnet-7-amsterdam-override + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam-bloatnet} + # Resolves @pytest.mark.stub_parametrize prefixes against pre-deployed + # state (e.g. bloated_eoa_10GB → 0x87a6…be04). Copy the committed + # stubs-repricing-amsterdam.json here (see header). Must be absolute. + address_stubs_file: ${EEST_STUBS_FILE:-/tmp/benchmarkoor/stubs-repricing-amsterdam.json} + tests: + - tests/benchmark/stateful/bloatnet + # This prestate provides the bloated_eoa_10GB stub + the template-backed + # pre-state (sequential EOAs, CREATE2 contracts, sender pool, Bittrex + # chain). Exclusions: + # erc20 / test_multi_opcode (bloatnet_factory_): need giant ERC20 / + # factory stubs this prestate doesn't provide (require_stub_match). + # test_setup_contracts: deploys 120k contracts from scratch in one + # fixture (~120k blocks) — a state-builder, not a quick benchmark. + # test_deep_branch: deploys worst-case deep MPT contracts at fill time + # whose blocks exceed the 300M genesis gas limit (EIP-1559 caps the + # per-block ramp, so the limit can't jump). Needs a much higher + # genesis gas_limit (bigger snapshot) or prestate-deployed mined + # contracts — see the worst_case_miner submodule. + # test_ether_transfers_onchain_receivers: the EOA(SENDER_BASE_KEY+i) + # sender pool the test expects funded doesn't match what state-actor's + # sequential_pkey_eoas plants (an address resolves to 0 balance). + # test_extcodesize_bytecode_sizes, the zero_to_nonzero sstore variants, + # test_sload_bloated_prefetch_miss, and the existing_slots_True + # multi_contract cases: amsterdam gas/BAL divergence (these target the + # post-amsterdam repricing; we fill against amsterdam). + # The two param-scoped excludes keep this functions' passing cases (12 + # sstore_variants, 2 multi_contract). Result: 207 filled, 0 failed. + filter: >- + not erc20 + and not test_multi_opcode + and not test_setup_contracts + and not test_deep_branch + and not test_ether_transfers_onchain_receivers + and not test_extcodesize_bytecode_sizes + and not test_sload_bloated_prefetch_miss + and not (test_sstore_variants and zero_to_nonzero) + and not (test_sload_bloated_multi_contract and existing_slots_True) + +## Stage 3: Run the bloatnet/repricing benchmarks +runner: + client_logs_to_stdout: true + cleanup_on_start: false + benchmark: + results_dir: ./results + generate_results_index: true + generate_suite_stats: true + tests: + source: + eest_fixtures: + local_fixtures_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam-bloatnet} + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + # genesis entries are only needed for the non-geth clients (commented out + # below); add them back alongside their instances + datadirs. + # nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + # reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json + # besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + # ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json + # erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/chainspec.json + datadirs: + geth: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + method: overlayfs + # reth: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth, method: overlayfs } + # nethermind: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind, method: overlayfs } + # besu: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu, method: overlayfs } + # ethrex: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex, method: overlayfs } + # erigon: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon, method: overlayfs } + + instances: + - id: geth + client: geth + image: skylenet/geth:bal-devnet-7-amsterdam-override + extra_args: + - --override.amsterdam=1 + # Other clients need their own ~77 GB snapshot (state_actor target above) + + # their genesis/override wiring — mirror config.state-actor-eest.simple.amsterdam.stateful.yaml. + # - id: reth + # client: reth + # image: ethpandaops/reth:bal-devnet-7 + # genesis_fork_override: { amsterdam: 1 } + # extra_args: [--engine.slow-block-threshold=0] + # retry_new_payloads_failed_state: { enabled: true, max_retries: 10, backoff: 1s } + # - id: nethermind + # client: nethermind + # image: nethermindeth/nethermind:master + # genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + # extra_args: [--Init.BaseDbPath=/data, --Blocks.ParallelExecution=true, --Blocks.ParallelExecutionBatchRead=true] + # - id: besu + # client: besu + # image: ethpandaops/besu:bal-devnet-7 + # genesis_fork_override: { amsterdam: 1 } + # environment: { BESU_OPTS: "-Xms8g -Xmx8g -XX:+AlwaysPreTouch" } + # extra_args: [--p2p-enabled=true, --Xplugin-rocksdb-high-spec-enabled=true, --Xbal-perfect-parallelization-enabled=true, --Xbal-state-root-enabled=true, --Xbal-prefetch-reading-enabled=true, --Xbal-prefetch-batch-size=8, --Xbal-processing-timeout=-1, --Xbal-state-root-timeout=-1] + # - id: ethrex + # client: ethrex + # image: ethpandaops/ethrex:bal-devnet-7 + # genesis_fork_override: { amsterdam: 1 } + # extra_args: [--skip-genesis-validation, --no-precompile-cache] + # - id: erigon + # client: erigon + # image: ethpandaops/erigon:bal-devnet-7 + # extra_args: [--override.amsterdam=1, --fcu.background.commit=false, --exec.no-merge=true, --exec.no-prune=true, --exec.no-background-maintenance] + # bootstrap_fcu: { enabled: true, max_retries: 120, backoff: 30s } diff --git a/examples/configuration/stubs-repricing-amsterdam.json b/examples/configuration/stubs-repricing-amsterdam.json new file mode 100644 index 000000000..a52d9d9b7 --- /dev/null +++ b/examples/configuration/stubs-repricing-amsterdam.json @@ -0,0 +1,6 @@ +{ + "bloated_eoa_10GB": { + "addr": "0x87a6314da5ac8832f6e7a176c8fb133b19f5be04", + "pkey": "0x4da32d29f6dcffa26e09dc4e102033f2d105de1444fb893493ae703289275e0e" + } +} From 6ffeebcc1514ceb8e405b50f09497cfc5343f1e8 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 19:53:40 +0200 Subject: [PATCH 35/83] fix(executor): cap over-long test result path components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EEST benchmark node ids — especially the verbose stateful/bloatnet params (AccountMode/CacheStrategy/... combinations) — can exceed the 255-byte per-filename limit, so creating the per-test results directory failed with "file name too long" and aborted the whole run. Add sanitizeResultPath, which truncates any "/"-separated path component over 200 bytes and appends a short content hash for uniqueness, applied at every site where a test name becomes a filesystem path (suite output, step results, post-test dir). Components within the limit are returned unchanged, so existing layouts don't move; the result.json still records the full test name and aggregation walks the tree, so truncating the directory name is safe. --- pkg/executor/executor.go | 2 +- pkg/executor/results.go | 38 +++++++++++++++++++++++++++++- pkg/executor/sanitize_path_test.go | 31 ++++++++++++++++++++++++ pkg/executor/suite.go | 4 ++-- 4 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 pkg/executor/sanitize_path_test.go diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 4c305f8e4..1126858e0 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -1761,7 +1761,7 @@ func executeSimpleRPC(ctx context.Context, endpoint, payload string) (string, er func (e *executor) dumpPostTestResponse( resultsDir, testName, filename, response string, ) error { - postTestDir := filepath.Join(resultsDir, testName, "post_test_rpc_calls") + postTestDir := filepath.Join(resultsDir, sanitizeResultPath(testName), "post_test_rpc_calls") if err := fsutil.MkdirAll(postTestDir, 0755, e.cfg.ResultsOwner); err != nil { return fmt.Errorf("creating post_test_rpc_calls directory: %w", err) } diff --git a/pkg/executor/results.go b/pkg/executor/results.go index caea97026..5397024f7 100644 --- a/pkg/executor/results.go +++ b/pkg/executor/results.go @@ -1,6 +1,8 @@ package executor import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "maps" @@ -13,6 +15,40 @@ import ( "github.com/ethpandaops/benchmarkoor/pkg/fsutil" ) +// maxResultPathComponent caps each path component of a test result directory +// below the common 255-byte filename limit, leaving headroom for suffixes like +// ".request". EEST benchmark node ids — especially the verbose stateful/bloatnet +// params — can exceed the limit and make MkdirAll fail with "file name too long". +const maxResultPathComponent = 200 + +// sanitizeResultPath caps each "/"-separated component of a test result path to +// maxResultPathComponent bytes. Over-long components are truncated and suffixed +// with a short content hash so they stay unique and stable; components within +// the limit are returned unchanged (so existing layouts don't move). Apply it +// consistently wherever a test name becomes a filesystem path (suite output, +// step results, post-test dir) so the directories match. The result.json inside +// still records the full test name, and result aggregation walks the tree, so +// truncating the directory name is safe. +func sanitizeResultPath(name string) string { + parts := strings.Split(name, "/") + + changed := false + + for i, p := range parts { + if len(p) > maxResultPathComponent { + sum := sha256.Sum256([]byte(p)) + parts[i] = p[:maxResultPathComponent-17] + "-" + hex.EncodeToString(sum[:8]) + changed = true + } + } + + if !changed { + return name + } + + return strings.Join(parts, "/") +} + // MethodStats contains aggregated statistics for a single method (int64 values). type MethodStats struct { Count int64 `json:"count"` @@ -570,7 +606,7 @@ func WriteStepResults( owner *fsutil.OwnerConfig, ) error { // Ensure the test directory exists. - testDir := filepath.Join(resultDir, testName) + testDir := filepath.Join(resultDir, sanitizeResultPath(testName)) if err := fsutil.MkdirAll(testDir, 0755, owner); err != nil { return fmt.Errorf("creating test result directory: %w", err) } diff --git a/pkg/executor/sanitize_path_test.go b/pkg/executor/sanitize_path_test.go new file mode 100644 index 000000000..b297e5c46 --- /dev/null +++ b/pkg/executor/sanitize_path_test.go @@ -0,0 +1,31 @@ +package executor + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeResultPath(t *testing.T) { + // Short components are returned unchanged. + short := "benchmark/stateful/bloatnet/test_x.py::test_y[fork_Amsterdam]" + assert.Equal(t, short, sanitizeResultPath(short)) + + // An over-long leaf component is truncated to the cap and suffixed with a hash. + longLeaf := strings.Repeat("a", 400) + name := "benchmark/bloatnet/" + longLeaf + out := sanitizeResultPath(name) + parts := strings.Split(out, "/") + leaf := parts[len(parts)-1] + assert.Len(t, leaf, maxResultPathComponent) + assert.Equal(t, "benchmark/bloatnet/", strings.Join(parts[:len(parts)-1], "/")+"/") + + // Distinct long names map to distinct sanitized paths (hash uniqueness). + a := sanitizeResultPath("p/" + strings.Repeat("a", 300) + "X") + b := sanitizeResultPath("p/" + strings.Repeat("a", 300) + "Y") + assert.NotEqual(t, a, b) + + // Deterministic. + assert.Equal(t, out, sanitizeResultPath(name)) +} diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 53a07445b..3d16c6e94 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -207,7 +207,7 @@ func CreateSuiteOutput( } // Create test directory. - testDir := filepath.Join(suiteDir, test.Name) + testDir := filepath.Join(suiteDir, sanitizeResultPath(test.Name)) if err := fsutil.MkdirAll(testDir, 0755, owner); err != nil { return fmt.Errorf("creating test dir for %s: %w", test.Name, err) } @@ -318,7 +318,7 @@ func CreateSuiteOutput( mergeOpcodeData(existing.Tests, prepared) lineProvider := func(testName string, step StepKind) []string { - reqPath := filepath.Join(suiteDir, testName, string(step)+".request") + reqPath := filepath.Join(suiteDir, sanitizeResultPath(testName), string(step)+".request") data, err := os.ReadFile(reqPath) if err != nil { // Missing files are normal — most tests don't have setup/cleanup. From bb6c8aa47489c52836630eea23a1b098ab898c69 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 20:19:37 +0200 Subject: [PATCH 36/83] refactor: align eest_payloads genesis/fork config with the runner Make the eest_payloads (filler) genesis vocabulary identical to the runner's: - Rename target `genesis_file` -> `genesis` (the boot genesis besu/ nethermind read forks from). - Drop `fork_activation_genesis` (which auto-derived a geth --override.= flag). geth/erigon fillers now pass an explicit --override.amsterdam in `filler_extra_args`, exactly like the runner's geth/erigon instances. Snapshot block 0 ts is always 0, so the derived ts+1 was just `1`. - Add `genesis_fork_override` / `genesis_eip_override` to eest_payloads targets, patching the boot genesis at filler boot for besu/reth/ethrex (geth-format) and nethermind (parity), same as the runner. Move the patch helpers out of pkg/runner into a shared pkg/genesis (ApplyForkOverrides / ApplyEIPOverrides) used by both the runner and the builder; remove the now-unused pkg/builder/genesis.go fork-flag deriver. Update the example amsterdam configs (compute/stateful/full) and the docs to the new fields. Verified end-to-end in the OrbStack VM: the bloatnet full fill still produces 207 passed, 0 failed with the geth filler activating amsterdam via --override.amsterdam=1 in filler_extra_args. --- docs/configuration.md | 9 +- ...te-actor-eest.full.amsterdam.stateful.yaml | 5 +- ...e-actor-eest.simple.amsterdam.compute.yaml | 21 ++-- ...-actor-eest.simple.amsterdam.stateful.yaml | 15 +-- .../config.state-actor-eest.yaml | 4 +- pkg/builder/eest_payloads.go | 116 ++++++++++++++---- pkg/builder/eest_payloads_test.go | 6 +- pkg/builder/genesis.go | 90 -------------- pkg/builder/genesis_test.go | 69 ----------- pkg/config/config.go | 66 +++++----- .../override.go} | 39 +++--- .../override_test.go} | 62 ++++------ pkg/runner/lifecycle.go | 7 +- 13 files changed, 207 insertions(+), 302 deletions(-) delete mode 100644 pkg/builder/genesis.go delete mode 100644 pkg/builder/genesis_test.go rename pkg/{runner/genesis_override.go => genesis/override.go} (74%) rename pkg/{runner/genesis_override_test.go => genesis/override_test.go} (74%) diff --git a/docs/configuration.md b/docs/configuration.md index dc6d4a023..62f336ae7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1629,7 +1629,10 @@ builder: - name: compute-geth filler_client: geth source_dir: /srv/state/geth-archive # PRISTINE snapshot (never mutated; a writable copy is filled) - genesis_file: /srv/state/geth-archive/genesis.json # chain config for the filler boot + # geth boots from the datadir; to fill a fork that activates after the + # snapshot, pass --override. here (besu/nethermind use `genesis` + + # genesis_fork_override / genesis_eip_override instead): + # filler_extra_args: [--override.amsterdam=1] output_dir: /srv/fixtures/compute tests: - tests/benchmark/compute # pytest paths inside the fill image @@ -1673,7 +1676,9 @@ Identity/locator fields are target-only; the rest mirror `config` and are resolv | `name` | string | `filler_client` | Used by `--target` to filter. Must be unique across targets. | | `filler_client` | string | – | Client booted as the filler. Must be `geth` (only client supporting `testing_buildBlockV1`). | | `source_dir` | string | – | **Absolute** host path to the pristine snapshot datadir (e.g. a `state_actor` `output_dir`). Never mutated — a writable copy is filled. Existence is checked at build time. | -| `genesis_file` | string | – | **Absolute** host path to the genesis/chain-config the filler boots with (`--override.genesis`). Must match the chain config used to produce `source_dir`. | +| `genesis` | string | – | **Absolute** host path to the genesis/chainspec the filler boots with (besu/nethermind read their fork schedule from it; passed via the client's genesis flag). Must match the chain config used to produce `source_dir`. geth/erigon boot from the datadir instead and need no `genesis`. | +| `genesis_fork_override` | map | – | Patch the geth-format `genesis` at filler boot to activate forks at given timestamps (`{amsterdam: 1}` → `config.amsterdamTime`, inheriting the blob schedule). For besu/reth/ethrex fillers. Same mechanism as the runner. Requires `genesis`. | +| `genesis_eip_override` | object | – | Patch a parity/nethermind `genesis` at filler boot, setting `params.eipTransitionTimestamp` for each listed EIP. Fields: `timestamp` (uint), `eips` ([]uint). For the nethermind filler. Requires `genesis`; mutually exclusive with `genesis_fork_override`. | | `output_dir` | string | – | **Absolute** host path for the generated fixtures. Skipped if already populated unless `--force` / `force: true`. Written under `/blockchain_tests_stateful_engine/`. | | `tests` | []string | – | **Required.** pytest paths inside the fill image, e.g. `tests/benchmark/compute`. | | `filter` | string | – | Optional pytest `-k` expression. | diff --git a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml index 641682f6f..9ce99e685 100644 --- a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml @@ -152,7 +152,10 @@ builder: filler_client: geth filler_image: skylenet/geth:bal-devnet-7-amsterdam-override source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth - fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam-bloatnet} # Resolves @pytest.mark.stub_parametrize prefixes against pre-deployed # state (e.g. bloated_eoa_10GB → 0x87a6…be04). Copy the committed diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml index e42cf050b..d02412d73 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml @@ -2,9 +2,9 @@ # # state-actor generates OSAKA snapshot datadirs (stage 1); the eest_payloads stage # fills AMSTERDAM tests/benchmark/compute fixtures by booting the geth filler on the -# osaka snapshot and activating amsterdam at the snapshot block's timestamp + 1 (via -# fork_activation_genesis → a geth --override.amsterdam= flag). So block 0 stays -# osaka and the first benchmark block (block 1) is amsterdam. +# osaka snapshot and activating amsterdam at block 0's timestamp + 1 via geth's +# --override.amsterdam=1 flag (filler_extra_args). So block 0 stays osaka and the +# first benchmark block (block 1) is amsterdam. # # Stateful (EIP-7928 / BAL) fixtures are built by the sibling config: # config.state-actor-eest.simple.amsterdam.stateful.yaml @@ -107,16 +107,17 @@ builder: # upstream gaps close. # # amsterdam fill: filler_image must be a geth that registers --override.amsterdam - # (bal-devnet-7 + the amsterdam-override patch). fork_activation_genesis reads - # the snapshot block's timestamp and the builder boots the filler with - # --override.amsterdam=. + # (bal-devnet-7 + the amsterdam-override patch); the geth target passes + # --override.amsterdam=1 via filler_extra_args (snapshot block 0 ts is 0). targets: - name: payload-generator-geth filler_client: geth filler_image: skylenet/geth:bal-devnet-7-amsterdam-override source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth - # Schedule amsterdam at the osaka snapshot block 0's timestamp + 1. - fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam} tests: - tests/benchmark/compute # pytest paths inside the fill image @@ -150,7 +151,7 @@ builder: # filler_client: besu # filler_image: hyperledger/besu:26.6.1 # >= 26.6 ships testing_buildBlockV1 # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu - # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + # genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-besu # tests: # - tests/benchmark/compute @@ -163,7 +164,7 @@ builder: # filler_client: nethermind # filler_image: nethermindeth/nethermind:testing_build_block_with_opcode_tracing # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind - # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + # genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-nethermind # tests: # - tests/benchmark/compute diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml index 777eba3cb..362611945 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml @@ -2,8 +2,8 @@ # # state-actor generates OSAKA snapshot datadirs (stage 1); the eest_payloads stage # fills AMSTERDAM tests/benchmark/stateful EIP-7928 (BAL) fixtures by booting the geth -# filler on the osaka snapshot and activating amsterdam at the snapshot block's -# timestamp + 1 (via fork_activation_genesis → a geth --override.amsterdam= flag). +# filler on the osaka snapshot and activating amsterdam at block 0's timestamp + 1 +# via geth's --override.amsterdam=1 flag (filler_extra_args). # # These BAL tests are self-contained (each deploys its own contracts via # pre.deploy_contract / pre.fund_eoa against the predeployed CREATE2 factory + funded @@ -104,16 +104,17 @@ builder: gas_benchmark_values: [10] # Millions of gas to parametrise against datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) # amsterdam fill: filler_image must be a geth that registers --override.amsterdam - # (bal-devnet-7 + the amsterdam-override patch). fork_activation_genesis reads - # the snapshot block's timestamp and the builder boots the filler with - # --override.amsterdam=. + # (bal-devnet-7 + the amsterdam-override patch); the geth target passes + # --override.amsterdam=1 via filler_extra_args (snapshot block 0 ts is 0). targets: - name: payload-generator-geth-stateful filler_client: geth filler_image: skylenet/geth:bal-devnet-7-amsterdam-override source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth - # Schedule amsterdam at the osaka snapshot block 0's timestamp + 1. - fork_activation_genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth/geth-genesis.json + # geth boots from the snapshot datadir; activate amsterdam at block 0's + # timestamp (0) + 1 via the override flag — same as the runner geth instance. + filler_extra_args: + - --override.amsterdam=1 output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam-stateful} tests: - tests/benchmark/stateful/eip7928_block_level_access_lists diff --git a/examples/configuration/config.state-actor-eest.yaml b/examples/configuration/config.state-actor-eest.yaml index 81d06b91f..407216d31 100644 --- a/examples/configuration/config.state-actor-eest.yaml +++ b/examples/configuration/config.state-actor-eest.yaml @@ -83,7 +83,7 @@ builder: # filler_client: besu # filler_image: hyperledger/besu:26.6.1 # >= 26.6 ships testing_buildBlockV1 # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu - # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + # genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-besu # tests: # - tests/benchmark/compute @@ -96,7 +96,7 @@ builder: # filler_client: nethermind # filler_image: nethermindeth/nethermind:testing_build_block_with_opcode_tracing # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind - # genesis_file: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + # genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-nethermind # tests: # - tests/benchmark/compute diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index 24ea6f08a..3da6690f0 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -16,6 +16,7 @@ import ( "github.com/ethpandaops/benchmarkoor/pkg/config" "github.com/ethpandaops/benchmarkoor/pkg/datadir" "github.com/ethpandaops/benchmarkoor/pkg/docker" + "github.com/ethpandaops/benchmarkoor/pkg/genesis" "github.com/ethpandaops/benchmarkoor/pkg/gitrepo" "github.com/sirupsen/logrus" ) @@ -182,15 +183,9 @@ func (b *EESTPayloadsBuilder) checkInputs(t *config.EESTPayloadTarget) error { return fmt.Errorf("source_dir %q is not a directory", t.SourceDir) } - if t.GenesisFile != "" { - if _, err := os.Stat(t.GenesisFile); err != nil { - return fmt.Errorf("genesis_file: %w", err) - } - } - - if t.ForkActivationGenesis != "" { - if _, err := os.Stat(t.ForkActivationGenesis); err != nil { - return fmt.Errorf("fork_activation_genesis: %w", err) + if t.Genesis != "" { + if _, err := os.Stat(t.Genesis); err != nil { + return fmt.Errorf("genesis: %w", err) } } @@ -268,21 +263,22 @@ func (b *EESTPayloadsBuilder) run(ctx context.Context, log logrus.FieldLogger, t } }() - // When fork_activation_genesis is set, boot the filler with a - // --override.= flag so the target fork - // activates on the first block it builds (block N+1). See - // forkOverrideActivationFlag for why --override.genesis can't be used. - if t.ForkActivationGenesis != "" { - flag, err := forkOverrideActivationFlag(t.ForkActivationGenesis, t.Fork) - if err != nil { - return fmt.Errorf("scheduling %s activation: %w", t.Fork, err) + // genesis_fork_override / genesis_eip_override patch the boot genesis before + // the filler mounts it, to activate a fork the file doesn't schedule (e.g. + // amsterdam on an osaka snapshot) — identical to the runner. Used by fillers + // that read forks from the genesis (besu/reth/ethrex/nethermind). geth/erigon + // boot from the datadir and instead activate forks via --override. in + // filler_extra_args. + if len(t.GenesisForkOverride) > 0 || + (t.GenesisEIPOverride != nil && len(t.GenesisEIPOverride.EIPs) > 0) { + patched, cleanup, perr := patchFillerGenesis(log, t) + if perr != nil { + return perr } - t.FillerExtraArgs = append(t.FillerExtraArgs, flag) + defer cleanup() - log.WithFields(logrus.Fields{ - "base_genesis": t.ForkActivationGenesis, "fork": t.Fork, "flag": flag, - }).Info("Scheduling fork activation on filler") + t.Genesis = patched } // Stream the filler's logs for the lifetime of this build. @@ -370,9 +366,9 @@ func (b *EESTPayloadsBuilder) startFiller( configCleanup = cfgCleanup } - if t.GenesisFile != "" { + if t.Genesis != "" { mounts = append(mounts, docker.Mount{ - Source: t.GenesisFile, Target: spec.GenesisPath(), Type: "bind", ReadOnly: true, + Source: t.Genesis, Target: spec.GenesisPath(), Type: "bind", ReadOnly: true, }) } @@ -629,7 +625,7 @@ func fillerGethCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { "--miner.gaslimit=" + minerGasLimit, } - if t.GenesisFile != "" { + if t.Genesis != "" { args = append(args, spec.GenesisFlag()+spec.GenesisPath()) } @@ -673,7 +669,7 @@ func fillerBesuCommand(t *config.EESTPayloadTarget, spec client.Spec) []string { "--target-gas-limit=" + minerGasLimit, } - if t.GenesisFile != "" { + if t.Genesis != "" { args = append(args, spec.GenesisFlag()+spec.GenesisPath()) } @@ -715,7 +711,7 @@ func fillerNethermindCommand(t *config.EESTPayloadTarget, spec client.Spec) []st "--Blocks.TargetBlockGasLimit=" + minerGasLimit, } - if t.GenesisFile != "" { + if t.Genesis != "" { args = append(args, spec.GenesisFlag()+spec.GenesisPath()) } @@ -805,6 +801,74 @@ func buildFillArgs( return args } +// patchFillerGenesis reads the target's boot genesis, applies its +// genesis_fork_override / genesis_eip_override (mutually exclusive; validated), +// and writes the result to a temp file the filler container can bind-mount. +// Returns the temp path and a cleanup callback. It is the builder-side analogue +// of the runner's per-instance genesis patching. +func patchFillerGenesis( + log logrus.FieldLogger, t *config.EESTPayloadTarget, +) (string, func(), error) { + raw, err := os.ReadFile(t.Genesis) + if err != nil { + return "", nil, fmt.Errorf("reading genesis %q: %w", t.Genesis, err) + } + + var patched []byte + + switch { + case len(t.GenesisForkOverride) > 0: + patched, err = genesis.ApplyForkOverrides(raw, t.GenesisForkOverride) + if err != nil { + return "", nil, fmt.Errorf("applying genesis_fork_override: %w", err) + } + + log.WithField("forks", t.GenesisForkOverride). + Info("Applied genesis fork-time overrides to filler genesis") + case t.GenesisEIPOverride != nil: + patched, err = genesis.ApplyEIPOverrides( + raw, t.GenesisEIPOverride.Timestamp, t.GenesisEIPOverride.EIPs, + ) + if err != nil { + return "", nil, fmt.Errorf("applying genesis_eip_override: %w", err) + } + + log.WithFields(logrus.Fields{ + "eips": t.GenesisEIPOverride.EIPs, "timestamp": t.GenesisEIPOverride.Timestamp, + }).Info("Applied genesis EIP-time overrides to filler genesis") + } + + f, err := os.CreateTemp(mountTempDir(), "benchmarkoor-eest-genesis-*") + if err != nil { + return "", nil, fmt.Errorf("creating temp genesis file: %w", err) + } + + path := f.Name() + + cleanup := func() { _ = os.Remove(path) } + + if _, err := f.Write(patched); err != nil { + _ = f.Close() + cleanup() + + return "", nil, fmt.Errorf("writing temp genesis file: %w", err) + } + + if err := f.Close(); err != nil { + cleanup() + + return "", nil, fmt.Errorf("closing temp genesis file: %w", err) + } + + if err := os.Chmod(path, 0o644); err != nil { + cleanup() + + return "", nil, fmt.Errorf("chmod temp genesis file: %w", err) + } + + return path, cleanup, nil +} + // writeTempJWT writes the JWT secret to a temp file readable by the // container UID (0644) and returns its path plus a cleanup callback. func writeTempJWT(secret string) (string, func(), error) { diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go index 1ba9873f1..46195f5e7 100644 --- a/pkg/builder/eest_payloads_test.go +++ b/pkg/builder/eest_payloads_test.go @@ -138,7 +138,7 @@ func TestFillerGethCommand(t *testing.T) { t.Run("with genesis and extra args", func(t *testing.T) { cmd := fillerGethCommand(&config.EESTPayloadTarget{ FillerClient: "geth", - GenesisFile: "/host/genesis.json", + Genesis: "/host/genesis.json", FillerExtraArgs: []string{"--verbosity=5"}, }, spec) @@ -153,7 +153,7 @@ func TestFillerCommand_Besu(t *testing.T) { cmd := fillerCommand(&config.EESTPayloadTarget{ FillerClient: "besu", - GenesisFile: "/host/besu-chainspec.json", + Genesis: "/host/besu-chainspec.json", FillerExtraArgs: []string{"--logging=DEBUG"}, }, spec) @@ -175,7 +175,7 @@ func TestFillerCommand_Nethermind(t *testing.T) { cmd := fillerCommand(&config.EESTPayloadTarget{ FillerClient: "nethermind", - GenesisFile: "/host/parity-chainspec.json", + Genesis: "/host/parity-chainspec.json", }, spec) assert.Contains(t, cmd, "--datadir=/data") diff --git a/pkg/builder/genesis.go b/pkg/builder/genesis.go deleted file mode 100644 index 94d3050f1..000000000 --- a/pkg/builder/genesis.go +++ /dev/null @@ -1,90 +0,0 @@ -package builder - -import ( - "encoding/json" - "fmt" - "os" - "strconv" - "strings" -) - -// forkOverrideActivationFlag returns the geth --override.= flag -// that schedules fork at the snapshot genesis block's timestamp + 1, read from -// baseGenesisPath (the state-actor snapshot's geth-genesis.json). The filler -// boots at the snapshot head (block N, timestamp T) and builds block N+1 at -// timestamp T+1, so fork activates exactly on the first block it builds. -// -// state-actor bakes the snapshot state into the DB under an empty-alloc genesis, -// so --override.genesis can't be used (geth recomputes an empty-state genesis -// and rejects the hash mismatch). The per-fork --override. flag instead -// amends only the in-memory chain config at boot, leaving the genesis block -// untouched. Requires a geth build that registers the flag (e.g. -// ethpandaops/geth:bal-devnet-7-amsterdam-override for amsterdam). -func forkOverrideActivationFlag(baseGenesisPath, fork string) (string, error) { - ts, err := genesisFileTimestamp(baseGenesisPath) - if err != nil { - return "", err - } - - return fmt.Sprintf("--override.%s=%d", strings.ToLower(fork), ts+1), nil -} - -// genesisFileTimestamp reads the genesis block timestamp from the top-level -// "timestamp" field of a geth genesis JSON file. -func genesisFileTimestamp(path string) (uint64, error) { - raw, err := os.ReadFile(path) - if err != nil { - return 0, fmt.Errorf("reading base genesis %q: %w", path, err) - } - - var genesis map[string]any - if err := json.Unmarshal(raw, &genesis); err != nil { - return 0, fmt.Errorf("parsing base genesis %q: %w", path, err) - } - - ts, err := genesisBlockTimestamp(genesis) - if err != nil { - return 0, fmt.Errorf("base genesis %q: %w", path, err) - } - - return ts, nil -} - -// genesisBlockTimestamp reads the genesis block's timestamp from the top-level -// "timestamp" field, accepting a 0x-prefixed hex string (geth's encoding) or a -// JSON number. -func genesisBlockTimestamp(genesis map[string]any) (uint64, error) { - v, ok := genesis["timestamp"] - if !ok { - return 0, fmt.Errorf("missing \"timestamp\" field") - } - - switch t := v.(type) { - case string: - ts, err := parseGenesisUint(t) - if err != nil { - return 0, fmt.Errorf("parsing \"timestamp\" %q: %w", t, err) - } - - return ts, nil - case float64: - return uint64(t), nil - default: - return 0, fmt.Errorf("unexpected \"timestamp\" type %T", v) - } -} - -// parseGenesisUint parses a uint from a genesis-encoded string: 0x-prefixed hex -// or plain decimal. -func parseGenesisUint(s string) (uint64, error) { - s = strings.TrimSpace(s) - if rest, ok := strings.CutPrefix(s, "0x"); ok { - return strconv.ParseUint(rest, 16, 64) - } - - if rest, ok := strings.CutPrefix(s, "0X"); ok { - return strconv.ParseUint(rest, 16, 64) - } - - return strconv.ParseUint(s, 10, 64) -} diff --git a/pkg/builder/genesis_test.go b/pkg/builder/genesis_test.go deleted file mode 100644 index 50fb70e5a..000000000 --- a/pkg/builder/genesis_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package builder - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func writeGenesisFixture(t *testing.T, timestamp string) string { - t.Helper() - - path := filepath.Join(t.TempDir(), "geth-genesis.json") - body := `{"timestamp":"` + timestamp + `","config":{"chainId":1337,"osakaTime":0}}` - require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) - - return path -} - -func TestForkOverrideActivationFlag(t *testing.T) { - tests := []struct { - name string - timestamp string - fork string - want string - }{ - {name: "zero timestamp", timestamp: "0x0", fork: "amsterdam", want: "--override.amsterdam=1"}, - {name: "hex timestamp", timestamp: "0x10", fork: "amsterdam", want: "--override.amsterdam=17"}, - {name: "fork lowercased", timestamp: "0x0", fork: "Amsterdam", want: "--override.amsterdam=1"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - flag, err := forkOverrideActivationFlag(writeGenesisFixture(t, tc.timestamp), tc.fork) - require.NoError(t, err) - assert.Equal(t, tc.want, flag) - }) - } -} - -func TestForkOverrideActivationFlag_Errors(t *testing.T) { - t.Run("missing file", func(t *testing.T) { - _, err := forkOverrideActivationFlag(filepath.Join(t.TempDir(), "nope.json"), "amsterdam") - require.Error(t, err) - }) - - t.Run("missing timestamp", func(t *testing.T) { - p := filepath.Join(t.TempDir(), "g.json") - require.NoError(t, os.WriteFile(p, []byte(`{"config":{}}`), 0o644)) - _, err := forkOverrideActivationFlag(p, "amsterdam") - require.ErrorContains(t, err, "timestamp") - }) -} - -func TestGenesisFileTimestamp(t *testing.T) { - assert := assert.New(t) - - hexTS, err := genesisFileTimestamp(writeGenesisFixture(t, "0xff")) - require.NoError(t, err) - assert.EqualValues(255, hexTS) - - decPath := filepath.Join(t.TempDir(), "g.json") - require.NoError(t, os.WriteFile(decPath, []byte(`{"timestamp":"42","config":{}}`), 0o644)) - decTS, err := genesisFileTimestamp(decPath) - require.NoError(t, err) - assert.EqualValues(42, decTS) -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 85d0233fa..d3a753b94 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -397,25 +397,29 @@ type EESTPayloadDefaults struct { } // EESTPayloadTarget is one fixture-generation run. Identity/locator fields -// (Name, FillerClient, SourceDir, OutputDir, GenesisFile, -// ForkActivationGenesis, Tests, Filter, AddressStubsFile) live exclusively on -// the target; the remaining fields mirror EESTPayloadDefaults and are resolved -// via ResolveTarget. +// (Name, FillerClient, SourceDir, OutputDir, Genesis, GenesisForkOverride, +// GenesisEIPOverride, Tests, Filter, AddressStubsFile) live exclusively on the +// target; the remaining fields mirror EESTPayloadDefaults and are resolved via +// ResolveTarget. type EESTPayloadTarget struct { Name string `yaml:"name,omitempty" mapstructure:"name"` FillerClient string `yaml:"filler_client" mapstructure:"filler_client"` SourceDir string `yaml:"source_dir" mapstructure:"source_dir"` OutputDir string `yaml:"output_dir" mapstructure:"output_dir"` - GenesisFile string `yaml:"genesis_file,omitempty" mapstructure:"genesis_file"` - // ForkActivationGenesis, when set, is a base genesis JSON (typically the - // state-actor snapshot's geth-genesis.json, generated at the prior fork) - // from which the builder derives the filler's genesis: it schedules Fork's - // activation at the base genesis block's timestamp + 1 (e.g. amsterdamTime) - // and boots the filler with --override.genesis. This fills a fork that - // activates one block after the snapshot. Mutually exclusive with - // GenesisFile; requires Fork. - ForkActivationGenesis string `yaml:"fork_activation_genesis,omitempty" mapstructure:"fork_activation_genesis"` - AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` + // Genesis is the genesis/chainspec the filler boots from (besu/nethermind + // read their fork schedule from it). geth/erigon boot from the snapshot + // datadir instead and activate forks via --override. in + // FillerExtraArgs, so they need no Genesis. Mirrors runner client `genesis`. + Genesis string `yaml:"genesis,omitempty" mapstructure:"genesis"` + // GenesisForkOverride / GenesisEIPOverride patch the Genesis at filler boot + // to activate a fork the file doesn't schedule (e.g. amsterdam on an osaka + // snapshot), identically to the runner. GenesisForkOverride sets + // config.Time in a geth-format genesis (besu/reth/ethrex); + // GenesisEIPOverride sets params.eipTransitionTimestamp in a parity + // chainspec (nethermind). + GenesisForkOverride map[string]uint64 `yaml:"genesis_fork_override,omitempty" mapstructure:"genesis_fork_override"` + GenesisEIPOverride *GenesisEIPOverride `yaml:"genesis_eip_override,omitempty" mapstructure:"genesis_eip_override"` + AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` // Tests are pytest paths inside the fill image, e.g. tests/benchmark/compute. Tests []string `yaml:"tests,omitempty" mapstructure:"tests"` Filter string `yaml:"filter,omitempty" mapstructure:"filter"` @@ -2137,7 +2141,7 @@ func (c *Config) validateEESTPayloads() error { return nil } -// validateEESTPayloadPaths checks output_dir / genesis_file / +// validateEESTPayloadPaths checks output_dir / genesis / // address_stubs_file are absolute and output_dir is unique. func validateEESTPayloadPaths(t *EESTPayloadTarget, prefix string, seenOutputs map[string]int, i int) error { if t.SourceDir == "" { @@ -2164,27 +2168,25 @@ func validateEESTPayloadPaths(t *EESTPayloadTarget, prefix string, seenOutputs m seenOutputs[t.OutputDir] = i - if t.GenesisFile != "" && !filepath.IsAbs(t.GenesisFile) { - return fmt.Errorf("%s.genesis_file must be an absolute path, got %q", prefix, t.GenesisFile) + if t.Genesis != "" && !filepath.IsAbs(t.Genesis) { + return fmt.Errorf("%s.genesis must be an absolute path, got %q", prefix, t.Genesis) } - if t.ForkActivationGenesis != "" { - if t.GenesisFile != "" { - return fmt.Errorf( - "%s: genesis_file and fork_activation_genesis are mutually exclusive", prefix, - ) - } + // genesis_fork_override / genesis_eip_override patch the boot genesis, so + // they require one. (geth/erigon fillers boot from the datadir and use + // --override. in filler_extra_args instead.) + if len(t.GenesisForkOverride) > 0 && t.Genesis == "" { + return fmt.Errorf("%s.genesis_fork_override requires genesis", prefix) + } - if !filepath.IsAbs(t.ForkActivationGenesis) { - return fmt.Errorf( - "%s.fork_activation_genesis must be an absolute path, got %q", - prefix, t.ForkActivationGenesis, - ) - } + if t.GenesisEIPOverride != nil && len(t.GenesisEIPOverride.EIPs) > 0 && t.Genesis == "" { + return fmt.Errorf("%s.genesis_eip_override requires genesis", prefix) + } - if t.Fork == "" { - return fmt.Errorf("%s.fork is required when fork_activation_genesis is set", prefix) - } + if len(t.GenesisForkOverride) > 0 && t.GenesisEIPOverride != nil { + return fmt.Errorf( + "%s: genesis_fork_override and genesis_eip_override are mutually exclusive", prefix, + ) } if t.AddressStubsFile != "" && !filepath.IsAbs(t.AddressStubsFile) { diff --git a/pkg/runner/genesis_override.go b/pkg/genesis/override.go similarity index 74% rename from pkg/runner/genesis_override.go rename to pkg/genesis/override.go index 562e8f5b0..c9e48ccd1 100644 --- a/pkg/runner/genesis_override.go +++ b/pkg/genesis/override.go @@ -1,11 +1,14 @@ -package runner +// Package genesis patches client genesis/chainspec JSON to activate forks that +// the file doesn't already schedule — the file-based equivalent of geth's +// --override. flag, for clients that read their fork schedule from the +// genesis. It is shared by the runner (boot genesis) and the eest_payloads +// builder (filler boot genesis) so both use the same vocabulary. +package genesis import ( "bytes" "encoding/json" "fmt" - - "github.com/ethpandaops/benchmarkoor/pkg/config" ) // blobForkOrder lists the blob-bearing forks in activation order. When a fork @@ -14,18 +17,18 @@ import ( // clients reject an active fork that carries no blob schedule. var blobForkOrder = []string{"cancun", "prague", "osaka", "amsterdam"} -// applyGenesisForkOverrides patches a geth-format genesis JSON so the given -// forks activate at the given timestamps. It is the genesis-file equivalent of -// geth's --override. flag, for clients that instead read their fork -// schedule from the genesis (besu, reth, ethrex). For each fork it sets -// config.Time and, when a blobSchedule is present but lacks the fork, -// inherits the latest preceding fork's blob parameters. +// ApplyForkOverrides patches a geth-format genesis JSON so the given forks +// activate at the given timestamps (fork name → unix-seconds). It is the +// genesis-file equivalent of geth's --override. flag, for clients that +// read their fork schedule from the genesis (besu, reth, ethrex). For each fork +// it sets config.Time and, when a blobSchedule is present but lacks the +// fork, inherits the latest preceding fork's blob parameters. // // Only the top-level "config" object is rewritten; every other field round-trips // verbatim and existing numbers are preserved exactly (so the genesis block hash // is unchanged). It returns an error if the genesis is not geth-format (has no // top-level "config" object), since the patch shape is format-specific. -func applyGenesisForkOverrides(genesis []byte, overrides map[string]uint64) ([]byte, error) { +func ApplyForkOverrides(genesis []byte, overrides map[string]uint64) ([]byte, error) { if len(overrides) == 0 { return genesis, nil } @@ -74,17 +77,17 @@ func applyGenesisForkOverrides(genesis []byte, overrides map[string]uint64) ([]b return patched, nil } -// applyGenesisEIPOverrides patches a parity/nethermind-format chainspec so the -// given EIPs activate at the override timestamp. It is the parity-format -// counterpart of applyGenesisForkOverrides: parity chainspecs schedule forks -// per-EIP (params.eipTransitionTimestamp) rather than by fork name, so the +// ApplyEIPOverrides patches a parity/nethermind-format chainspec so the given +// EIPs activate at timestamp. It is the parity-format counterpart of +// ApplyForkOverrides: parity chainspecs schedule forks per-EIP +// (params.eipTransitionTimestamp) rather than by fork name, so the // devnet-specific EIP list comes from config. // // Only the "params" object is rewritten; every other field round-trips verbatim. // It returns an error if the genesis is not parity-format (has no top-level // "params" object). -func applyGenesisEIPOverrides(genesis []byte, override *config.GenesisEIPOverride) ([]byte, error) { - if override == nil || len(override.EIPs) == 0 { +func ApplyEIPOverrides(genesis []byte, timestamp uint64, eips []uint64) ([]byte, error) { + if len(eips) == 0 { return genesis, nil } @@ -110,8 +113,8 @@ func applyGenesisEIPOverrides(genesis []byte, override *config.GenesisEIPOverrid } // Parity transition timestamps are hex-encoded strings (e.g. "0x1"). - ts := fmt.Sprintf("0x%x", override.Timestamp) - for _, eip := range override.EIPs { + ts := fmt.Sprintf("0x%x", timestamp) + for _, eip := range eips { params[fmt.Sprintf("eip%dTransitionTimestamp", eip)] = ts } diff --git a/pkg/runner/genesis_override_test.go b/pkg/genesis/override_test.go similarity index 74% rename from pkg/runner/genesis_override_test.go rename to pkg/genesis/override_test.go index a4971466f..c676744f3 100644 --- a/pkg/runner/genesis_override_test.go +++ b/pkg/genesis/override_test.go @@ -1,4 +1,4 @@ -package runner +package genesis import ( "encoding/json" @@ -6,25 +6,22 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/ethpandaops/benchmarkoor/pkg/config" ) -func TestApplyGenesisForkOverrides(t *testing.T) { +func TestApplyForkOverrides(t *testing.T) { t.Run("no overrides returns input unchanged", func(t *testing.T) { in := []byte(`{"config":{"osakaTime":0}}`) - out, err := applyGenesisForkOverrides(in, nil) + out, err := ApplyForkOverrides(in, nil) require.NoError(t, err) assert.Equal(t, in, out) }) t.Run("non-geth genesis errors", func(t *testing.T) { - // Parity-format chainspec has params, not config. in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"}}`) - _, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + _, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) require.Error(t, err) assert.Contains(t, err.Error(), "geth-format") @@ -42,7 +39,7 @@ func TestApplyGenesisForkOverrides(t *testing.T) { "gasLimit": "0x11e1a300" }`) - out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) require.NoError(t, err) cfg := decodeConfig(t, out) @@ -56,7 +53,6 @@ func TestApplyGenesisForkOverrides(t *testing.T) { assert.EqualValues(t, 12, amsterdam["max"]) assert.EqualValues(t, 9, amsterdam["target"]) - // Untouched top-level fields survive. var top map[string]any require.NoError(t, json.Unmarshal(out, &top)) assert.Equal(t, "0x11e1a300", top["gasLimit"]) @@ -69,7 +65,7 @@ func TestApplyGenesisForkOverrides(t *testing.T) { "osaka":{"max":12} }}}`) - out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) require.NoError(t, err) bs := decodeConfig(t, out)["blobSchedule"].(map[string]any) @@ -83,7 +79,7 @@ func TestApplyGenesisForkOverrides(t *testing.T) { "amsterdam":{"max":99} }}}`) - out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) require.NoError(t, err) bs := decodeConfig(t, out)["blobSchedule"].(map[string]any) @@ -94,7 +90,7 @@ func TestApplyGenesisForkOverrides(t *testing.T) { t.Run("no blob schedule only sets the time", func(t *testing.T) { in := []byte(`{"config":{"osakaTime":0}}`) - out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) require.NoError(t, err) cfg := decodeConfig(t, out) @@ -104,10 +100,9 @@ func TestApplyGenesisForkOverrides(t *testing.T) { }) t.Run("preserves large integers without float corruption", func(t *testing.T) { - // A value beyond float64's exact-integer range must round-trip exactly. in := []byte(`{"config":{"terminalTotalDifficulty":115792089237316195423570985008687907853269984665640564039457584007913129639936}}`) - out, err := applyGenesisForkOverrides(in, map[string]uint64{"amsterdam": 1}) + out, err := ApplyForkOverrides(in, map[string]uint64{"amsterdam": 1}) require.NoError(t, err) assert.Contains(t, string(out), @@ -115,15 +110,11 @@ func TestApplyGenesisForkOverrides(t *testing.T) { }) } -func TestApplyGenesisEIPOverrides(t *testing.T) { - t.Run("nil or empty override returns input unchanged", func(t *testing.T) { +func TestApplyEIPOverrides(t *testing.T) { + t.Run("no eips returns input unchanged", func(t *testing.T) { in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"}}`) - out, err := applyGenesisEIPOverrides(in, nil) - require.NoError(t, err) - assert.Equal(t, in, out) - - out, err = applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{Timestamp: 1}) + out, err := ApplyEIPOverrides(in, 1, nil) require.NoError(t, err) assert.Equal(t, in, out) }) @@ -131,9 +122,7 @@ func TestApplyGenesisEIPOverrides(t *testing.T) { t.Run("non-parity genesis errors", func(t *testing.T) { in := []byte(`{"config":{"osakaTime":0}}`) - _, err := applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{ - Timestamp: 1, EIPs: []uint64{7928}, - }) + _, err := ApplyEIPOverrides(in, 1, []uint64{7928}) require.Error(t, err) assert.Contains(t, err.Error(), "parity") @@ -142,15 +131,12 @@ func TestApplyGenesisEIPOverrides(t *testing.T) { t.Run("sets eip transition timestamps as hex", func(t *testing.T) { in := []byte(`{"params":{"eip7825TransitionTimestamp":"0x0"},"name":"x"}`) - out, err := applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{ - Timestamp: 1, EIPs: []uint64{7928, 8037}, - }) + out, err := ApplyEIPOverrides(in, 1, []uint64{7928, 8037}) require.NoError(t, err) params := decodeParams(t, out) assert.Equal(t, "0x1", params["eip7928TransitionTimestamp"]) assert.Equal(t, "0x1", params["eip8037TransitionTimestamp"]) - // Pre-existing params survive. assert.Equal(t, "0x0", params["eip7825TransitionTimestamp"]) var top map[string]any @@ -161,9 +147,7 @@ func TestApplyGenesisEIPOverrides(t *testing.T) { t.Run("encodes larger timestamps as hex", func(t *testing.T) { in := []byte(`{"params":{}}`) - out, err := applyGenesisEIPOverrides(in, &config.GenesisEIPOverride{ - Timestamp: 1769856767, EIPs: []uint64{7928}, - }) + out, err := ApplyEIPOverrides(in, 1769856767, []uint64{7928}) require.NoError(t, err) params := decodeParams(t, out) @@ -171,26 +155,26 @@ func TestApplyGenesisEIPOverrides(t *testing.T) { }) } -func decodeParams(t *testing.T, genesis []byte) map[string]any { +func decodeConfig(t *testing.T, genesis []byte) map[string]any { t.Helper() var top map[string]json.RawMessage require.NoError(t, json.Unmarshal(genesis, &top)) - var params map[string]any - require.NoError(t, json.Unmarshal(top["params"], ¶ms)) + var cfg map[string]any + require.NoError(t, json.Unmarshal(top["config"], &cfg)) - return params + return cfg } -func decodeConfig(t *testing.T, genesis []byte) map[string]any { +func decodeParams(t *testing.T, genesis []byte) map[string]any { t.Helper() var top map[string]json.RawMessage require.NoError(t, json.Unmarshal(genesis, &top)) - var cfg map[string]any - require.NoError(t, json.Unmarshal(top["config"], &cfg)) + var params map[string]any + require.NoError(t, json.Unmarshal(top["params"], ¶ms)) - return cfg + return params } diff --git a/pkg/runner/lifecycle.go b/pkg/runner/lifecycle.go index 8f8d8c489..267359a1b 100644 --- a/pkg/runner/lifecycle.go +++ b/pkg/runner/lifecycle.go @@ -22,6 +22,7 @@ import ( "github.com/ethpandaops/benchmarkoor/pkg/docker" "github.com/ethpandaops/benchmarkoor/pkg/executor" "github.com/ethpandaops/benchmarkoor/pkg/fsutil" + "github.com/ethpandaops/benchmarkoor/pkg/genesis" "github.com/ethpandaops/benchmarkoor/pkg/podman" "github.com/ethpandaops/benchmarkoor/pkg/version" "github.com/shirou/gopsutil/v4/cpu" @@ -198,7 +199,7 @@ func (r *runner) runContainerLifecycle( ) } - patched, overrideErr := applyGenesisForkOverrides( + patched, overrideErr := genesis.ApplyForkOverrides( genesisContent, instance.GenesisForkOverride, ) if overrideErr != nil { @@ -223,8 +224,8 @@ func (r *runner) runContainerLifecycle( ) } - patched, overrideErr := applyGenesisEIPOverrides( - genesisContent, instance.GenesisEIPOverride, + patched, overrideErr := genesis.ApplyEIPOverrides( + genesisContent, instance.GenesisEIPOverride.Timestamp, instance.GenesisEIPOverride.EIPs, ) if overrideErr != nil { return fmt.Errorf( From 437135a5bb99774fb8e95809144795aaccff46ce Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Wed, 24 Jun 2026 20:31:47 +0200 Subject: [PATCH 37/83] fix(executor): preserve full test name when result dir is shortened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 255-byte path fix (sanitizeResultPath) truncated+hashed the per-test directory name, but result aggregation derived the test's identity FROM that directory — so stats.json keys and the UI's EEST node-id parsing got the mangled name for any shortened (e.g. bloatnet) test. - WriteStepResults now writes a .test-name marker holding the full, un-sanitized name into each per-test result directory. - GenerateRunResult reads it (resolveTestName) so result.json — and thus stats.json and the UI — key by the real test name again. Falls back to the directory path for older results without a marker. Also harden CreateSuiteOutput: only treat a suite as already built when a summary.json with tests exists. A partial directory left by an aborted run (e.g. the original "file name too long" crash) previously made a later run skip rebuilding info.Tests and rewrite summary.json with "tests": null. Verified in the VM: re-running the 207-test bloatnet suite yields summary.json with 207 tests and stats.json with full, untruncated keys. --- pkg/executor/results.go | 46 +++++++++++++++++++++++++++++- pkg/executor/sanitize_path_test.go | 21 ++++++++++++++ pkg/executor/suite.go | 12 ++++++-- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/pkg/executor/results.go b/pkg/executor/results.go index 5397024f7..8ad2c0848 100644 --- a/pkg/executor/results.go +++ b/pkg/executor/results.go @@ -15,6 +15,33 @@ import ( "github.com/ethpandaops/benchmarkoor/pkg/fsutil" ) +// testNameMarker is the file written into each per-test result directory holding +// the full, un-sanitized test name. Result aggregation reads it to recover the +// real name when the directory was shortened by sanitizeResultPath. +const testNameMarker = ".test-name" + +// resolveTestName returns the full test name for a result directory relative to +// resultsDir, reading the testNameMarker written by WriteStepResults. It falls +// back to the directory path when the marker is absent (older results), and +// caches per directory. +func resolveTestName(resultsDir, dir string, cache map[string]string) string { + if v, ok := cache[dir]; ok { + return v + } + + name := dir + + if data, err := os.ReadFile(filepath.Join(resultsDir, dir, testNameMarker)); err == nil { + if s := strings.TrimSpace(string(data)); s != "" { + name = s + } + } + + cache[dir] = name + + return name +} + // maxResultPathComponent caps each path component of a test result directory // below the common 255-byte filename limit, leaving headroom for suffixes like // ".request". EEST benchmark node ids — especially the verbose stateful/bloatnet @@ -611,6 +638,17 @@ func WriteStepResults( return fmt.Errorf("creating test result directory: %w", err) } + // Persist the full, un-sanitized test name alongside the results so result + // aggregation can recover it even when the directory name was shortened to + // fit the filesystem's filename limit (see sanitizeResultPath). Without this, + // downstream consumers (stats.json, the UI's EEST node-id parsing) would see + // the truncated+hashed directory name instead of the real test name. + if err := fsutil.WriteFile( + filepath.Join(testDir, testNameMarker), []byte(testName), 0644, owner, + ); err != nil { + return fmt.Errorf("writing test name marker: %w", err) + } + // Base path is the step type (e.g., "setup", "test", "cleanup"). basePath := filepath.Join(testDir, string(stepType)) @@ -663,6 +701,9 @@ func GenerateRunResult(resultsDir string) (*RunResult, error) { Tests: make(map[string]*TestEntry), } + // Caches the directory → full test name lookup (see testNameMarker). + nameCache := make(map[string]string) + // Walk the results directory looking for .result-aggregated.json files. err := filepath.Walk(resultsDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -719,10 +760,13 @@ func GenerateRunResult(resultsDir string) (*RunResult, error) { return nil } - // The test name is the directory containing the step files. + // The test name is the directory containing the step files; recover the + // full name from the marker when the directory was shortened. testName := dir if testName == "." { testName = "" + } else { + testName = resolveTestName(resultsDir, dir, nameCache) } // Set the step result. diff --git a/pkg/executor/sanitize_path_test.go b/pkg/executor/sanitize_path_test.go index b297e5c46..61c896b3a 100644 --- a/pkg/executor/sanitize_path_test.go +++ b/pkg/executor/sanitize_path_test.go @@ -1,10 +1,12 @@ package executor import ( + "os" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSanitizeResultPath(t *testing.T) { @@ -29,3 +31,22 @@ func TestSanitizeResultPath(t *testing.T) { // Deterministic. assert.Equal(t, out, sanitizeResultPath(name)) } + +func TestResolveTestName(t *testing.T) { + dir := t.TempDir() + full := "benchmark/stateful/bloatnet/test_x.py::test_y[" + strings.Repeat("p", 400) + "]" + sanitized := sanitizeResultPath(full) + + // WriteStepResults writes the marker into the (sanitized) test dir. + testDir := dir + "/" + sanitized + require.NoError(t, os.MkdirAll(testDir, 0o755)) + require.NoError(t, os.WriteFile(testDir+"/"+testNameMarker, []byte(full), 0o644)) + + cache := map[string]string{} + // The marker restores the full name for a shortened directory. + assert.Equal(t, full, resolveTestName(dir, sanitized, cache)) + // Cached on second call. + assert.Equal(t, full, resolveTestName(dir, sanitized, cache)) + // No marker → falls back to the directory path. + assert.Equal(t, "some/dir", resolveTestName(dir, "some/dir", cache)) +} diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 3d16c6e94..558ac88ae 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -166,9 +166,15 @@ func CreateSuiteOutput( suiteExists := false - // Check if suite already exists. - if _, err := os.Stat(suiteDir); err == nil { - suiteExists = true + // Treat the suite as already built only when a complete summary.json with + // tests is present. A bare or partial directory — e.g. left behind by a run + // that aborted mid-creation — must be rebuilt; otherwise info.Tests stays + // nil and summary.json gets (re)written with "tests": null. + if data, err := os.ReadFile(filepath.Join(suiteDir, "summary.json")); err == nil { + var existing SuiteInfo + if json.Unmarshal(data, &existing) == nil && len(existing.Tests) > 0 { + suiteExists = true + } } if !suiteExists { From a35a10941440f0d3e591b60c09066f78f2cb837b Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 07:06:41 +0200 Subject: [PATCH 38/83] =?UTF-8?q?feat(examples):=20bloatnet=20full=20state?= =?UTF-8?q?ful=20=E2=80=94=20fix=20ether=5Ftransfers=20funding,=20enable?= =?UTF-8?q?=20nethermind?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sender-pool-300m start_pkey -> keccak256("gas-repricings-private-key") (0xe1e1d345…); the gist's 0x1111… left every derived sender unfunded, so all 18 test_ether_transfers_onchain_receivers failed "insufficient funds". With the correct base key, ether_transfers fills 18/18 (suite now 225/0). - gas_limit 300M -> 1000M for general setup-block headroom. Verified this does NOT make test_deep_branch fillable (its worst-case deploy needs a single block > 1G; EIP-1559 caps the per-block ramp), so deep stays excluded. - enable nethermind state-actor target + runner instance (genesis_eip_override with the devnet-7 amsterdam EIP set); geth/nethermind genesis state roots match. - refresh comments/STATUS to the validated result. Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- ...te-actor-eest.full.amsterdam.stateful.yaml | 69 ++++++++++--------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml index 9ce99e685..0c04c35ca 100644 --- a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml @@ -28,10 +28,10 @@ # snapshot (state-actor builds one per target), so 6 clients ≈ 460 GB. Uncomment # the others once you have the disk. # -# STATUS: the overlayfs pipeline is validated end-to-end on the small eip7928 -# stateful set; this full bloatnet variant is a starting point — the exact -# test/stub coverage (which bloatnet tests the gist prestate satisfies) still -# needs iteration in the VM. See the filter note on the geth target. +# STATUS: validated end-to-end in the OrbStack VM — geth fills 225 bloatnet +# cases (0 failed) against this prestate, and the geth/nethermind genesis state +# roots match. The excluded tests (see the filter note on the geth target) need +# prestate/stub coverage this gist doesn't provide. # global: log_level: info @@ -41,9 +41,9 @@ builder: state_actor: images: geth: ghcr.io/ethereum/state-actor:main + nethermind: ghcr.io/ethereum/state-actor-nethermind:main # reth: ghcr.io/ethereum/state-actor-reth:main # besu: ghcr.io/ethereum/state-actor-besu:main - # nethermind: ghcr.io/ethereum/state-actor-nethermind:main # ethrex: ghcr.io/ethereum/state-actor-ethrex:main # erigon: ghcr.io/ethereum/state-actor-erigon:main pull_policy: always @@ -52,8 +52,13 @@ builder: fork: osaka # snapshot baseline (amsterdam is forked in at fill/run) chain: 1337 # gist minimum is 85M (an amsterdam max-size code deploy needs gas_limit - # >= 67M); 300M leaves room for larger benchmark blocks too. - gas_limit: 300000000 + # >= 67M). 1000M (1 G) gives generous headroom for the benchmark setup + # blocks. NB: this does NOT make test_deep_branch fillable — its worst-case + # deep-MPT setup needs a single block > 1 G, and EIP-1559 caps the per-block + # ramp to +/-1/1024, so the limit can't jump there (verified at 1000M; deep + # stays excluded below). All 225 other bloatnet cases fill fine at any of + # these values; the funding fixes, not the gas_limit, are what matter. + gas_limit: 1000000000 # Must exceed the prestate size (~77 GB) or state-actor truncates entities. target_size: 200GB # jochem-brouwer's production-minimum repricing prestate. The big cost is @@ -98,13 +103,16 @@ builder: parameters: code_pattern: unique_jumpdest_pre_amsterdam salt_count: 150000 - # Sender pool for test_ether_transfers_onchain_receivers - # (EOA(key=SENDER_BASE_KEY + i), 150000 funded). + # Sender pool for test_ether_transfers_onchain_receivers, which derives + # senders as EOA(key=SENDER_BASE_KEY + i) and assumes them pre-funded. + # SENDER_BASE_KEY = keccak256("gas-repricings-private-key") on this EEST + # branch (NOT the gist's 0x1111…, which left every sender unfunded); the + # first sender is 0x4e5e4cbb…377a. 150000 funded. - kind: contract name: sender-pool-300m template: sequential_pkey_eoas parameters: - start_pkey: "0x1111111111111111111111111111111111111111111111111111111111111111" + start_pkey: "0xe1e1d3457c4e69b29cba0f7e1f92ce080d4db56d221bed913b09b2753bd97c7a" count: 150000 balance: "1000000000000000000" # Bittrex CREATE-preimage chain (test_account_access EXISTING_CONTRACT). @@ -121,10 +129,10 @@ builder: targets: - client: geth output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth + - client: nethermind + output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind # - client: reth # +~77 GB each — enable only with disk to spare # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth - # - client: nethermind - # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind # - client: besu # output_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu # - client: ethrex @@ -142,7 +150,7 @@ builder: fork: amsterdam # Hardhat/Anvil dev key #0 → 0xf39Fd6…2266 (the pre-funded seed above). rpc_seed_key: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - # The bloatnet prestate has 300M-gas headroom; start lower for faster fills. + # The bloatnet prestate has ample gas headroom; start lower for faster fills. gas_benchmark_values: [30] # COW is mandatory at this size — overlayfs (Linux/OrbStack). On the host # this would be `copy` (infeasible). See .hack/orbstack.sh. @@ -171,25 +179,21 @@ builder: # test_setup_contracts: deploys 120k contracts from scratch in one # fixture (~120k blocks) — a state-builder, not a quick benchmark. # test_deep_branch: deploys worst-case deep MPT contracts at fill time - # whose blocks exceed the 300M genesis gas limit (EIP-1559 caps the - # per-block ramp, so the limit can't jump). Needs a much higher - # genesis gas_limit (bigger snapshot) or prestate-deployed mined - # contracts — see the worst_case_miner submodule. - # test_ether_transfers_onchain_receivers: the EOA(SENDER_BASE_KEY+i) - # sender pool the test expects funded doesn't match what state-actor's - # sequential_pkey_eoas plants (an address resolves to 0 balance). + # in a single block that exceeds the block gas limit (EIP-1559 caps + # the per-block ramp, so the limit can't jump to fit it). Verified + # unfillable even at gas_limit 1000M; needs prestate-deployed mined + # contracts instead — see the worst_case_miner submodule. # test_extcodesize_bytecode_sizes, the zero_to_nonzero sstore variants, # test_sload_bloated_prefetch_miss, and the existing_slots_True # multi_contract cases: amsterdam gas/BAL divergence (these target the # post-amsterdam repricing; we fill against amsterdam). # The two param-scoped excludes keep this functions' passing cases (12 - # sstore_variants, 2 multi_contract). Result: 207 filled, 0 failed. + # sstore_variants, 2 multi_contract). Result: 225 filled, 0 failed. filter: >- not erc20 and not test_multi_opcode and not test_setup_contracts and not test_deep_branch - and not test_ether_transfers_onchain_receivers and not test_extcodesize_bytecode_sizes and not test_sload_bloated_prefetch_miss and not (test_sstore_variants and zero_to_nonzero) @@ -214,9 +218,10 @@ runner: enabled: true max_retries: 10 backoff: 1s - # genesis entries are only needed for the non-geth clients (commented out - # below); add them back alongside their instances + datadirs. - # nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + # nethermind reads a parity-format chainspec; amsterdam is activated via the + # per-instance genesis_eip_override below. Other clients (reth/besu/ethrex/ + # erigon) need their genesis added alongside their instances + datadirs. + nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json # reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json # besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json # ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json @@ -225,8 +230,10 @@ runner: geth: source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth method: overlayfs + nethermind: + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + method: overlayfs # reth: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth, method: overlayfs } - # nethermind: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind, method: overlayfs } # besu: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu, method: overlayfs } # ethrex: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex, method: overlayfs } # erigon: { source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon, method: overlayfs } @@ -245,11 +252,11 @@ runner: # genesis_fork_override: { amsterdam: 1 } # extra_args: [--engine.slow-block-threshold=0] # retry_new_payloads_failed_state: { enabled: true, max_retries: 10, backoff: 1s } - # - id: nethermind - # client: nethermind - # image: nethermindeth/nethermind:master - # genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } - # extra_args: [--Init.BaseDbPath=/data, --Blocks.ParallelExecution=true, --Blocks.ParallelExecutionBatchRead=true] + - id: nethermind + client: nethermind + image: nethermindeth/nethermind:master + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + extra_args: [--Init.BaseDbPath=/data, --Blocks.ParallelExecution=true, --Blocks.ParallelExecutionBatchRead=true] # - id: besu # client: besu # image: ethpandaops/besu:bal-devnet-7 From 8914b1b4ab8fd7eb2eaa62919ce755d9f0b5d4df Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 07:11:02 +0200 Subject: [PATCH 39/83] fix(examples): nest runner genesis under client.config.genesis in bloatnet full MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nethermind chainspec was placed directly under client.config, but the runner reads per-client genesis from the client.config.genesis map (see the simple stateful config). As a result the runner logged "No genesis configured" and aborted with "sets genesis_eip_override but has no genesis file". Nesting it under genesis: lets the override apply — nethermind now boots from the parity chainspec, applies the amsterdam EIP-time overrides, and reports the same genesis state root as geth (0xba79d01f…a202). Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- ...ate-actor-eest.full.amsterdam.stateful.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml index 0c04c35ca..f7b7fa5bf 100644 --- a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml @@ -218,14 +218,16 @@ runner: enabled: true max_retries: 10 backoff: 1s - # nethermind reads a parity-format chainspec; amsterdam is activated via the - # per-instance genesis_eip_override below. Other clients (reth/besu/ethrex/ - # erigon) need their genesis added alongside their instances + datadirs. - nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json - # reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json - # besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json - # ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json - # erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/chainspec.json + # Per-client genesis/chainspec the runner boots each instance from. geth + # needs none (it boots from the datadir + --override.amsterdam flag), so it + # is omitted; nethermind reads a parity-format chainspec and gets amsterdam + # via the per-instance genesis_eip_override below. + genesis: + nethermind: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + # reth: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/reth/chainspec.json + # besu: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + # ethrex: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/ethrex/ethrex-genesis.json + # erigon: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/erigon/chainspec.json datadirs: geth: source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/geth From 4687f216104bee6ad71c41ea99f0c25c977c1ade Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 10:37:08 +0200 Subject: [PATCH 40/83] feat(examples): add nethermind cross-client filler to bloatnet full stateful MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a second eest_payloads target that builds the bloatnet fixtures with nethermind instead of geth, to prove nethermind can drive fill-stateful. Validated: fills 225/0 and the fixtures replay clean on both geth and nethermind (0 InvalidBlockLevelAccessList). - filler_image must be nethermindeth/nethermind:master — it has testing_buildBlockV1 AND correct EIP-7928 BALs. The older testing_build_block_with_opcode_tracing image builds invalid BALs. - nethermind has no debug_setHead, so the per-test rewind relies on the eest_ref branch's debug_resetHead fallback + reading nonces/rewind state at start_block rather than "latest" (which nethermind's resetHead leaves stale). - bump eest_ref to devnets/bal/7-bench-cap-aware-deploy-nm-resetHead, which carries those fill-stateful fixes; geth filling is unaffected by them. The target writes to its own output_dir (EEST_FIXTURES_DIR_NM) and is independent of the runner, which still consumes the geth-filled set. Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- ...te-actor-eest.full.amsterdam.stateful.yaml | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml index f7b7fa5bf..633643e9d 100644 --- a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml @@ -145,7 +145,12 @@ builder: fill_dockerfile: Dockerfile.eest-filler pull_policy: if-not-present eest_repo: https://github.com/skylenet/execution-specs.git - eest_ref: devnets/bal/7-bench-cap-aware-deploy + # = devnets/bal/7-bench-cap-aware-deploy + the fill-stateful tooling fixes + # that let nethermind drive fill-stateful (debug_resetHead fallback, receipt + # `error` strip, and start_block-not-"latest" nonce/rewind reads). Geth + # filling is unaffected by them. PR'd to ethereum/execution-specs + # devnets/bal/7-bench; move this ref there once it lands. + eest_ref: devnets/bal/7-bench-cap-aware-deploy-nm-resetHead config: fork: amsterdam # Hardhat/Anvil dev key #0 → 0xf39Fd6…2266 (the pre-funded seed above). @@ -199,6 +204,42 @@ builder: and not (test_sstore_variants and zero_to_nonzero) and not (test_sload_bloated_multi_contract and existing_slots_True) + # Cross-client filler: build the SAME fixtures via nethermind instead of + # geth. EEST fixtures are client-agnostic, so the runner only needs one + # set (geth's, above) — this target exists to prove nethermind can drive + # fill-stateful. Validated: fills 225/0 and the fixtures replay clean on + # BOTH geth and nethermind (0 InvalidBlockLevelAccessList). It writes to + # its own output_dir and is otherwise independent; comment it out to skip + # the extra ~3 min fill if you only want the geth fixtures. + # + # MUST use the master image: it has testing_buildBlockV1 AND correct + # EIP-7928 BALs. The older nethermind testing_build_block_with_opcode_ + # tracing image builds INVALID BALs (rejected by geth and nethermind on + # replay). nethermind has no debug_setHead, so the per-test chain rewind + # relies on the eest_ref branch's debug_resetHead fallback. + - name: payload-generator-nethermind-stateful-full + filler_client: nethermind + filler_image: nethermindeth/nethermind:master + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + output_dir: ${EEST_FIXTURES_DIR_NM:-/tmp/benchmarkoor/eest-fixtures-amsterdam-bloatnet-nm} + address_stubs_file: ${EEST_STUBS_FILE:-/tmp/benchmarkoor/stubs-repricing-amsterdam.json} + tests: + - tests/benchmark/stateful/bloatnet + # Same exclusions as the geth target above (see those comments). + filter: >- + not erc20 + and not test_multi_opcode + and not test_setup_contracts + and not test_deep_branch + and not test_extcodesize_bytecode_sizes + and not test_sload_bloated_prefetch_miss + and not (test_sstore_variants and zero_to_nonzero) + and not (test_sload_bloated_multi_contract and existing_slots_True) + ## Stage 3: Run the bloatnet/repricing benchmarks runner: client_logs_to_stdout: true From 770290f98aca77900f238e471cb99b5c15c40198 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 12:34:44 +0200 Subject: [PATCH 41/83] feat(examples): add nethermind cross-client filler to simple compute/stateful configs Mirrors the bloatnet full-stateful config: each gains a nethermind eest_payloads target (filler_image nethermindeth/nethermind:master + genesis_eip_override) that builds the same fixtures via nethermind, writing to its own output_dir. The runner still consumes the geth-filled set. - bump eest_ref to devnets/bal/7-bench-cap-aware-deploy-nm-resetHead, whose per-test rewind falls back to debug_resetHead (nethermind has no debug_setHead) and reads nonces at start_block not "latest"; geth filling is unaffected. - master image is required (testing_buildBlockV1 + correct EIP-7928 BALs); the old testing_build_block_with_opcode_tracing image builds invalid BALs. - refresh the stale "blocked upstream" comments. Validated end-to-end with nethermind:master in the OrbStack VM: - simple stateful (eip7928): 8/8 filled, fixtures replay clean on geth + nethermind. - compute: 1032 filled (4 skipped, 0 failed) with the same filter as geth; fixtures replay clean on geth + nethermind (0 InvalidBlockLevelAccessList). Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- ...e-actor-eest.simple.amsterdam.compute.yaml | 60 +++++++++++++------ ...-actor-eest.simple.amsterdam.stateful.yaml | 26 +++++++- 2 files changed, 66 insertions(+), 20 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml index d02412d73..ca3eb936a 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml @@ -89,7 +89,11 @@ builder: fill_dockerfile: Dockerfile.eest-filler pull_policy: if-not-present eest_repo: https://github.com/skylenet/execution-specs.git - eest_ref: devnets/bal/7-bench-cap-aware-deploy + # = devnets/bal/7-bench-cap-aware-deploy + the fill-stateful fixes that let + # nethermind fill: the per-test rewind falls back to debug_resetHead + # (nethermind has no debug_setHead) and reads nonces at start_block, not + # "latest". Geth filling is unaffected by them. + eest_ref: devnets/bal/7-bench-cap-aware-deploy-nm-resetHead config: # filler_image is set per-target below so each target can fill with a # different client. Shared knobs stay here and are hoisted into every target. @@ -100,11 +104,10 @@ builder: rpc_seed_key: "0x0000000000000000000000000000000000000000000000000000000000000001" gas_benchmark_values: [10] # Millions of gas to parametrise against datadir_method: copy # 1GB snapshot; copy is fine (For bigger dirs we should use zfs/overlayfs/schelk) - # fill-stateful forces testing_buildBlockV1 + debug_setHead, so a filler must - # implement both. geth (ethpandaops/geth) does. besu/nethermind are plumbed - # in benchmarkoor (see fillerCommand in pkg/builder) but blocked upstream, so - # their targets are commented out below — uncomment to experiment once the - # upstream gaps close. + # fill-stateful drives the filler over testing_buildBlockV1; the per-test + # rewind uses debug_setHead (geth) or debug_resetHead (nethermind), and works + # without either (the block is built on its explicit start_block parent). geth + # and nethermind both fill; besu's target stays commented (upstream bug). # # amsterdam fill: filler_image must be a geth that registers --override.amsterdam # (bal-devnet-7 + the amsterdam-override patch); the geth target passes @@ -156,19 +159,38 @@ builder: # tests: # - tests/benchmark/compute # filter: bn128 - # nethermind: testing_buildBlockV1 works (Testing module in the special - # nethermindeth/nethermind:testing_build_block_with_opcode_tracing image), - # but debug_setHead is unimplemented (the per-test rewind aborts the run) - # and its receipt trips EEST's strict model. Re-enable when fixed. - # - name: payload-generator-nethermind - # filler_client: nethermind - # filler_image: nethermindeth/nethermind:testing_build_block_with_opcode_tracing - # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind - # genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json - # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-nethermind - # tests: - # - tests/benchmark/compute - # filter: bn128 + # nethermind cross-client filler. EEST fixtures are client-agnostic, so the + # runner only needs one set (geth's, above); this proves nethermind can fill. + # MUST use the master image (testing_buildBlockV1 + correct EIP-7928 BALs); + # the per-test rewind falls back to debug_resetHead (nethermind has no + # debug_setHead). Validated: nethermind passes the same filter as geth + # (1032 filled, 4 skipped, 0 failed; the fixtures replay clean on geth and + # nethermind). + - name: payload-generator-nethermind + filler_client: nethermind + filler_image: nethermindeth/nethermind:master + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + output_dir: ${EEST_FIXTURES_DIR_NM:-/tmp/benchmarkoor/eest-fixtures-amsterdam-nm} + tests: + - tests/benchmark/compute + filter: >- + not test_blockhash + and not test_blobhash + and not test_create + and not test_auth_transaction + and not test_unchunkified_bytecode + and not test_storage_access_cold + and not test_jumpdest_analysis + and not test_state_root_computation + and not test_contract_creation + and not test_contract_calling_many_addresses + and not test_selfdestruct_created + and not test_selfdestruct_initcode + and not test_block_full_data + and not test_prefetch_cold_storage + and not test_ext_account_query_cold ## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads runner: diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml index 362611945..251101939 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml @@ -94,7 +94,11 @@ builder: fill_dockerfile: Dockerfile.eest-filler pull_policy: if-not-present eest_repo: https://github.com/skylenet/execution-specs.git - eest_ref: devnets/bal/7-bench-cap-aware-deploy + # = devnets/bal/7-bench-cap-aware-deploy + the fill-stateful fixes that let + # nethermind fill: the per-test rewind falls back to debug_resetHead + # (nethermind has no debug_setHead) and reads nonces at start_block, not + # "latest". Geth filling is unaffected by them. + eest_ref: devnets/bal/7-bench-cap-aware-deploy-nm-resetHead config: fork: amsterdam # fill amsterdam fixtures (snapshot stays osaka) # Pin the seed key so its account (0x7e5f…bdf, privkey 1) can be pre-funded @@ -118,6 +122,26 @@ builder: output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures-amsterdam-stateful} tests: - tests/benchmark/stateful/eip7928_block_level_access_lists + # nethermind cross-client filler: builds the same fixtures via nethermind + # instead of geth (EEST fixtures are client-agnostic, so the runner only + # needs one set — geth's, above; this proves nethermind can fill). Writes + # to its own output_dir; comment it out to skip the extra fill. + # + # MUST use the master image: it has testing_buildBlockV1 AND correct + # EIP-7928 BALs. nethermind has no debug_setHead, so the per-test rewind + # relies on the eest_ref branch's debug_resetHead fallback. Validated: + # 8/8 filled, fixtures replay clean on geth and nethermind. + - name: payload-generator-nethermind-stateful + filler_client: nethermind + filler_image: nethermindeth/nethermind:master + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind + # nethermind reads a parity chainspec; amsterdam is activated via the + # per-instance genesis_eip_override (same EIP set as the runner instance). + genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/nethermind/parity-chainspec.json + genesis_eip_override: { timestamp: 1, eips: [7708, 7778, 7843, 7928, 7954, 7976, 7981, 8024, 8037] } + output_dir: ${EEST_FIXTURES_DIR_NM:-/tmp/benchmarkoor/eest-fixtures-amsterdam-stateful-nm} + tests: + - tests/benchmark/stateful/eip7928_block_level_access_lists ## Stage 3: Run benchmarks using the state-actor datadirs and the eest payloads runner: From 39831b8a2406a29728bc3e47739df7cd2b559c3a Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 13:23:18 +0200 Subject: [PATCH 42/83] feat(eest_payloads): add per-target `marker` for pytest -m Mirrors `filter` (-k): a new `marker` string on each eest_payloads target maps to fill-stateful's pytest `-m` marker expression, orthogonal to the `-k` node-id filter. e.g. `marker: repricing` selects the gas-repricing reference benchmarks, `marker: "not repricing"` excludes them. - config: EESTPayloadTarget.Marker (yaml `marker`). - builder: buildFillArgs appends `-m ` when set. - test + docs. Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- docs/configuration.md | 3 ++- pkg/builder/eest_payloads.go | 4 ++++ pkg/builder/eest_payloads_test.go | 10 ++++++++++ pkg/config/config.go | 17 +++++++++++------ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 62f336ae7..84c844980 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1681,7 +1681,8 @@ Identity/locator fields are target-only; the rest mirror `config` and are resolv | `genesis_eip_override` | object | – | Patch a parity/nethermind `genesis` at filler boot, setting `params.eipTransitionTimestamp` for each listed EIP. Fields: `timestamp` (uint), `eips` ([]uint). For the nethermind filler. Requires `genesis`; mutually exclusive with `genesis_fork_override`. | | `output_dir` | string | – | **Absolute** host path for the generated fixtures. Skipped if already populated unless `--force` / `force: true`. Written under `/blockchain_tests_stateful_engine/`. | | `tests` | []string | – | **Required.** pytest paths inside the fill image, e.g. `tests/benchmark/compute`. | -| `filter` | string | – | Optional pytest `-k` expression. | +| `filter` | string | – | Optional pytest `-k` expression (substring/node-id selection). | +| `marker` | string | – | Optional pytest `-m` marker expression (orthogonal to `filter`'s `-k`), e.g. `repricing` to select the gas-repricing reference benchmarks, or `not repricing`. | | `address_stubs_file` | string | – | **Absolute** host path to a `--address-stubs` JSON map, required by stub-dependent tests (e.g. bloatnet opcode tests). | | `force` | bool | `false` | Per-target override of `--force`: wipe `output_dir` before filling. | | `filler_image`, `fork`, `gas_benchmark_values`, `fixed_opcode_count`, `datadir_method`, `max_gas_per_test`, `rpc_seed_key`, `filler_extra_args` | — | from `config` | See the `config` table above. `fork` and `filler_image` are required after resolution. | diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index 3da6690f0..c1aa66666 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -798,6 +798,10 @@ func buildFillArgs( args = append(args, "-k", t.Filter) } + if t.Marker != "" { + args = append(args, "-m", t.Marker) + } + return args } diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go index 46195f5e7..1e2c84339 100644 --- a/pkg/builder/eest_payloads_test.go +++ b/pkg/builder/eest_payloads_test.go @@ -115,6 +115,16 @@ func TestBuildFillArgs(t *testing.T) { require.GreaterOrEqual(t, idx, 0, "-k flag must be present") require.Less(t, idx+1, len(got)) assert.Equal(t, "expr", got[idx+1]) + assert.NotContains(t, got, "-m", "-m absent when marker is unset") + + // The -m marker value must immediately follow the -m flag. + gotMarker := buildFillArgs(prefix, &config.EESTPayloadTarget{ + FillerClient: "geth", Fork: "Osaka", Tests: []string{"t"}, Marker: "repricing", + }, "1.2.3.4", spec, "0x1") + mIdx := slices.Index(gotMarker, "-m") + require.GreaterOrEqual(t, mIdx, 0, "-m flag must be present") + require.Less(t, mIdx+1, len(gotMarker)) + assert.Equal(t, "repricing", gotMarker[mIdx+1]) } func TestFillerGethCommand(t *testing.T) { diff --git a/pkg/config/config.go b/pkg/config/config.go index d3a753b94..fe606dc04 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -398,9 +398,9 @@ type EESTPayloadDefaults struct { // EESTPayloadTarget is one fixture-generation run. Identity/locator fields // (Name, FillerClient, SourceDir, OutputDir, Genesis, GenesisForkOverride, -// GenesisEIPOverride, Tests, Filter, AddressStubsFile) live exclusively on the -// target; the remaining fields mirror EESTPayloadDefaults and are resolved via -// ResolveTarget. +// GenesisEIPOverride, Tests, Filter, Marker, AddressStubsFile) live exclusively +// on the target; the remaining fields mirror EESTPayloadDefaults and are +// resolved via ResolveTarget. type EESTPayloadTarget struct { Name string `yaml:"name,omitempty" mapstructure:"name"` FillerClient string `yaml:"filler_client" mapstructure:"filler_client"` @@ -421,9 +421,14 @@ type EESTPayloadTarget struct { GenesisEIPOverride *GenesisEIPOverride `yaml:"genesis_eip_override,omitempty" mapstructure:"genesis_eip_override"` AddressStubsFile string `yaml:"address_stubs_file,omitempty" mapstructure:"address_stubs_file"` // Tests are pytest paths inside the fill image, e.g. tests/benchmark/compute. - Tests []string `yaml:"tests,omitempty" mapstructure:"tests"` - Filter string `yaml:"filter,omitempty" mapstructure:"filter"` - Force bool `yaml:"force,omitempty" mapstructure:"force"` + Tests []string `yaml:"tests,omitempty" mapstructure:"tests"` + // Filter is a pytest -k expression (substring/node-id selection). + Filter string `yaml:"filter,omitempty" mapstructure:"filter"` + // Marker is a pytest -m marker expression, orthogonal to Filter's -k. e.g. + // "repricing" to select the gas-repricing reference benchmarks, or + // "not repricing" to exclude them. + Marker string `yaml:"marker,omitempty" mapstructure:"marker"` + Force bool `yaml:"force,omitempty" mapstructure:"force"` // Hoistable fields (mirror EESTPayloadDefaults). FillerImage string `yaml:"filler_image,omitempty" mapstructure:"filler_image"` From 2485b4337b16c0f0cc822de7599b9b9a5f00ae61 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 13:24:34 +0200 Subject: [PATCH 43/83] docs: nethermind is also a supported eest_payloads filler The fill-stateful filler is no longer geth-only: nethermind:master implements testing_buildBlockV1 with correct EIP-7928 BALs, and the per-test rewind falls back to debug_resetHead (it has no debug_setHead). Update the stale "filler_client must be geth" callout and table row. Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 84c844980..95180d7aa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1601,7 +1601,7 @@ builder: `eest_payloads` generates **stateful** EEST benchmark fixtures: it boots a filler EL client on a *writable copy* of a pre-populated snapshot datadir, runs `fill-stateful` against the live client (recording engine-API payloads anchored to the snapshot's head block), and writes the fixtures to each target's `output_dir`. `fill-stateful` itself does not manage datadirs — benchmarkoor boots the filler and snapshots it. -> **Filler client:** only `geth` implements the `testing_buildBlockV1` API that `fill-stateful` drives, so `filler_client` must be `geth` today; `ethpandaops/geth:master` is the production-ready image. +> **Filler client:** `geth` (`ethpandaops/geth:master`) is the production-ready filler. `nethermind` (`nethermindeth/nethermind:master`) also works — it implements `testing_buildBlockV1` with correct EIP-7928 block-access-lists, and `fill-stateful`'s per-test rewind falls back to `debug_resetHead` for it (nethermind has no `debug_setHead`). `besu` is plumbed in benchmarkoor but blocked upstream. > > **Fill image:** there is no published `fill-stateful` image yet (the command lands in execution-specs [#2637](https://github.com/ethereum/execution-specs/pull/2637)). Build one from the repo's `Dockerfile.eest-filler` (it bundles `uv` + execution-specs) and point `fill_image` at it: > ```bash @@ -1674,7 +1674,7 @@ Identity/locator fields are target-only; the rest mirror `config` and are resolv | Option | Type | Default | Description | |---|---|---|---| | `name` | string | `filler_client` | Used by `--target` to filter. Must be unique across targets. | -| `filler_client` | string | – | Client booted as the filler. Must be `geth` (only client supporting `testing_buildBlockV1`). | +| `filler_client` | string | – | Client booted as the filler. `geth` or `nethermind` (both implement `testing_buildBlockV1`). | | `source_dir` | string | – | **Absolute** host path to the pristine snapshot datadir (e.g. a `state_actor` `output_dir`). Never mutated — a writable copy is filled. Existence is checked at build time. | | `genesis` | string | – | **Absolute** host path to the genesis/chainspec the filler boots with (besu/nethermind read their fork schedule from it; passed via the client's genesis flag). Must match the chain config used to produce `source_dir`. geth/erigon boot from the datadir instead and need no `genesis`. | | `genesis_fork_override` | map | – | Patch the geth-format `genesis` at filler boot to activate forks at given timestamps (`{amsterdam: 1}` → `config.amsterdamTime`, inheriting the blob schedule). For besu/reth/ethrex fillers. Same mechanism as the runner. Requires `genesis`. | From 7ade07cf628e13e4aea38ab21c153e04003b6f38 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 16:03:53 +0200 Subject: [PATCH 44/83] feat(examples): keep test_blockhash on the nethermind compute filler (geth excludes it) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compute geth filler can't fill test_blockhash: geth runs pathdb without retained history, so after the test's 256-block chain, start_block's state is unreachable ("historical state is not available") — debug_setHead silently resets geth to genesis and corrupts the state DB, and skipping debug doesn't help either (the natural reorg also can't rebuild on start_block). nethermind fills it fine, so drop `not test_blockhash` from the nethermind target only. Validated: nethermind fills 1037 (was 1032; +5 test_blockhash variants), 4 skipped, 0 failed; fixtures replay clean on nethermind (0 InvalidBlockLevelAccessList). Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- ...ig.state-actor-eest.simple.amsterdam.compute.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml index ca3eb936a..8c76f2167 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml @@ -163,9 +163,9 @@ builder: # runner only needs one set (geth's, above); this proves nethermind can fill. # MUST use the master image (testing_buildBlockV1 + correct EIP-7928 BALs); # the per-test rewind falls back to debug_resetHead (nethermind has no - # debug_setHead). Validated: nethermind passes the same filter as geth - # (1032 filled, 4 skipped, 0 failed; the fixtures replay clean on geth and - # nethermind). + # debug_setHead). Uses the geth filter minus `not test_blockhash` — geth + # can't fill that test but nethermind can. Validated: 1037 filled, 4 + # skipped, 0 failed; fixtures replay clean on nethermind. - name: payload-generator-nethermind filler_client: nethermind filler_image: nethermindeth/nethermind:master @@ -175,9 +175,11 @@ builder: output_dir: ${EEST_FIXTURES_DIR_NM:-/tmp/benchmarkoor/eest-fixtures-amsterdam-nm} tests: - tests/benchmark/compute + # test_blockhash is kept here but excluded on the geth target: geth's + # pathdb can't rebuild on start_block after the test's 256-block chain + # (historical state unavailable), so geth corrupts; nethermind fills it. filter: >- - not test_blockhash - and not test_blobhash + not test_blobhash and not test_create and not test_auth_transaction and not test_unchunkified_bytecode From 9be2af990aa7064035f13a98997a2e9572ed1b07 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 25 Jun 2026 21:51:23 +0200 Subject: [PATCH 45/83] feat(examples): enable the besu compute filler target besu can now fill EEST amsterdam compute (EIP-7928 BAL) fixtures over testing_buildBlockV1 once the filler image carries the TestingBuildBlockV1 coinbase fix (besu-eth/besu#10710): testing_buildBlockV1 credited fees + the block access list to suggestedFeeRecipient but wrote the node's configured coinbase (Address.ZERO) into the header, so every block self-rejected on engine_newPayload with a BAL hash mismatch. Replaces the stale commented-out besu target with a working one pointing at skylenet/besu:bal-devnet-7-testing-coinbase-fix, using the same test filter as the nethermind cross-client filler. Validated: 0 BAL mismatches, fixtures replay clean. Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- ...e-actor-eest.simple.amsterdam.compute.yaml | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml index 8c76f2167..7d7341d3a 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml @@ -146,19 +146,53 @@ builder: and not test_block_full_data and not test_prefetch_cold_storage and not test_ext_account_query_cold - # besu: testing_buildBlockV1 (TESTING namespace, besu-eth/besu#9838), - # debug_setHead and the session-fee pin all work, but besu's self-built - # block fails its own engine_newPayloadV4 with a World-State-Root mismatch - # (upstream besu bug in testing_buildBlockV1). Re-enable when fixed. - # - name: payload-generator-besu - # filler_client: besu - # filler_image: hyperledger/besu:26.6.1 # >= 26.6 ships testing_buildBlockV1 - # source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu - # genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json - # output_dir: ${EEST_FIXTURES_DIR:-/tmp/benchmarkoor/eest-fixtures}-besu - # tests: - # - tests/benchmark/compute - # filter: bn128 + # besu cross-client filler. besu drives the fill over testing_buildBlockV1 + # (TESTING namespace, besu-eth/besu#9838) just like nethermind. Earlier besu + # self-rejected every block on engine_newPayloadV5 with a BAL hash mismatch: + # testing_buildBlockV1 credited fees + the EIP-7928 block access list to the + # request's suggestedFeeRecipient but wrote the node's configured coinbase + # (Address.ZERO) into the header, so re-execution recomputed the BAL under a + # different coinbase. Fixed by a one-line change in TestingBuildBlockV1 + # (miningConfiguration.setCoinbase(suggestedFeeRecipient) before building) — + # the filler_image below MUST carry that patch (besu bal-devnet-7 + + # TestingBuildBlockV1 coinbase fix). The eest builder auto-pins besu's session + # priority fee, so no extra args are needed. Validated against the same filter + # as nethermind: 0 BAL mismatches, fixtures replay clean. (besu fills the + # BLS12-381 MSM benchmarks too — they only fail at gas_benchmark_values [1], + # where the k=128 MSM call's ~1.33M intrinsic calldata gas exceeds the 1M tx + # limit; at [10] above they fit.) + - name: payload-generator-besu + filler_client: besu + # besu bal-devnet-7 built with the TestingBuildBlockV1 coinbase fix (see the + # comment above). Build+push from a patched besu checkout, or swap for an + # upstream image once the fix lands in besu-eth/besu's bal-devnet-7 line. + filler_image: skylenet/besu:bal-devnet-7-testing-coinbase-fix + source_dir: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu + genesis: ${STATE_DIR_PREFIX:-/tmp/benchmarkoor/state-actor}/besu/besu-chainspec.json + # besu reads forks from the geth-format genesis; benchmarkoor patches + # amsterdamTime in at boot (snapshot block 0 ts is 0, so amsterdam at ts 1). + genesis_fork_override: { amsterdam: 1 } + output_dir: ${EEST_FIXTURES_DIR_BESU:-/tmp/benchmarkoor/eest-fixtures-amsterdam-besu} + tests: + - tests/benchmark/compute + # Identical exclusions to nethermind (keeps test_blockhash, which besu also + # fills). besu fills the same compute set as nethermind once the BAL coinbase + # fix is in the filler_image. + filter: >- + not test_blobhash + and not test_create + and not test_auth_transaction + and not test_unchunkified_bytecode + and not test_storage_access_cold + and not test_jumpdest_analysis + and not test_state_root_computation + and not test_contract_creation + and not test_contract_calling_many_addresses + and not test_selfdestruct_created + and not test_selfdestruct_initcode + and not test_block_full_data + and not test_prefetch_cold_storage + and not test_ext_account_query_cold # nethermind cross-client filler. EEST fixtures are client-agnostic, so the # runner only needs one set (geth's, above); this proves nethermind can fill. # MUST use the master image (testing_buildBlockV1 + correct EIP-7928 BALs); From cafd153ff5e92684c85b86fddc763e7cfcdd9fb3 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 26 Jun 2026 07:17:51 +0200 Subject: [PATCH 46/83] chore(typos): allow the BAL / BALs identifiers typos splits the identifier "BALs" (EIP-7928 block access lists) into "BA" + "Ls" and flags "BA" as a typo of "BY"/"BE", failing CI on the amsterdam EEST config comments. Whitelist the BAL/BALs identifiers via [default.extend-identifiers] so the token is skipped before word splitting. Verified clean with typos v1.42.0 (the CI version). Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- _typos.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/_typos.toml b/_typos.toml index 32b561106..6e94fac16 100644 --- a/_typos.toml +++ b/_typos.toml @@ -5,3 +5,10 @@ extend-exclude = ["go.mod"] PANC = "PANC" ERRO = "ERRO" BULD = "BULD" + +# EIP-7928 block access lists. typos splits the identifier "BALs" into "BA"+"Ls" +# and flags "BA" as a typo of "BY"/"BE"; allow the whole identifier (and the +# singular "BAL") so the BAL terminology in comments/config passes. +[default.extend-identifiers] +BAL = "BAL" +BALs = "BALs" From feeb56527eca2ae6eeac8333d42753b64aa91756 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 26 Jun 2026 10:49:42 +0200 Subject: [PATCH 47/83] feat(runner): copy EEST .meta into suite output as .eest-meta When the runner sources tests from an EEST fixtures dir that contains a .meta directory (fill provenance: fixtures.ini with the fill command + python/pytest/t8n versions, index.json, report_fill.html), copy it into each suite's output as results/suites//.eest-meta and set eest_metadata in summary.json so the UI can surface an "EEST Metadata" view without statting the suite directory. - fsutil.CopyDir: recursive, owner-aware directory copy - PreparedSource.MetaDir: set by the EEST source when /.meta exists (covers local dir, built, artifact and tarball modes via discoverTests) - CreateSuiteOutput: copies MetaDir -> .eest-meta (best-effort; a copy failure warns but never fails the suite) and flags eest_metadata from the on-disk dir so it stays correct on re-runs Verified end-to-end: a nethermind run over besu-filled compute fixtures produced results/suites//.eest-meta with the fill report/ini/index and summary.json eest_metadata=true. Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- pkg/executor/eest_source.go | 10 +++++ pkg/executor/source.go | 4 ++ pkg/executor/suite.go | 24 ++++++++++++ pkg/executor/suite_test.go | 75 +++++++++++++++++++++++++++++++++++++ pkg/fsutil/fsutil.go | 38 +++++++++++++++++++ pkg/fsutil/fsutil_test.go | 50 +++++++++++++++++++++++++ 6 files changed, 201 insertions(+) create mode 100644 pkg/fsutil/fsutil_test.go diff --git a/pkg/executor/eest_source.go b/pkg/executor/eest_source.go index 1b0a0a391..00a6f56f5 100644 --- a/pkg/executor/eest_source.go +++ b/pkg/executor/eest_source.go @@ -637,6 +637,16 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { Tests: make([]*TestWithSteps, 0), } + // EEST writes a .meta directory (fixtures.ini with the fill command + python/ + // tool versions, index.json, report_fill.html) at the root of the fixtures + // dir. Attach it when present so each suite's output can carry the provenance. + metaDir := filepath.Join(s.fixturesDir, ".meta") + if fi, err := os.Stat(metaDir); err == nil && fi.IsDir() { + result.MetaDir = metaDir + + s.log.WithField("meta_dir", metaDir).Debug("Found EEST .meta directory") + } + s.log.WithField("path", searchDir).Info("Searching for fixtures") // Load shared pre_run files (stateful-engine format). Keyed by start block diff --git a/pkg/executor/source.go b/pkg/executor/source.go index d000b3893..6eb8d8e73 100644 --- a/pkg/executor/source.go +++ b/pkg/executor/source.go @@ -57,6 +57,10 @@ type PreparedSource struct { BasePath string PreRunSteps []*StepFile Tests []*TestWithSteps + // MetaDir is the path to an auxiliary metadata directory to attach to each + // suite's output (e.g. an EEST fill's .meta dir with fixtures.ini, the fill + // report and index). Empty when the source has no such directory. + MetaDir string } // Source provides test files from local or git sources. diff --git a/pkg/executor/suite.go b/pkg/executor/suite.go index 558ac88ae..c4d703f99 100644 --- a/pkg/executor/suite.go +++ b/pkg/executor/suite.go @@ -24,6 +24,10 @@ type SuiteInfo struct { Metadata *config.MetadataConfig `json:"metadata,omitempty"` PreRunSteps []SuiteFile `json:"pre_run_steps,omitempty"` Tests []SuiteTest `json:"tests"` + // EESTMetadata is true when the suite output contains an .eest-meta + // directory copied from the EEST fixtures' .meta (fill provenance). Lets the + // UI surface an "EEST Metadata" view without statting the suite directory. + EESTMetadata bool `json:"eest_metadata,omitempty"` } // SuiteSource contains source information for the suite. @@ -183,6 +187,19 @@ func CreateSuiteOutput( return fmt.Errorf("creating suite dir: %w", err) } + // Attach the source's metadata directory (EEST .meta) as .eest-meta. + // Best-effort: the metadata is auxiliary provenance, so a copy failure + // must not fail the whole suite. + if prepared.MetaDir != "" { + metaDst := filepath.Join(suiteDir, ".eest-meta") + if err := fsutil.CopyDir(prepared.MetaDir, metaDst, owner); err != nil { + log.WithError(err).WithField("src", prepared.MetaDir). + Warn("Failed to copy EEST .meta into suite output") + } else { + log.WithField("dst", metaDst).Debug("Copied EEST .meta into suite output") + } + } + // Copy pre-run steps. // Structure: //pre_run.request (same pattern as tests). for _, f := range prepared.PreRunSteps { @@ -307,6 +324,13 @@ func CreateSuiteOutput( } } + // Reflect whether the suite carries an .eest-meta directory. Derived from + // the on-disk dir (not just this run's copy) so it stays correct on re-runs + // where the suite already existed and the copy step above was skipped. + if fi, err := os.Stat(filepath.Join(suiteDir, ".eest-meta")); err == nil && fi.IsDir() { + info.EESTMetadata = true + } + // Always write summary.json — metadata (e.g. labels) can change between // runs without affecting the suite hash, so we update it every time. summaryPath := filepath.Join(suiteDir, "summary.json") diff --git a/pkg/executor/suite_test.go b/pkg/executor/suite_test.go index 8bc3cf170..cec0339da 100644 --- a/pkg/executor/suite_test.go +++ b/pkg/executor/suite_test.go @@ -76,6 +76,81 @@ func TestSuiteInfo_BackwardCompat_LoadsOldSummary(t *testing.T) { assert.Nil(t, parsed.Tests[0].PayloadSizes) } +func TestCreateSuiteOutput_CopiesEESTMeta(t *testing.T) { + tmp := t.TempDir() + + // Build a source .meta dir with a top-level file and a nested file. + metaDir := filepath.Join(tmp, "fixtures", ".meta") + require.NoError(t, os.MkdirAll(filepath.Join(metaDir, "assets"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(metaDir, "fixtures.ini"), + []byte("[environment]\npython = 3.12.13\n"), 0o644)) + require.NoError(t, os.WriteFile( + filepath.Join(metaDir, "assets", "style.css"), []byte("body{}"), 0o644)) + + prepared := &PreparedSource{ + MetaDir: metaDir, + Tests: []*TestWithSteps{ + { + Name: "test_meta", + Test: &StepFile{ + Name: "test_meta", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + info := &SuiteInfo{Hash: "abc123"} + + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "abc123", info, prepared, nil)) + + suiteMeta := filepath.Join(tmp, "suites", "abc123", ".eest-meta") + + gotIni, err := os.ReadFile(filepath.Join(suiteMeta, "fixtures.ini")) + require.NoError(t, err) + assert.Contains(t, string(gotIni), "python = 3.12.13") + + gotCSS, err := os.ReadFile(filepath.Join(suiteMeta, "assets", "style.css")) + require.NoError(t, err) + assert.Equal(t, "body{}", string(gotCSS)) + + // summary.json flags the metadata so the UI can surface it. + data, err := os.ReadFile(filepath.Join(tmp, "suites", "abc123", "summary.json")) + require.NoError(t, err) + + var parsed SuiteInfo + require.NoError(t, json.Unmarshal(data, &parsed)) + assert.True(t, parsed.EESTMetadata) +} + +func TestCreateSuiteOutput_NoEESTMetaWhenAbsent(t *testing.T) { + tmp := t.TempDir() + prepared := &PreparedSource{ + Tests: []*TestWithSteps{ + { + Name: "test_nometa", + Test: &StepFile{ + Name: "test_nometa", + Provider: &inlineProvider{lines: []string{minimalDenebRequest(t)}}, + }, + }, + }, + } + info := &SuiteInfo{Hash: "nometa01"} + + require.NoError(t, CreateSuiteOutput(logrus.New(), tmp, "nometa01", info, prepared, nil)) + + _, err := os.Stat(filepath.Join(tmp, "suites", "nometa01", ".eest-meta")) + assert.True(t, os.IsNotExist(err)) + + data, err := os.ReadFile(filepath.Join(tmp, "suites", "nometa01", "summary.json")) + require.NoError(t, err) + + var parsed SuiteInfo + require.NoError(t, json.Unmarshal(data, &parsed)) + assert.False(t, parsed.EESTMetadata) +} + func TestCreateSuiteOutput_MergesPayloadSizesOnSecondRun(t *testing.T) { tmp := t.TempDir() testLine := minimalDenebRequest(t) diff --git a/pkg/fsutil/fsutil.go b/pkg/fsutil/fsutil.go index 96aa4d5b9..9cc7a1bc9 100644 --- a/pkg/fsutil/fsutil.go +++ b/pkg/fsutil/fsutil.go @@ -97,3 +97,41 @@ func Create(path string, owner *OwnerConfig) (*os.File, error) { return f, nil } + +// CopyDir recursively copies the directory tree rooted at src into dst, +// creating dst (and any missing parents) and applying owner to every directory +// and file it writes. Only regular files and directories are copied; symlinks +// and other special files are skipped. Intended for small auxiliary trees. +func CopyDir(src, dst string, owner *OwnerConfig) error { + entries, err := os.ReadDir(src) + if err != nil { + return fmt.Errorf("reading source dir %q: %w", src, err) + } + + if err := MkdirAll(dst, 0755, owner); err != nil { + return fmt.Errorf("creating dest dir %q: %w", dst, err) + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + switch { + case entry.IsDir(): + if err := CopyDir(srcPath, dstPath, owner); err != nil { + return err + } + case entry.Type().IsRegular(): + data, err := os.ReadFile(srcPath) + if err != nil { + return fmt.Errorf("reading %q: %w", srcPath, err) + } + + if err := WriteFile(dstPath, data, 0644, owner); err != nil { + return fmt.Errorf("writing %q: %w", dstPath, err) + } + } + } + + return nil +} diff --git a/pkg/fsutil/fsutil_test.go b/pkg/fsutil/fsutil_test.go new file mode 100644 index 000000000..505582459 --- /dev/null +++ b/pkg/fsutil/fsutil_test.go @@ -0,0 +1,50 @@ +package fsutil + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyDir(t *testing.T) { + src := t.TempDir() + + // Lay out a small tree: top-level file + a nested dir with a file. + require.NoError(t, os.MkdirAll(filepath.Join(src, "assets"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "fixtures.ini"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "assets", "style.css"), []byte("body{}"), 0o644)) + + dst := filepath.Join(t.TempDir(), "copy") + + require.NoError(t, CopyDir(src, dst, nil)) + + top, err := os.ReadFile(filepath.Join(dst, "fixtures.ini")) + require.NoError(t, err) + assert.Equal(t, "hello", string(top)) + + nested, err := os.ReadFile(filepath.Join(dst, "assets", "style.css")) + require.NoError(t, err) + assert.Equal(t, "body{}", string(nested)) +} + +func TestCopyDir_CreatesMissingParents(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "f.txt"), []byte("x"), 0o644)) + + // Destination parents do not exist yet. + dst := filepath.Join(t.TempDir(), "a", "b", "c") + + require.NoError(t, CopyDir(src, dst, nil)) + + got, err := os.ReadFile(filepath.Join(dst, "f.txt")) + require.NoError(t, err) + assert.Equal(t, "x", string(got)) +} + +func TestCopyDir_MissingSource(t *testing.T) { + err := CopyDir(filepath.Join(t.TempDir(), "does-not-exist"), t.TempDir(), nil) + require.Error(t, err) +} From 6fe7c9fa1ff63f2389132cc06bc58bc90d30ff9e Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 26 Jun 2026 11:17:28 +0200 Subject: [PATCH 48/83] feat(ui): show EEST build metadata in the suite Source tab When a suite carries an .eest-meta directory (summary.json eest_metadata=true), render a collapsible "EEST Build Metadata" section inside the Source tab: the parsed fixtures.ini (python/t8n/pytest versions, packages, plugins and the fill command line), links to the fill report and fixture index, and an inline iframe of the fill report. The report is a light-mode HTML page (often served cross-origin, so its document can't be styled from here); in dark mode the iframe is inverted with invert(.9) hue-rotate(180deg) so it blends with the UI palette (dark-gray bg, near-white text) instead of going stark black/white, tracking the UI theme via the dark variant. - ui SuiteInfo gains the eest_metadata field - new EESTMetadata component fetches and parses .eest-meta/fixtures.ini - pkg/api: regression test confirming the local file server serves the dot-prefixed .eest-meta files the UI fetches (no API change was needed) Claude-Session: https://claude.ai/code/session_0143C1YkFtZ7gYbYkxbadHVw --- pkg/api/local_test.go | 19 ++ ui/src/api/types.ts | 3 + .../components/suite-detail/EESTMetadata.tsx | 167 ++++++++++++++++++ ui/src/pages/SuiteDetailPage.tsx | 6 +- 4 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 ui/src/components/suite-detail/EESTMetadata.tsx diff --git a/pkg/api/local_test.go b/pkg/api/local_test.go index d2351223b..2c9f5b0c5 100644 --- a/pkg/api/local_test.go +++ b/pkg/api/local_test.go @@ -75,6 +75,25 @@ func TestLocalFileServer_ServeFile(t *testing.T) { assert.Contains(t, rec.Body.String(), `{"ok":true}`) }) + t.Run("serves dotfile directory (.eest-meta)", func(t *testing.T) { + metaDir := filepath.Join(root, "suites", "abc", ".eest-meta") + require.NoError(t, os.MkdirAll(metaDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(metaDir, "fixtures.ini"), + []byte("[environment]\npython = 3.12.13\n"), 0o644)) + + req := httptest.NewRequest( + http.MethodGet, + "/mydata/suites/abc/.eest-meta/fixtures.ini", nil, + ) + rec := httptest.NewRecorder() + + err := srv.ServeFile(rec, req, "mydata/suites/abc/.eest-meta/fixtures.ini") + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), "python = 3.12.13") + }) + t.Run("returns error for missing file", func(t *testing.T) { req := httptest.NewRequest( http.MethodGet, diff --git a/ui/src/api/types.ts b/ui/src/api/types.ts index dd9b4d781..5f983e5c9 100644 --- a/ui/src/api/types.ts +++ b/ui/src/api/types.ts @@ -439,6 +439,9 @@ export interface SuiteInfo { } pre_run_steps?: SuiteFile[] tests: SuiteTest[] + // eest_metadata is true when the suite output carries an .eest-meta directory + // (EEST fill provenance) copied from the fixtures' .meta. + eest_metadata?: boolean } export interface SuiteTestEEST { diff --git a/ui/src/components/suite-detail/EESTMetadata.tsx b/ui/src/components/suite-detail/EESTMetadata.tsx new file mode 100644 index 000000000..1512fa09c --- /dev/null +++ b/ui/src/components/suite-detail/EESTMetadata.tsx @@ -0,0 +1,167 @@ +import { useMemo } from 'react' + +import { useQuery } from '@tanstack/react-query' +import { ExternalLink } from 'lucide-react' + +import { fetchText } from '@/api/client' +import { Card } from '@/components/shared/Card' +import { getDataUrl, loadRuntimeConfig } from '@/config/runtime' + +interface EESTMetadataProps { + suiteHash: string +} + +interface IniSection { + name: string + entries: [string, string][] +} + +// parseIni parses a minimal INI document (sections + key=value pairs) into +// ordered sections, preserving order and skipping comments and blank lines. +function parseIni(text: string): IniSection[] { + const sections: IniSection[] = [] + let current: IniSection | null = null + + for (const rawLine of text.split('\n')) { + const line = rawLine.trim() + if (line === '' || line.startsWith(';') || line.startsWith('#')) continue + + const section = line.match(/^\[(.+)\]$/) + if (section) { + current = { name: section[1], entries: [] } + sections.push(current) + + continue + } + + const eq = line.indexOf('=') + if (eq === -1) continue + + if (!current) { + current = { name: '', entries: [] } + sections.push(current) + } + + current.entries.push([line.slice(0, eq).trim(), line.slice(eq + 1).trim()]) + } + + return sections +} + +// EESTMetadata renders the EEST fill provenance copied into a suite's +// .eest-meta directory: the parsed fixtures.ini (tool/python versions, fill +// command, packages/plugins) plus links to and an embed of the fill report. +// Rendered as a single collapsible card so it sits inside the Source tab. +export function EESTMetadata({ suiteHash }: EESTMetadataProps) { + const iniQuery = useQuery({ + queryKey: ['eest-meta-fixtures-ini', suiteHash], + queryFn: () => fetchText(`suites/${suiteHash}/.eest-meta/fixtures.ini`), + }) + + const { data: config } = useQuery({ + queryKey: ['runtime-config'], + queryFn: loadRuntimeConfig, + staleTime: Infinity, + }) + + const sections = useMemo( + () => (iniQuery.data?.data ? parseIni(iniQuery.data.data) : []), + [iniQuery.data], + ) + + // Don't render the section at all while loading or if the metadata is + // unreadable — the caller only mounts this when the suite advertises it. + if (iniQuery.isLoading || iniQuery.data?.data == null) { + return null + } + + const reportUrl = config + ? getDataUrl(`suites/${suiteHash}/.eest-meta/report_fill.html`, config) + : undefined + const indexUrl = config + ? getDataUrl(`suites/${suiteHash}/.eest-meta/index.json`, config) + : undefined + + return ( + +
+

+ Provenance recorded by execution-spec-tests when these fixtures were filled + (.eest-meta/fixtures.ini). +

+ + {sections.map((section) => ( +
+

+ {section.name || 'general'} +

+
+ {section.entries.map(([key, value]) => ( +
+
+ {key} +
+
+ {value} +
+
+ ))} +
+
+ ))} + + {(reportUrl || indexUrl) && ( +
+

+ fill report +

+
+ {reportUrl && ( + + Open fill report + + + )} + {indexUrl && ( + + View fixture index (JSON) + + + )} +
+ {reportUrl && ( +
+ {/* The fill report is a light-mode HTML page rendered in an iframe + (often cross-origin, so its document can't be styled from here). + Approximate dark mode by inverting the rendered frame when the + UI is in dark mode. invert(.9) (not full 1.0) softens the + extremes — white bg → dark gray (~#1a1a1a) and black text → + near-white — so it blends with the UI palette instead of going + stark black/white; hue-rotate keeps hues roughly correct. Tracks + the UI theme automatically via the dark variant. */} +