Replace dry run mode with a test relay (flashbots#421)

## 📝 Summary

* removes dry run mode and validation from the rbuilder
* adds a test_relay binary that is used to accept and validate
submissions
* removes optimistic mode validation feature

## 💡 Motivation and Context

Dry run mode does not allow us to fully test submissions to the relays
so here its replaced with separate service that implements relay API.

Test builders does not need to point to real relays as test relays can
server validator data itself. This improves robustness in the tests as
production relays are not even mentioned in the test builder configs.


## ✅ I have completed the following steps:

* [x] Run `make lint`
* [x] Run `make test`
* [x] Added tests (if applicable)

---------

Co-authored-by: Chris Hager <[email protected]>

Author: dvush

.dockerignore

@@ -0,0 +1,13 @@
+/cargo
+/data
+/mev-test-contract/cache
+/mev-test-contract/out
+/target
+/scripts/benchmark-results.*
+/test/
+/integration_logs
+
+# editors
+.code
+.idea
+.vscode

Cargo.lock

@@ -15,9 +15,10 @@ members = [
     "crates/rbuilder/src/test_utils",
     "crates/rbuilder/src/telemetry/metrics_macros",
     "crates/eth-sparse-mpt",
-    "crates/sysperf"
+    "crates/sysperf",
+    "crates/test-relay",
 ]
-default-members = ["crates/rbuilder", "crates/reth-rbuilder"]
+default-members = ["crates/rbuilder", "crates/reth-rbuilder", "crates/test-relay"]
 resolver = "2"
 
 # Like release, but with full debug symbols. Useful for e.g. `perf`.
@@ -130,16 +131,15 @@ alloy-signer-local = { version = "0.9.2" }
 alloy-rpc-client = { version = "0.9.2" }
 alloy-genesis = { version = "0.9.2" }
 alloy-trie = { version = "0.7" }
-
-# op
 op-alloy-rpc-types = { version = "0.9.0", default-features = false }
 op-alloy-rpc-types-engine = { version =  "0.9.0", default-features = false }
 op-alloy-rpc-jsonrpsee = { version =  "0.9.0", default-features = false }
 op-alloy-network = { version =  "0.9.0", default-features = false }
 op-alloy-consensus = { version =  "0.9.0", default-features = false }
 
 async-trait = { version = "0.1.83" }
-clap = { version = "4.4.3" }
+clap = { version = "4.4.3", features = ["derive", "env"] }
+clap_builder = { version = "4.5.19" }
 thiserror = { version = "1.0.64" }
 eyre = { version = "0.6.12" }
 jsonrpsee = { version = "0.24.4" }
@@ -152,17 +152,24 @@ serde = { version = "1.0.210" }
 serde_json = { version = "1.0.128" }
 serde_with = { version = "3.8.1" }
 secp256k1 = { version = "0.29" }
-clap_builder = { version = "4.5.19" }
 derive_more = { version = "1" }
 tokio-stream = "0.1.16"
 tokio-util = "0.7.12"
 url = "2.5.2"
+warp = "0.3.7"
+flate2 = "1.0.35"
+prometheus = "0.13.4"
+ctor = "0.2"
 
 libc = { version = "0.2.161" }
 lazy_static = "1.4.0"
 tikv-jemallocator = { version = "0.6" }
 tracing = "0.1.37"
 metrics = { version = "0.24.1" }
+ahash = "0.8.6"
+time = { version = "0.3.36", features = ["macros", "formatting", "parsing"] }
 
 eth-sparse-mpt = { path = "crates/eth-sparse-mpt" }
+rbuilder = { path = "crates/rbuilder" }
 sysperf = { path = "crates/sysperf" }
+metrics_macros = { path = "crates/rbuilder/src/telemetry/metrics_macros"}

Cargo.toml

@@ -25,9 +25,7 @@ ENV SCCACHE_DIR=/sccache
 #
 FROM base AS planner
 WORKDIR /app
-
 COPY . .
-
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
     --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
@@ -38,28 +36,35 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
 #
 FROM base as builder
 WORKDIR /app
-# Default binary filename rbuilder
-# Alternatively can be set to "reth-rbuilder" - to have reth included in the binary
-ARG RBUILDER_BIN="rbuilder"
 COPY --from=planner /app/recipe.json recipe.json
-
 RUN --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
     cargo chef cook --release --recipe-path recipe.json
-
 COPY . .
 
+
+FROM builder as rbuilder
+ARG RBUILDER_BIN="rbuilder"
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
     --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
     cargo build --release --features="$FEATURES" --package=${RBUILDER_BIN}
 
-#
-# Runtime container
-#
-FROM gcr.io/distroless/cc-debian12
-WORKDIR /app
+FROM builder as test-relay
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
+    cargo build --release --features="$FEATURES" --package=test-relay
 
-ARG RBUILDER_BIN="rbuilder"
-COPY --from=builder /app/target/release/${RBUILDER_BIN} /app/rbuilder
 
+
+# Runtime container for rbuilder
+FROM gcr.io/distroless/cc-debian12 as rbuilder-runtime
+WORKDIR /app
+COPY --from=rbuilder /app/target/release/rbuilder /app/rbuilder
 ENTRYPOINT ["/app/rbuilder"]
+
+# Runtime container for test-relay
+FROM gcr.io/distroless/cc-debian12 as test-relay-runtime
+WORKDIR /app
+COPY --from=test-relay /app/target/release/test-relay /app/test-relay
+ENTRYPOINT ["/app/test-relay"]

Dockerfile

@@ -27,9 +27,13 @@ clean: ## Clean up
 build: ## Build (debug version)
 	cargo build --features "$(FEATURES)"
 
-.PHONY: docker-image
-docker-image: ## Build a rbuilder Docker image
-	docker build --platform linux/amd64 --build-arg FEATURES="$(FEATURES)" . -t rbuilder
+.PHONY: docker-image-rubilder
+docker-image-rubilder: ## Build a rbuilder Docker image
+	docker build --platform linux/amd64 --target rbuilder-runtime --build-arg FEATURES="$(FEATURES)"  . -t rbuilder
+
+.PHONY: docker-image-test-relay
+docker-image-test-relay: ## Build a test relay Docker image
+	docker build --platform linux/amd64 --target test-relay-runtime --build-arg FEATURES="$(FEATURES)" . -t test-relay
 
 ##@ Dev
 

Makefile

@@ -40,7 +40,6 @@ To run rbuilder you need:
 * Source of bundles that sends `eth_sendBundle`, `mev_sendBundle`, `eth_sendRawTransaction` as JSON rpc calls. (`jsonrpc_server_port`)
   (by default rbuilder will take raw txs from the reth node mempool)
 * Relays so submit to (`relays`)
-* Alternatively it can submit to the block validation API if run in the dry run mode (`dry_run`, `dry_run_validation_url`)
 
 A sample configuration for running Lighthouse and triggering payload events would be:
 ```
@@ -78,6 +77,12 @@ rbuilder has a solid initial benchmarking setup (based on [Criterion.rs](https:/
 - Benchmarks are located in [`crates/rbuilder/benches`](./crates/rbuilder/benches/). We'd love to add more meaningful benchmarks there!
 - Let us know about further improvement ideas and additional relevant benchmarks.
 
+### Testing with a fake relay
+
+This repo includes a `test-relay` tool for testing live builders without submitting blocks to live production relays. This standalone binary implements the MEV-Boost Relay API required for builder to function. The test-relay only performs block validation and compares profits between builders who submit blocks to it, without actually sending blocks to the network.
+
+The test relay exposes several Prometheus metrics about the blocks it received.
+
 ### End-to-end local testing
 
 You can use [builder-playground](https://github.com/flashbots/builder-playground) to deploy a fully functional local setup for the builder ([Lighthouse](https://github.com/sigp/lighthouse) consensus client (proposer + validator) + [Reth](https://github.com/paradigmxyz/reth/) execution client + [MEV-Boost-Relay](https://github.com/flashbots/mev-boost-relay))) to test rbuilder.
@@ -200,7 +205,7 @@ Big shoutout to the [Reth](https://github.com/paradigmxyz/reth) team for buildin
 
 
 | Binary                      | Description                                                                                           |
-| --------------------------- | ----------------------------------------------------------------------------------------------------- |
+|-----------------------------|-------------------------------------------------------------------------------------------------------|
 | `rbuilder`                  | Live block builder                                                                                    |
 | `backtest-build-block`      | Run backtests for a single block                                                                      |
 | `backtest-build-range`      | Run backtests for a range of block                                                                    |
@@ -211,3 +216,4 @@ Big shoutout to the [Reth](https://github.com/paradigmxyz/reth) team for buildin
 | `debug-order-input`         | Observe input of the bundles and transactions                                                         |
 | `debug-order-sim`           | Observe simulation of the bundles and transactions                                                    |
 | `debug-slot-data-generator` | Shows new payload jobs coming from CL with attached data from relays.                                 |
+| `test-relay`                | Test MEV-boost relay that accepts blocks and validates them.                                          |

README.md

@@ -22,9 +22,6 @@ extra_data = "⚡🤖"
 
 blocklist_file_path = "./blocklist.json"
 
-dry_run = true
-dry_run_validation_url = "http://localhost:8545"
-
 ignore_cancellable_orders = true
 
 # genesis_fork_version = "0x00112233"

config-live-example.toml

@@ -16,9 +16,6 @@ jsonrpc_server_ip = "0.0.0.0"
 el_node_ipc_path = "/tmp/reth.ipc"
 extra_data = "⚡🤖"
 
-dry_run = false
-dry_run_validation_url = "http://localhost:8545"
-
 ignore_cancellable_orders = true
 
 sbundle_mergeable_signers = []

config-optimism-local.toml

@@ -38,7 +38,7 @@ alloy-trie.workspace = true
 # test deps
 hash-db = { version = "0.15.2", optional = true }
 triehash = { version = "0.8.4", optional = true }
-flate2 = { version = "1.0.35", optional = true }
+flate2 = { workspace = true, optional = true }
 eyre = { workspace = true, optional = true}
 
 [features]

crates/eth-sparse-mpt/Cargo.toml

@@ -85,19 +85,19 @@ csv = "1.2.2"
 zip = "0.6.6"
 atoi = "2.0.0"
 futures = "0.3.28"
-time = { version = "0.3.36", features = ["macros", "formatting", "parsing"] }
+time.workspace = true
 bigdecimal = "0.4.1"
 mempool-dumpster = "0.1.1"
 itertools = "0.11.0"
-clap = { workspace = true, features = ["derive", "env"] }
+clap.workspace = true
 priority-queue = "2.0.3"
 secp256k1 = { workspace = true, features = [
     "global-context",
     "rand-std",
     "recovery",
 ] }
 rayon = "1.8.0"
-flate2 = "1.0.27"
+flate2.workspace = true
 # Version required by ethereum-consensus beacon-api-client
 mev-share-sse = { git = "https://github.com/paradigmxyz/mev-share-rs", rev = "9eb2b0138ab3202b9eb3af4b19c7b3bf40b0faa8", default-features = false }
 jsonrpsee = { version = "0.20.3", features = ["full"] }
@@ -106,13 +106,12 @@ beacon-api-client = { git = "https://github.com/ralexstokes/ethereum-consensus/"
 ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus/", rev = "cf3c404043230559660810bc0c9d6d5a8498d819" }
 ssz_rs_derive = { git = "https://github.com/ralexstokes/ssz-rs.git", version = "0.9.0" }
 uuid = { version = "1.6.1", features = ["serde", "v5", "v4"] }
-prometheus = "0.13.4"
-hyper = { version = "1.3.1", features = ["server", "full"] }
-warp = "0.3.7"
+prometheus.workspace = true
+warp.workspace = true
 lazy_static.workspace = true
-ctor = "0.2"
+ctor.workspace = true
 toml = "0.8.8"
-ahash = "0.8.6"
+ahash.workspace = true
 rand = "0.8.5"
 lru = "0.12.1"
 humantime = "2.1.0"

crates/rbuilder/Cargo.toml

@@ -9,4 +9,3 @@ pub mod provider;
 pub mod roothash;
 pub mod telemetry;
 pub mod utils;
-pub mod validation_api_client;