diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index 28abb67c06..6e1632cb5e 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -51,6 +51,10 @@ jobs: dockerfile: ./lez/sequencer/service/Dockerfile build_args: | STANDALONE=true + - name: sequencer_service-mdns + dockerfile: ./lez/sequencer/service/Dockerfile + build_args: | + FEATURES=mdns - name: indexer_service dockerfile: ./lez/indexer/service/Dockerfile build_args: "" @@ -94,5 +98,5 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: ${{ matrix.build_args }} build-contexts: risc0_base=docker-image://${{ secrets.DOCKER_REGISTRY }}/${{ github.repository }}/risc0_base:sha-${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.name }} diff --git a/.gitignore b/.gitignore index eb9abce970..48fdea85ad 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ result wallet-ffi/wallet_ffi.h bedrock_signing_key +# The devnet committee's keys are fixed and generated, and the compose file mounts them. +!lez/configs/docker-all-in-one/devnet/*/bedrock_signing_key sequencer_stake_signing_key integration_tests/configs/debug/ diff --git a/Cargo.lock b/Cargo.lock index a2e055515a..22b19b2807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2692,6 +2692,18 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" +[[package]] +name = "devnet_configs" +version = "0.1.0" +dependencies = [ + "anyhow", + "lee", + "logos-blockchain-key-management-system-service", + "sequencer_core", + "sequencer_stake_core", + "serde_json", +] + [[package]] name = "digest" version = "0.10.7" diff --git a/Cargo.toml b/Cargo.toml index af0fbf3599..63ff2dd29f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ members = [ "tools/integration_bench", "tools/cross_zone_chat", "tools/dashboard_gen", + "tools/devnet_configs", ] [workspace.dependencies] diff --git a/Justfile b/Justfile index 22e94d7006..dc82cd575c 100644 --- a/Justfile +++ b/Justfile @@ -53,6 +53,12 @@ regenerate-test-fixture: @echo "🧪 Regenerating test fixture" RISC0_DEV_MODE=1 RUST_LOG=info cargo run -p test_fixtures --bin regenerate_test_fixture +# Regenerate the four-node docker devnet's shared sequencer config and per-node keys (the genesis +# stakes the whole committee, so the signatures are resigned; commit the result). +regenerate-devnet-configs: + @echo "🕸️ Regenerating devnet sequencer config and keys" + @cargo run -q -p devnet_configs + # Regenerate the committed Grafana dashboards from the Rust generator # (tools/dashboard_gen) and commit the result. CI checks these are up to date. regenerate-dashboards: @@ -209,5 +215,6 @@ clean: rm -rf lez/wallet/configs/debug/storage.json rm -rf lez/wallet/configs/debug/statistics.json rm -rf rocksdb* + docker compose down -v cd bedrock && docker compose down -v && cd .. cd monitoring && docker compose down -v && cd .. diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 4cdb486ff2..673c3fd558 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -5,12 +5,25 @@ services: environment: - RUST_LOG=error - sequencer_service: + sequencer_service_0: + depends_on: + - logos-blockchain-node-0 + - indexer_service + + sequencer_service_1: + depends_on: + - logos-blockchain-node-0 + - indexer_service + + sequencer_service_2: + depends_on: + - logos-blockchain-node-0 + - indexer_service + + sequencer_service_3: depends_on: - logos-blockchain-node-0 - indexer_service - volumes: - - ./lez/configs/docker-all-in-one/sequencer_config.json:/etc/sequencer_service/sequencer_config.json indexer_service: depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index 4da0d04c93..ffade30072 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ include: - path: bedrock/docker-compose.yml - path: - lez/sequencer/service/docker-compose.yml + lez/sequencer/service/docker-compose.devnet.yml - path: lez/indexer/service/docker-compose.yml - path: diff --git a/lez/configs/docker-all-in-one/devnet/seq-0/bedrock_signing_key b/lez/configs/docker-all-in-one/devnet/seq-0/bedrock_signing_key new file mode 100644 index 0000000000..acbce995b8 --- /dev/null +++ b/lez/configs/docker-all-in-one/devnet/seq-0/bedrock_signing_key @@ -0,0 +1 @@ +"gZ\#$G%:ps \ No newline at end of file diff --git a/lez/configs/docker-all-in-one/devnet/seq-0/signing_key b/lez/configs/docker-all-in-one/devnet/seq-0/signing_key new file mode 100644 index 0000000000..acbce995b8 --- /dev/null +++ b/lez/configs/docker-all-in-one/devnet/seq-0/signing_key @@ -0,0 +1 @@ +"gZ\#$G%:ps \ No newline at end of file diff --git a/lez/configs/docker-all-in-one/devnet/seq-1/bedrock_signing_key b/lez/configs/docker-all-in-one/devnet/seq-1/bedrock_signing_key new file mode 100644 index 0000000000..8a67f50059 --- /dev/null +++ b/lez/configs/docker-all-in-one/devnet/seq-1/bedrock_signing_key @@ -0,0 +1,2 @@ +呟4R +PHa+ \ No newline at end of file diff --git a/lez/configs/docker-all-in-one/devnet/seq-1/signing_key b/lez/configs/docker-all-in-one/devnet/seq-1/signing_key new file mode 100644 index 0000000000..8a67f50059 --- /dev/null +++ b/lez/configs/docker-all-in-one/devnet/seq-1/signing_key @@ -0,0 +1,2 @@ +呟4R +PHa+ \ No newline at end of file diff --git a/lez/configs/docker-all-in-one/devnet/seq-2/bedrock_signing_key b/lez/configs/docker-all-in-one/devnet/seq-2/bedrock_signing_key new file mode 100644 index 0000000000..3caafd069e --- /dev/null +++ b/lez/configs/docker-all-in-one/devnet/seq-2/bedrock_signing_key @@ -0,0 +1 @@ +}N>N'~t%T+_AR4` \ No newline at end of file diff --git a/lez/configs/docker-all-in-one/devnet/seq-2/signing_key b/lez/configs/docker-all-in-one/devnet/seq-2/signing_key new file mode 100644 index 0000000000..3caafd069e --- /dev/null +++ b/lez/configs/docker-all-in-one/devnet/seq-2/signing_key @@ -0,0 +1 @@ +}N>N'~t%T+_AR4` \ No newline at end of file diff --git a/lez/configs/docker-all-in-one/devnet/seq-3/bedrock_signing_key b/lez/configs/docker-all-in-one/devnet/seq-3/bedrock_signing_key new file mode 100644 index 0000000000..3349551f67 Binary files /dev/null and b/lez/configs/docker-all-in-one/devnet/seq-3/bedrock_signing_key differ diff --git a/lez/configs/docker-all-in-one/devnet/seq-3/signing_key b/lez/configs/docker-all-in-one/devnet/seq-3/signing_key new file mode 100644 index 0000000000..3349551f67 Binary files /dev/null and b/lez/configs/docker-all-in-one/devnet/seq-3/signing_key differ diff --git a/lez/configs/docker-all-in-one/devnet/sequencer_config.json b/lez/configs/docker-all-in-one/devnet/sequencer_config.json new file mode 100644 index 0000000000..9eae79b3ec --- /dev/null +++ b/lez/configs/docker-all-in-one/devnet/sequencer_config.json @@ -0,0 +1,217 @@ +{ + "home": "/var/lib/sequencer_service", + "max_num_tx_in_block": 20, + "max_block_size": "1.0 MiB", + "mempool_max_size": 10000, + "block_create_timeout": "10s", + "retry_pending_blocks_timeout": "7s", + "bedrock_config": { + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "node_url": "http://logos-blockchain-node-0:18080/", + "auth": null, + "funding_key": "2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26", + "priority_fee_percent": 12, + "channel_params": { + "minimum_sequencer_stake": 149, + "posting_timeframe": 60, + "posting_timeout": 25 + } + }, + "genesis": [ + { + "stake_sequencer": { + "sequencer_key": [ + 39, + 202, + 62, + 216, + 114, + 73, + 186, + 255, + 190, + 189, + 100, + 49, + 125, + 130, + 186, + 33, + 203, + 209, + 90, + 45, + 93, + 198, + 58, + 196, + 235, + 238, + 249, + 72, + 112, + 76, + 14, + 128 + ], + "ownership_public_key": "766d57e60d3e8b22d40920c428941191fb820d8e321a52bc6028dcf37ab6a4c1", + "stake_signature": "f05d0a9663e456ce4dff883d9133e46a85225d86db24a21d3d8700b15fda65482086b38a5aa9de5cda1f5772e632b7b1d505556dff4d914e1b3dc4c5cd560419" + } + }, + { + "stake_sequencer": { + "sequencer_key": [ + 99, + 109, + 167, + 55, + 119, + 13, + 230, + 240, + 217, + 130, + 164, + 76, + 216, + 51, + 124, + 252, + 208, + 17, + 126, + 55, + 91, + 224, + 170, + 168, + 120, + 133, + 142, + 248, + 164, + 44, + 23, + 206 + ], + "ownership_public_key": "5b53a6c0682f008dfae4e57aed512129a9b9565d833716dd6700d48fdbca6255", + "stake_signature": "bc110502ff0df80a6f2159f8de36adeeed804c39612522d51fc62dd224292f518ad3f7cbaeb899bdd42e4627a237bd3b6fdb16ec64ac33f8f9e493129e9c1880" + } + }, + { + "stake_sequencer": { + "sequencer_key": [ + 233, + 15, + 97, + 77, + 72, + 75, + 237, + 17, + 32, + 239, + 58, + 155, + 0, + 55, + 82, + 25, + 94, + 118, + 76, + 11, + 173, + 164, + 80, + 63, + 128, + 80, + 184, + 65, + 109, + 94, + 253, + 13 + ], + "ownership_public_key": "a9183a30a9cc2de17d9b75c217e59811303d69b86963603728350e2fd626c6f1", + "stake_signature": "e77a3502e7d5395905c89a6857e33ae45fb64a2b18020e68e6dfe9bd9cdf3ad0648539359f93ddbfd1d71f26700e7a11a03695d5f32f8a7ca1150058d2f12819" + } + }, + { + "stake_sequencer": { + "sequencer_key": [ + 69, + 240, + 101, + 139, + 207, + 1, + 135, + 210, + 194, + 142, + 98, + 226, + 240, + 132, + 168, + 26, + 29, + 204, + 168, + 234, + 24, + 146, + 49, + 36, + 232, + 100, + 118, + 10, + 140, + 96, + 211, + 111 + ], + "ownership_public_key": "94824e3f613d983b8092fea29454d3f777b551f95a12b1d2dbdc2f00c0130a23", + "stake_signature": "0740899cd9bf8115873adbdb0cc73af8b844557ea278e573be527a826044d0097acdb31be2f233280fc8d01a6d81c1677376c75b3127be2bc08064d47583462f" + } + }, + { + "supply_bridge_account": { + "balance": 1000000 + } + }, + { + "supply_account": { + "account_id": "6iArKUXxhUJqS7kCaPNhwMWt3ro71PDyBj7jwAyE2VQV", + "balance": 10000 + } + }, + { + "supply_account": { + "account_id": "7wHg9sbJwc6h3NP1S9bekfAzB8CHifEcxKswCKUt3YQo", + "balance": 20000 + } + }, + { + "supply_account": { + "account_id": "61EsoYN6gvTLkveh1YSTMG3yJkncpHy5EGmxhSK4ew29", + "balance": 10000 + } + }, + { + "supply_account": { + "account_id": "3m6HQmCgmAvsxZtxAHPqqEqoBG4335fCG8TzxigyW7rE", + "balance": 20000 + } + } + ], + "cross_zone": null, + "metrics_address": "0.0.0.0:9000", + "gossip": { + "listen_addr": "/ip4/0.0.0.0/udp/0/quic-v1", + "bootstrap_peers": [] + } +} diff --git a/lez/sequencer/actors/executor/src/actor/tests.rs b/lez/sequencer/actors/executor/src/actor/tests.rs index 8c55820acb..814e73dc91 100644 --- a/lez/sequencer/actors/executor/src/actor/tests.rs +++ b/lez/sequencer/actors/executor/src/actor/tests.rs @@ -38,7 +38,7 @@ fn sequencer_config() -> (SequencerConfig, TempDir) { mempool_max_size: 10, block_create_timeout: std::time::Duration::from_secs(5), retry_pending_blocks_timeout: std::time::Duration::from_secs(5), - signing_key: [37; 32], + signing_key: Some([37; 32]), bedrock_config: BedrockConfig { channel_id: [0; 32].into(), node_url: "http://not-used".parse().expect("Failed to parse URL"), diff --git a/lez/sequencer/core/src/config.rs b/lez/sequencer/core/src/config.rs index 73d6288f3d..246241d955 100644 --- a/lez/sequencer/core/src/config.rs +++ b/lez/sequencer/core/src/config.rs @@ -6,7 +6,7 @@ use std::{ time::Duration, }; -use anyhow::{Result, ensure}; +use anyhow::{Context as _, Result, ensure}; use bytesize::ByteSize; use common::config::BasicAuth; pub use cross_zone_inbox_core::{CrossZoneConfig, CrossZonePeer, CrossZoneRoute}; @@ -89,8 +89,13 @@ pub struct SequencerConfig { /// Interval in which pending blocks are retried. #[serde(with = "humantime_serde")] pub retry_pending_blocks_timeout: Duration, - /// Sequencer own signing key. - pub signing_key: [u8; 32], + /// The key this sequencer signs its blocks with. + /// + /// Absent when the node is given one separately (`--signing-key`), which is + /// what lets a set of sequencers share a single config file — it is all + /// that otherwise differs between them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signing_key: Option<[u8; 32]>, /// Bedrock configuration options. pub bedrock_config: BedrockConfig, /// Genesis configuration. @@ -150,6 +155,14 @@ impl SequencerConfig { Ok(config) } + /// The key [`Self::signing_key`] names, ready to sign blocks with. + pub fn block_signing_key(&self) -> Result { + let bytes = self.signing_key.context( + "No block signing key: set `signing_key` in the config, or pass --signing-key", + )?; + lee::PrivateKey::try_new(bytes).context("Block signing key is not a valid private key") + } + /// Where this sequencer's database lives, suffixed with the channel id like /// the indexer's, so several sequencers can share a home directory. Only the /// database is per-channel; `bedrock_signing_key` stays unsuffixed, so diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 068e43414e..07e013b1cd 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -323,7 +323,9 @@ impl SequencerCore { info!("Channel does not exist yet; starting it as channel creator"); } let bootstrap_sequencer_key = (!channel_already_exists).then_some(own_sequencer_key); - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + let signing_key = config + .block_signing_key() + .expect("Failed to load the block signing key"); Self::seed_genesis_if_absent(&storage_ref, &signing_key, bootstrap_sequencer_key, &config) .await; diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 85525844e6..b99c874524 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -135,7 +135,7 @@ fn setup_sequencer_config() -> SequencerConfig { max_block_size: bytesize::ByteSize::mib(1), mempool_max_size: 10000, block_create_timeout: Duration::from_secs(1), - signing_key: *sequencer_sign_key_for_testing().value(), + signing_key: Some(*sequencer_sign_key_for_testing().value()), bedrock_config: BedrockConfig { channel_id: ChannelId::from([0; 32]), node_url: "http://not-used-in-unit-tests".parse().unwrap(), @@ -551,7 +551,7 @@ async fn start_from_config_opens_existing_db_if_it_exists() { config.home = temp_dir.path().to_path_buf(); let bootstrap_sequencer_key = test_bootstrap_sequencer_key(&config); - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + let signing_key = config.block_signing_key().unwrap(); let (genesis_state, genesis_txs) = build_genesis_state(&signing_key, &config, Some(bootstrap_sequencer_key)); let genesis_hashable_data = HashableBlockData { @@ -4387,7 +4387,7 @@ fn a_fully_exited_ownership_account_can_stake_again() { fn genesis_stakes_the_bootstrap_sequencer_at_the_configured_account() { let config = setup_sequencer_config(); let bootstrap_sequencer_key = test_bootstrap_sequencer_key(&config); - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + let signing_key = config.block_signing_key().unwrap(); let (state, _genesis_txs) = build_genesis_state(&signing_key, &config, Some(bootstrap_sequencer_key)); @@ -4424,7 +4424,7 @@ fn genesis_stakes_the_bootstrap_sequencer_at_the_configured_account() { fn the_bootstrap_sequencer_can_request_an_unstake_of_its_genesis_stake() { let config = setup_sequencer_config(); let bootstrap_sequencer_key = test_bootstrap_sequencer_key(&config); - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + let signing_key = config.block_signing_key().unwrap(); let (mut state, _genesis_txs) = build_genesis_state(&signing_key, &config, Some(bootstrap_sequencer_key)); @@ -4824,7 +4824,7 @@ fn genesis_cross_zone_transactions_follow_the_declaration() { let mut config = setup_sequencer_config(); config.home = temp_dir.path().to_path_buf(); let key = test_bootstrap_sequencer_key(&config); - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + let signing_key = config.block_signing_key().unwrap(); let (state, txs) = build_genesis_state(&signing_key, &config, Some(key)); assert!( !txs.iter() @@ -4844,7 +4844,7 @@ fn genesis_cross_zone_transactions_follow_the_declaration() { source_governance: None, }); let key = test_bootstrap_sequencer_key(&config); - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + let signing_key = config.block_signing_key().unwrap(); let (state, txs) = build_genesis_state(&signing_key, &config, Some(key)); let cross_zone_txs: Vec<_> = txs .iter() diff --git a/lez/sequencer/core/src/tests/reconstruction.rs b/lez/sequencer/core/src/tests/reconstruction.rs index bebc122916..ac26508dd2 100644 --- a/lez/sequencer/core/src/tests/reconstruction.rs +++ b/lez/sequencer/core/src/tests/reconstruction.rs @@ -28,7 +28,7 @@ async fn fresh_store_and_chain( let storage_ref = StorageActor::spawn(storage); // What `start_from_config` does before it opens a store, mirrored here // because these cases drive `verify_and_reconstruct` directly. - let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); + let signing_key = config.block_signing_key().unwrap(); let bootstrap_sequencer_key = Some(test_bootstrap_sequencer_key(config)); SequencerCore::::seed_genesis_if_absent( &storage_ref, diff --git a/lez/sequencer/service/Dockerfile b/lez/sequencer/service/Dockerfile index 2e0b89a9d6..247b7692d2 100644 --- a/lez/sequencer/service/Dockerfile +++ b/lez/sequencer/service/Dockerfile @@ -5,6 +5,8 @@ WORKDIR /sequencer_service # Build argument to enable standalone feature (defaults to false) ARG STANDALONE=false +# Comma-separated extra cargo features, e.g. FEATURES=mdns for LAN peer discovery +ARG FEATURES="" # Planner stage - generates dependency recipe FROM chef AS planner @@ -14,17 +16,17 @@ RUN cargo chef prepare --bin sequencer_service --recipe-path recipe.json # Builder stage - builds dependencies and application FROM chef AS builder ARG STANDALONE +ARG FEATURES COPY --from=planner /sequencer_service/recipe.json recipe.json # Build dependencies only (this layer will be cached) RUN --mount=type=cache,target=/usr/local/cargo/registry/index \ --mount=type=cache,target=/usr/local/cargo/registry/cache \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/sequencer_service/target \ - if [ "$STANDALONE" = "true" ]; then \ - cargo chef cook --bin sequencer_service --features standalone --release --recipe-path recipe.json; \ - else \ - cargo chef cook --bin sequencer_service --release --recipe-path recipe.json; \ - fi + features="$FEATURES"; \ + if [ "$STANDALONE" = "true" ]; then features="${features:+$features,}standalone"; fi; \ + cargo chef cook --bin sequencer_service ${features:+--features "$features"} \ + --release --recipe-path recipe.json # Copy source code COPY . . @@ -34,11 +36,9 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry/index \ --mount=type=cache,target=/usr/local/cargo/registry/cache \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/sequencer_service/target \ - if [ "$STANDALONE" = "true" ]; then \ - cargo build --release --features standalone --bin sequencer_service; \ - else \ - cargo build --release --bin sequencer_service; \ - fi \ + features="$FEATURES"; \ + if [ "$STANDALONE" = "true" ]; then features="${features:+$features,}standalone"; fi; \ + cargo build --release ${features:+--features "$features"} --bin sequencer_service \ && strip /sequencer_service/target/release/sequencer_service \ && cp /sequencer_service/target/release/sequencer_service /usr/local/bin/sequencer_service diff --git a/lez/sequencer/service/docker-compose.devnet.yml b/lez/sequencer/service/docker-compose.devnet.yml new file mode 100644 index 0000000000..19b6509708 --- /dev/null +++ b/lez/sequencer/service/docker-compose.devnet.yml @@ -0,0 +1,125 @@ +# Four sequencers on one channel, the committee arrangement +# `integration_tests/tests/multi_sequencer.rs` drives in-process. +# +# All four run off one config file, whose genesis stakes all four block-signing +# keys: seq-0 opens the channel already accrediting the whole committee, and the +# others join a channel that expects them. A node's own directory holds all that +# is left to differ — the key it signs blocks with, handed over by +# `--signing-key`, and the Bedrock identity its stake accredits. +# `just regenerate-devnet-configs` rebuilds both. +# +# The image is built with the `mdns` feature, so the nodes find each other on +# the compose network and gossip their mempools without any address configured. +# +# Included by the repo-root compose file, which is where the Bedrock node and +# the indexer these configs name come from — hence the configs living with the +# rest of that project's, up in lez/configs/docker-all-in-one. + +x-sequencer-node: &sequencer_node + image: lez/sequencer_service:mdns + build: + context: ../../.. + dockerfile: lez/sequencer/service/Dockerfile + additional_contexts: + risc0_base: "service:risc0_base" + args: + FEATURES: mdns + # Lets a node reach a Bedrock published on the host, on Linux too, for a + # config pointed there rather than at the compose network. + extra_hosts: + - "host.docker.internal:host-gateway" + +# A follower that starts before the channel exists races seq-0 into creating a +# second one, so it waits the way the integration test's harness does. The node +# and the channel are read out of the config, so this follows whichever one is +# mounted. +x-await-channel: &await_channel + command: [] + entrypoint: + - /bin/sh + - -c + - | + config=/etc/sequencer_service/sequencer_config.json + node=$$(grep -o '"node_url"[^,]*' "$$config" | cut -d'"' -f4) + channel=$$(grep -o '"channel_id"[^,]*' "$$config" | cut -d'"' -f4) + until curl -s "$${node%/}/channel/$$channel" | grep -q accredited_keys; do + echo "waiting for the channel to land on Bedrock" + sleep 2 + done + exec sequencer_service "$$config" --signing-key /etc/sequencer_service/signing_key + +services: + # Build-only: shared base image (toolchain + r0vm) referenced as the + # `risc0_base` named context above. It has no long-running command, so it + # only gets built — it exits immediately if started. + risc0_base: + image: lez/risc0_base + build: + context: ../../.. + dockerfile: lez/docker/risc0-base.Dockerfile + + # Creates the channel and produces the genesis block; the others follow it. + sequencer_service_0: + <<: *sequencer_node + container_name: sequencer_service_0 + command: + - sequencer_service + - /etc/sequencer_service/sequencer_config.json + - --signing-key + - /etc/sequencer_service/signing_key + ports: + - "3040:3040" + - "9000:9000" + volumes: + - ../../configs/docker-all-in-one/devnet/sequencer_config.json:/etc/sequencer_service/sequencer_config.json + - ../../configs/docker-all-in-one/devnet/seq-0/signing_key:/etc/sequencer_service/signing_key + - ../../configs/docker-all-in-one/devnet/seq-0/bedrock_signing_key:/var/lib/sequencer_service/bedrock_signing_key + - sequencer_devnet_data_0:/var/lib/sequencer_service + + sequencer_service_1: + <<: [*sequencer_node, *await_channel] + container_name: sequencer_service_1 + depends_on: + - sequencer_service_0 + ports: + - "3041:3040" + - "9001:9000" + volumes: + - ../../configs/docker-all-in-one/devnet/sequencer_config.json:/etc/sequencer_service/sequencer_config.json + - ../../configs/docker-all-in-one/devnet/seq-1/signing_key:/etc/sequencer_service/signing_key + - ../../configs/docker-all-in-one/devnet/seq-1/bedrock_signing_key:/var/lib/sequencer_service/bedrock_signing_key + - sequencer_devnet_data_1:/var/lib/sequencer_service + + sequencer_service_2: + <<: [*sequencer_node, *await_channel] + container_name: sequencer_service_2 + depends_on: + - sequencer_service_0 + ports: + - "3042:3040" + - "9002:9000" + volumes: + - ../../configs/docker-all-in-one/devnet/sequencer_config.json:/etc/sequencer_service/sequencer_config.json + - ../../configs/docker-all-in-one/devnet/seq-2/signing_key:/etc/sequencer_service/signing_key + - ../../configs/docker-all-in-one/devnet/seq-2/bedrock_signing_key:/var/lib/sequencer_service/bedrock_signing_key + - sequencer_devnet_data_2:/var/lib/sequencer_service + + sequencer_service_3: + <<: [*sequencer_node, *await_channel] + container_name: sequencer_service_3 + depends_on: + - sequencer_service_0 + ports: + - "3043:3040" + - "9003:9000" + volumes: + - ../../configs/docker-all-in-one/devnet/sequencer_config.json:/etc/sequencer_service/sequencer_config.json + - ../../configs/docker-all-in-one/devnet/seq-3/signing_key:/etc/sequencer_service/signing_key + - ../../configs/docker-all-in-one/devnet/seq-3/bedrock_signing_key:/var/lib/sequencer_service/bedrock_signing_key + - sequencer_devnet_data_3:/var/lib/sequencer_service + +volumes: + sequencer_devnet_data_0: + sequencer_devnet_data_1: + sequencer_devnet_data_2: + sequencer_devnet_data_3: diff --git a/lez/sequencer/service/docker-compose.yml b/lez/sequencer/service/docker-compose.yml deleted file mode 100644 index d9c573d38b..0000000000 --- a/lez/sequencer/service/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -services: - # Build-only: shared base image (toolchain + r0vm) referenced as the - # `risc0_base` named context below. It has no long-running command, so it - # only gets built — it exits immediately if started. - risc0_base: - image: lez/risc0_base - build: - context: ../../.. - dockerfile: lez/docker/risc0-base.Dockerfile - - sequencer_service: - image: lez/sequencer_service - build: - context: ../../.. - dockerfile: lez/sequencer/service/Dockerfile - additional_contexts: - risc0_base: "service:risc0_base" - container_name: sequencer_service - ports: - - "3040:3040" - - "9000:9000" - volumes: - # Mount configuration file - - ./configs/docker/sequencer_config.json:/etc/sequencer_service/sequencer_config.json - # Mount data volume - - sequencer_data:/var/lib/sequencer_service - -volumes: - sequencer_data: diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index 9945ecd610..2770f24b74 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -3,7 +3,7 @@ use std::{ path::PathBuf, }; -use anyhow::{Context as _, Result}; +use anyhow::{Context as _, Result, anyhow}; use clap::Parser; use log::error; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; @@ -29,6 +29,12 @@ struct Args { /// one config file without fighting over the exporter port. #[clap(long)] metrics_address: Option, + /// File holding the 32-byte key to sign blocks with, overriding the + /// config's `signing_key`. It is the only thing that has to differ between + /// the sequencers of one committee, so passing it separately lets them all + /// run off one config file. + #[clap(long)] + signing_key: Option, } #[tokio::main] @@ -44,7 +50,10 @@ async fn main() -> Result<()> { let cancellation_token = listen_for_shutdown_signal(); let mut config = sequencer_service::SequencerConfig::from_path(&args.config_path)?; - apply_config_overrides(&args, &mut config); + apply_config_overrides(&args, &mut config)?; + // Resolved here so a node without a usable one says so now, rather than + // after the store is open and Bedrock has been probed. + config.block_signing_key()?; if let Some(metrics_address) = config.metrics_address { install_prometheus_recorder(metrics_address)?; @@ -73,10 +82,14 @@ async fn main() -> Result<()> { Ok(()) } -fn apply_config_overrides(args: &Args, config: &mut sequencer_service::SequencerConfig) { +fn apply_config_overrides( + args: &Args, + config: &mut sequencer_service::SequencerConfig, +) -> Result<()> { let Args { home, metrics_address, + signing_key, config_path: _, port: _, listen_address: _, @@ -88,6 +101,19 @@ fn apply_config_overrides(args: &Args, config: &mut sequencer_service::Sequencer if let Some(metrics_address) = metrics_address { config.metrics_address = Some(*metrics_address); } + if let Some(path) = signing_key { + let bytes = std::fs::read(path) + .with_context(|| format!("Failed to read the signing key at {}", path.display()))?; + config.signing_key = Some(bytes.try_into().map_err(|bytes: Vec| { + anyhow!( + "Signing key at {} is {} bytes, not 32", + path.display(), + bytes.len() + ) + })?); + } + + Ok(()) } /// Installs the recorder on `metrics_address`. diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index a4793aba77..9ca26e6d12 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -6,5 +6,9 @@ scrape_configs: - job_name: sequencer metrics_path: /metrics static_configs: + # The four-node devnet, each node's exporter published on its own port. - targets: - host.docker.internal:9000 + - host.docker.internal:9001 + - host.docker.internal:9002 + - host.docker.internal:9003 diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index b71927641a..6f40ed8159 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -164,7 +164,7 @@ pub fn sequencer_config( block_create_timeout, retry_pending_blocks_timeout: Duration::from_secs(5), genesis: genesis_transactions, - signing_key: signing_key.unwrap_or(SEQUENCER_SIGNING_KEY), + signing_key: Some(signing_key.unwrap_or(SEQUENCER_SIGNING_KEY)), bedrock_config: BedrockConfig { channel_id, node_url: addr_to_url(UrlProtocol::Http, bedrock_addr) diff --git a/tools/devnet_configs/Cargo.toml b/tools/devnet_configs/Cargo.toml new file mode 100644 index 0000000000..bc6fffc1e4 --- /dev/null +++ b/tools/devnet_configs/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "devnet_configs" +version = "0.1.0" +edition = "2024" +license.workspace = true + +[lints] +workspace = true + +[dependencies] +lee.workspace = true +sequencer_core.workspace = true +sequencer_stake_core.workspace = true + +logos-blockchain-key-management-system-service.workspace = true + +anyhow.workspace = true +serde_json.workspace = true diff --git a/tools/devnet_configs/README.md b/tools/devnet_configs/README.md new file mode 100644 index 0000000000..2568a2b79e --- /dev/null +++ b/tools/devnet_configs/README.md @@ -0,0 +1,34 @@ +# devnet_configs + +Builds what the four-node docker devnet runs on — `lez/sequencer/service/docker-compose.devnet.yml`, +included by the repo-root compose file. + +```sh +just regenerate-devnet-configs # cargo run -p devnet_configs +``` + +It writes, under `lez/configs/docker-all-in-one/devnet/`: + +- `sequencer_config.json`, shared by all four nodes. Extended from the single-node config next to it + with the genesis that stakes all four block-signing keys — so the leader opens the channel already + accrediting the whole committee — plus mDNS-discovered gossip and a turn short enough to watch + rotation happen. It carries no `signing_key`. +- `seq-/signing_key` and `seq-/bedrock_signing_key`, the only things left that differ between + the nodes. The first is handed to the binary with `--signing-key`; the second is the Bedrock + identity the node's stake accredits, mounted into its home. Both hold the same 32 random bytes, + drawn fresh on every run. + +The committee is arranged the way `integration_tests/tests/multi_sequencer.rs` arranges its own — +staked at genesis rather than joining later — but keyed independently of it. + +## When to rerun + +Most of the shared config is safe to edit by hand. The genesis stake entries are not: each +`stake_signature` covers the minimum stake and the node's position in the committee, so changing +`minimum_sequencer_stake`, the number of nodes, their order, or the stake message format invalidates +all four, and a node with an invalid founding stake panics as it applies genesis. Rerun here instead. + +Every run draws new keys and new signatures, so it always rewrites everything — there is no such +thing as a no-op rerun here. That also makes it a new committee on a new genesis block: wipe the +nodes' stores (`just clean`) before bringing the devnet back up, or they will come back to a channel +their fresh keys were never accredited on. diff --git a/tools/devnet_configs/src/main.rs b/tools/devnet_configs/src/main.rs new file mode 100644 index 0000000000..a8fb86ed0e --- /dev/null +++ b/tools/devnet_configs/src/main.rs @@ -0,0 +1,163 @@ +//! Regenerate what the four-node docker devnet runs on +//! (`lez/sequencer/service/docker-compose.devnet.yml`): one sequencer config +//! all four nodes share, and a key directory per node. +//! +//! The shared config carries the genesis that stakes all four block-signing +//! keys, so the leader opens the channel already accrediting the whole +//! committee and the followers replay the same chain — the arrangement +//! `integration_tests/tests/multi_sequencer.rs` drives in-process. What is left +//! per node is its keys, handed to the binary with `--signing-key`. +//! +//! Run via `just regenerate-devnet-configs`, then commit the result. See this +//! crate's README for what is safe to edit by hand instead. + +#![expect(clippy::print_stdout, reason = "It's normal in this small cli")] + +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result}; +use lee::{PrivateKey, PublicKey}; +use logos_blockchain_key_management_system_service::keys::Ed25519Key; +use sequencer_core::{ + config::{ChannelParams, GenesisAction, GossipConfig, SequencerConfig}, + sign_genesis_stake, +}; +use sequencer_stake_core::SequencerKey; + +/// Committee size the devnet compose file runs. +const NODES: usize = 4; + +/// Slots a node's posting turn lasts on the devnet channel. +/// +/// Below the production default the template carries: at 300 slots a turn comes +/// back round to a given node only every twenty minutes, so a devnet started to +/// watch rotation would show none. +const POSTING_TIMEFRAME: u32 = 60; + +/// Every interface, on an OS-assigned port: peers find each other over mDNS, so +/// nothing has to be reachable at a fixed address. +const GOSSIP_LISTEN_ADDR: &str = "/ip4/0.0.0.0/udp/0/quic-v1"; + +/// The single-node config the devnet extends, and where its own go. The devnet +/// runs as part of the all-in-one compose project, so it inherits that config's +/// genesis and its Bedrock node, reached over the compose network. +const CONFIGS_DIR: &str = "lez/configs/docker-all-in-one"; + +/// Name of the shared config the compose file mounts into every node. +const CONFIG_NAME: &str = "sequencer_config.json"; + +fn main() -> Result<()> { + let devnet_dir = repo_root().join(CONFIGS_DIR).join("devnet"); + let template_path = repo_root().join(CONFIGS_DIR).join(CONFIG_NAME); + let mut config = SequencerConfig::from_path(&template_path).with_context(|| { + format!( + "Failed to read the single-node template config at {}", + template_path.display() + ) + })?; + + // Nothing here may differ between the nodes: they build this genesis + // independently and have to arrive at the same chain. + config.bedrock_config.channel_params.posting_timeframe = POSTING_TIMEFRAME; + config.gossip = Some(GossipConfig { + listen_addr: GOSSIP_LISTEN_ADDR + .parse() + .expect("hardcoded gossip listen multiaddr is valid"), + bootstrap_peers: vec![], + }); + // Each node is given its own with --signing-key instead. + config.signing_key = None; + + let signing_keys: Vec<[u8; 32]> = std::iter::repeat_with(random_key).take(NODES).collect(); + let stakes = genesis_sequencer_stakes(&signing_keys, config.bedrock_config.channel_params) + .context("Failed to build the founding sequencer stakes")?; + // Ahead of the template's supplies: the stakes are funded from the faucet, + // and the accounts they credit are not the supplied ones. + config.genesis = stakes.into_iter().chain(config.genesis).collect(); + + write_config(&devnet_dir, &config)?; + println!("✅ Wrote {}", devnet_dir.join(CONFIG_NAME).display()); + + for (index, signing_key) in signing_keys.iter().enumerate() { + let dir = devnet_dir.join(format!("seq-{index}")); + write_keys(&dir, *signing_key) + .with_context(|| format!("Failed to write the keys for node {index}"))?; + println!("✅ Wrote {}", dir.display()); + } + + Ok(()) +} + +/// Fresh 32 bytes for one identity. +/// +/// Minted as a `PrivateKey` rather than straight off the RNG because a node's +/// key is read back as one through `--signing-key`, and not every 32 bytes are +/// a valid one — this is the constructor that keeps drawing until they are. +fn random_key() -> [u8; 32] { + *PrivateKey::new_os_random().value() +} + +/// Genesis entries staking every node, so the leader opens the channel already +/// accrediting all of them. +/// +/// `channel_params` must be the ones the channel is created with: the minimum +/// stake is signed over, and a founding stake below it accredits nobody. +fn genesis_sequencer_stakes( + signing_keys: &[[u8; 32]], + channel_params: ChannelParams, +) -> Result> { + let minimum_stake = channel_params.minimum_sequencer_stake; + signing_keys + .iter() + .enumerate() + .map(|(index, signing_key)| { + let public_key = Ed25519Key::from_bytes(signing_key).public_key(); + let sequencer_key = SequencerKey::new(public_key.to_bytes()) + .context("Sequencer signing key is not a valid Ed25519 point")?; + // Separate from the signing key: block signing and stake control + // are distinct roles. + let owner = PrivateKey::new_os_random(); + Ok(GenesisAction::StakeSequencer { + sequencer_key, + ownership_public_key: PublicKey::new_from_private_key(&owner), + // The index is signed over too, so these must stay in the order + // they are written to the genesis list. + stake_signature: sign_genesis_stake(index, sequencer_key, &owner, minimum_stake), + }) + }) + .collect() +} + +fn write_config(dir: &Path, config: &SequencerConfig) -> Result<()> { + std::fs::create_dir_all(dir).with_context(|| format!("Failed to create {}", dir.display()))?; + + let mut serialized = + serde_json::to_string_pretty(config).context("Failed to serialize the sequencer config")?; + serialized.push('\n'); + let path = dir.join(CONFIG_NAME); + std::fs::write(&path, serialized).context("Failed to write the sequencer config")?; + // The node reads this back at boot, and a config it rejects there is a + // devnet that only fails once it is started. + SequencerConfig::from_path(&path).context("Wrote a config the sequencer cannot load")?; + + Ok(()) +} + +/// Writes one node's two identities. Both are the same 32 bytes: the stake in +/// the shared genesis accredits this key, and it is the Bedrock identity that +/// has to post under it. +fn write_keys(dir: &Path, signing_key: [u8; 32]) -> Result<()> { + std::fs::create_dir_all(dir).with_context(|| format!("Failed to create {}", dir.display()))?; + + for name in ["signing_key", "bedrock_signing_key"] { + std::fs::write(dir.join(name), signing_key) + .with_context(|| format!("Failed to write {name}"))?; + } + + Ok(()) +} + +/// This crate sits two levels down, at `tools/devnet_configs`. +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") +}