diff --git a/Cargo.lock b/Cargo.lock index 81d301a37..eccb61ffc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18853,6 +18853,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "async-trait", + "backon", "clap", "dotenvy", "hex", diff --git a/proofs/metrics/src/lib.rs b/proofs/metrics/src/lib.rs index d4f995093..6b9231b46 100644 --- a/proofs/metrics/src/lib.rs +++ b/proofs/metrics/src/lib.rs @@ -38,6 +38,10 @@ pub const METRICS_PROOF_JOBS_CLAIMED: &str = "proof_jobs.claimed"; pub const METRICS_PROOF_JOBS_COMPLETED: &str = "proof_jobs.completed"; /// End-to-end worker proof-job attempt duration. pub const METRICS_PROOF_JOB_DURATION_SECONDS: &str = "proof_job.duration_seconds"; +/// Whether this worker's enclave signing key is registered on-chain. +pub const METRICS_ENCLAVE_KEY_REGISTERED: &str = "enclave_key.registered"; +/// Enclave key registration attempts, by outcome. +pub const METRICS_ENCLAVE_REGISTRATION_ATTEMPTS: &str = "enclave_key.registration_attempts"; /// Registers shared metric descriptions. pub fn describe_metrics() { @@ -86,6 +90,31 @@ pub fn describe_metrics() { metrics::Unit::Seconds, "End-to-end worker proof-job attempt duration by backend and outcome." ); + metrics::describe_gauge!( + METRICS_ENCLAVE_KEY_REGISTERED, + metrics::Unit::Count, + "1 when this worker's enclave signing key is registered on-chain, 0 otherwise. \ + A worker holding 0 leases no proof jobs, because proofs signed by an unregistered \ + key do not verify." + ); + metrics::describe_counter!( + METRICS_ENCLAVE_REGISTRATION_ATTEMPTS, + metrics::Unit::Count, + "Enclave key registration attempts by outcome (registered, already_registered, failed)." + ); +} + +/// Sets the enclave-key registration gauge. +/// +/// Emitted eagerly at `0` on startup so the "never registered" case is a visible zero rather +/// than an absent series that a threshold monitor would silently ignore. +pub fn set_enclave_key_registered(registered: bool) { + metrics::gauge!(METRICS_ENCLAVE_KEY_REGISTERED).set(if registered { 1.0 } else { 0.0 }); +} + +/// Records an enclave key registration attempt and its outcome. +pub fn increment_enclave_registration_attempts(outcome: &'static str) { + metrics::counter!(METRICS_ENCLAVE_REGISTRATION_ATTEMPTS, "outcome" => outcome).increment(1); } /// Updates the latest finalized L2 block gauge. diff --git a/proofs/nitro/worker/Cargo.toml b/proofs/nitro/worker/Cargo.toml index 8396831bc..ca4f4f2a7 100644 --- a/proofs/nitro/worker/Cargo.toml +++ b/proofs/nitro/worker/Cargo.toml @@ -36,6 +36,7 @@ alloy-sol-types.workspace = true # Async / runtime tokio = { workspace = true, features = ["full", "signal"] } +backon.workspace = true # Error handling / logging anyhow.workspace = true diff --git a/proofs/nitro/worker/src/cmd/run.rs b/proofs/nitro/worker/src/cmd/run.rs index 8e4c6d011..d5cd837b0 100644 --- a/proofs/nitro/worker/src/cmd/run.rs +++ b/proofs/nitro/worker/src/cmd/run.rs @@ -4,8 +4,9 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; use alloy_primitives::B256; use anyhow::{Context, Result}; +use backon::{ExponentialBuilder, Retryable}; use clap::Parser; -use tracing::info; +use tracing::{error, info}; use world_chain_chainspec::WorldChainSpec; use world_chain_nitro_worker::{NitroBackend, NitroBackendConfig, build_expected_pcrs}; use world_chain_proof_kona_host_utils::online::{ @@ -25,6 +26,81 @@ const DEFAULT_SUBMIT_PROOF_RETRY_MAX_DELAY_MS: u64 = 10_000; const DEFAULT_WORKER_HEARTBEAT_INTERVAL_SEC: u64 = 30; const DEFAULT_WORKER_MAX_CONSECUTIVE_HEARTBEAT_FAILURES: u32 = 5; +/// First backoff interval between enclave key registration attempts. +const REGISTER_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(5); +/// Ceiling for the registration backoff. Registration can block on a human (approving a PCR +/// set, funding the key), so the ceiling is minutes rather than seconds. +const REGISTER_RETRY_MAX_DELAY: Duration = Duration::from_secs(300); + +/// Backoff for enclave key registration: exponential, jittered, and **unbounded**. +/// +/// Jitter matters because worker replicas share one funding key — un-jittered retries would +/// collide on the same nonce every interval and keep failing as a group. +fn registration_backoff() -> ExponentialBuilder { + ExponentialBuilder::default() + .with_min_delay(REGISTER_RETRY_INITIAL_DELAY) + .with_max_delay(REGISTER_RETRY_MAX_DELAY) + .with_jitter() + .without_max_times() +} + +/// Registers the enclave key on-chain, retrying until it succeeds or shutdown is requested. +/// Returns `false` only when shutdown won. +/// +/// This deliberately never aborts the process. Every way registration can fail — PCR set not +/// yet approved on-chain, registration key unfunded, L1 unreachable, certificate chain not yet +/// verifiable — is a condition an operator resolves *while the worker is running*. +async fn register_with_retry(params: RegisterParams) -> bool { + let attempt = || { + let params = params.clone(); + async move { register_enclave_key(params).await } + }; + + let registration = attempt + .retry(registration_backoff()) + .notify(|error, delay| { + world_chain_proof_metrics::increment_enclave_registration_attempts("failed"); + error!( + ?error, + retry_in_secs = delay.as_secs(), + "enclave key registration failed; worker stays up and will retry without \ + leasing proof jobs" + ); + }); + + // Race the (unbounded) retry against shutdown so a pod being rolled does not have to wait + // out a full backoff interval. + tokio::select! { + outcome = registration => match outcome { + Ok(outcome) => { + let label = match outcome { + RegistrationOutcome::AlreadyRegistered => { + info!("enclave key already registered on-chain"); + "already_registered" + } + RegistrationOutcome::Registered { tx_hash } => { + info!(%tx_hash, "enclave key registered on-chain"); + "registered" + } + }; + world_chain_proof_metrics::increment_enclave_registration_attempts(label); + world_chain_proof_metrics::set_enclave_key_registered(true); + true + } + // Unreachable while the backoff is unbounded, but treat it the same as shutdown + // rather than leasing jobs with an unregistered key. + Err(error) => { + error!(?error, "enclave key registration gave up; not leasing proof jobs"); + false + } + }, + _ = tokio::signal::ctrl_c() => { + info!("received ctrl-c while retrying registration, shutting down"); + false + } + } +} + #[derive(Debug, Clone, Copy, clap::ValueEnum)] enum Network { #[value(name = "worldchain")] @@ -197,8 +273,10 @@ pub async fn run(args: WorkerArgs) -> Result<()> { args.pcr2.as_deref(), )?; - // Optionally self-register the enclave's generated signing key on-chain before leasing - // any jobs. Without a registered key the proofs this worker submits would not verify. + // Self-register the enclave's generated signing key on-chain before leasing any jobs: + // proofs signed by an unregistered key do not verify, so an unregistered worker must not + // take work. Registration is a *precondition*, not a fatal error — see + // [`register_with_retry`] for why this never aborts the process. if args.auto_register { let registry = args .registry @@ -215,12 +293,16 @@ pub async fn run(args: WorkerArgs) -> Result<()> { REGISTER_PRIVATE_KEY, or PRIVATE_KEY", )?; + // Publish the gauge before the first attempt so "never registered" is a visible zero + // rather than an absent series a threshold monitor would silently ignore. + world_chain_proof_metrics::set_enclave_key_registered(false); info!( registry = %registry, enclave_cid = args.enclave_cid, - "auto-register enabled; registering enclave key on-chain before starting" + "auto-register enabled; registering enclave key on-chain before leasing jobs" ); - let outcome = register_enclave_key(RegisterParams { + + let registered = register_with_retry(RegisterParams { enclave_cid: args.enclave_cid, enclave_port: args.enclave_port, expected_pcrs, @@ -228,15 +310,10 @@ pub async fn run(args: WorkerArgs) -> Result<()> { registry, private_key, }) - .await - .context("auto-registration failed")?; - match outcome { - RegistrationOutcome::AlreadyRegistered => { - info!("enclave key already registered on-chain"); - } - RegistrationOutcome::Registered { tx_hash } => { - info!(%tx_hash, "enclave key registered on-chain"); - } + .await; + if !registered { + // Shutdown was requested while retrying; exit cleanly rather than starting up. + return Ok(()); } } @@ -296,3 +373,39 @@ pub async fn run(args: WorkerArgs) -> Result<()> { worker.await; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use backon::BackoffBuilder; + + /// The registration backoff must be unbounded: the whole point of this change is that a + /// worker keeps retrying (and stays exec-able) instead of crashlooping. A `with_max_times` + /// creeping in here would silently restore the give-up behaviour. + #[test] + fn registration_backoff_never_gives_up() { + let mut backoff = registration_backoff().build(); + // Far more attempts than any bounded policy would allow. + for i in 0..10_000 { + assert!( + backoff.next().is_some(), + "backoff stopped yielding delays at attempt {i}" + ); + } + } + + /// Delays must stay within the configured ceiling so a stuck worker retries on a + /// predictable cadence rather than backing off unboundedly. + #[test] + fn registration_backoff_respects_the_delay_ceiling() { + let mut backoff = registration_backoff().build(); + for _ in 0..256 { + let delay = backoff.next().expect("unbounded backoff yields a delay"); + // `with_jitter` only ever adds to the base delay, bounded by the base itself. + assert!( + delay <= REGISTER_RETRY_MAX_DELAY * 2, + "delay {delay:?} exceeded the jittered ceiling" + ); + } + } +}