fix(nitro-worker): retry registration instead of crashlooping

proofs/metrics/src/lib.rs

@@ -38,6 +38,10 @@ pub const METRICS_PROOF_JOBS_CLAIMED: &str = "proof_jobs.claimed";
 pub const METRICS_PROOF_JOBS_COMPLETED: &str = "proof_jobs.completed";
 /// End-to-end worker proof-job attempt duration.
 pub const METRICS_PROOF_JOB_DURATION_SECONDS: &str = "proof_job.duration_seconds";
+/// Whether this worker's enclave signing key is registered on-chain.
+pub const METRICS_ENCLAVE_KEY_REGISTERED: &str = "enclave_key.registered";
+/// Enclave key registration attempts, by outcome.
+pub const METRICS_ENCLAVE_REGISTRATION_ATTEMPTS: &str = "enclave_key.registration_attempts";
 
 /// Registers shared metric descriptions.
 pub fn describe_metrics() {
@@ -86,6 +90,31 @@ pub fn describe_metrics() {
         metrics::Unit::Seconds,
         "End-to-end worker proof-job attempt duration by backend and outcome."
     );
+    metrics::describe_gauge!(
+        METRICS_ENCLAVE_KEY_REGISTERED,
+        metrics::Unit::Count,
+        "1 when this worker's enclave signing key is registered on-chain, 0 otherwise. \
+         A worker holding 0 leases no proof jobs, because proofs signed by an unregistered \
+         key do not verify."
+    );
+    metrics::describe_counter!(
+        METRICS_ENCLAVE_REGISTRATION_ATTEMPTS,
+        metrics::Unit::Count,
+        "Enclave key registration attempts by outcome (registered, already_registered, failed)."
+    );
+}
+
+/// Sets the enclave-key registration gauge.
+///
+/// Emitted eagerly at `0` on startup so the "never registered" case is a visible zero rather
+/// than an absent series that a threshold monitor would silently ignore.
+pub fn set_enclave_key_registered(registered: bool) {
+    metrics::gauge!(METRICS_ENCLAVE_KEY_REGISTERED).set(if registered { 1.0 } else { 0.0 });
+}
+
+/// Records an enclave key registration attempt and its outcome.
+pub fn increment_enclave_registration_attempts(outcome: &'static str) {
+    metrics::counter!(METRICS_ENCLAVE_REGISTRATION_ATTEMPTS, "outcome" => outcome).increment(1);
 }
 
 /// Updates the latest finalized L2 block gauge.

proofs/nitro/worker/Cargo.toml

@@ -36,6 +36,7 @@ alloy-sol-types.workspace = true
 
 # Async / runtime
 tokio = { workspace = true, features = ["full", "signal"] }
+backon.workspace = true
 
 # Error handling / logging
 anyhow.workspace = true

proofs/nitro/worker/src/cmd/run.rs

@@ -4,8 +4,9 @@ use std::{path::PathBuf, sync::Arc, time::Duration};
 
 use alloy_primitives::B256;
 use anyhow::{Context, Result};
+use backon::{ExponentialBuilder, Retryable};
 use clap::Parser;
-use tracing::info;
+use tracing::{error, info};
 use world_chain_chainspec::WorldChainSpec;
 use world_chain_nitro_worker::{NitroBackend, NitroBackendConfig, build_expected_pcrs};
 use world_chain_proof_kona_host_utils::online::{
@@ -25,6 +26,81 @@ const DEFAULT_SUBMIT_PROOF_RETRY_MAX_DELAY_MS: u64 = 10_000;
 const DEFAULT_WORKER_HEARTBEAT_INTERVAL_SEC: u64 = 30;
 const DEFAULT_WORKER_MAX_CONSECUTIVE_HEARTBEAT_FAILURES: u32 = 5;
 
+/// First backoff interval between enclave key registration attempts.
+const REGISTER_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(5);
+/// Ceiling for the registration backoff. Registration can block on a human (approving a PCR
+/// set, funding the key), so the ceiling is minutes rather than seconds.
+const REGISTER_RETRY_MAX_DELAY: Duration = Duration::from_secs(300);
+
+/// Backoff for enclave key registration: exponential, jittered, and **unbounded**.
+///
+/// Jitter matters because worker replicas share one funding key — un-jittered retries would
+/// collide on the same nonce every interval and keep failing as a group.
+fn registration_backoff() -> ExponentialBuilder {
+    ExponentialBuilder::default()
+        .with_min_delay(REGISTER_RETRY_INITIAL_DELAY)
+        .with_max_delay(REGISTER_RETRY_MAX_DELAY)
+        .with_jitter()
+        .without_max_times()
+}
+
+/// Registers the enclave key on-chain, retrying until it succeeds or shutdown is requested.
+/// Returns `false` only when shutdown won.
+///
+/// This deliberately never aborts the process. Every way registration can fail — PCR set not
+/// yet approved on-chain, registration key unfunded, L1 unreachable, certificate chain not yet
+/// verifiable — is a condition an operator resolves *while the worker is running*.
+async fn register_with_retry(params: RegisterParams) -> bool {
+    let attempt = || {
+        let params = params.clone();
+        async move { register_enclave_key(params).await }
+    };
+
+    let registration = attempt
+        .retry(registration_backoff())
+        .notify(|error, delay| {
+            world_chain_proof_metrics::increment_enclave_registration_attempts("failed");
+            error!(
+                ?error,
+                retry_in_secs = delay.as_secs(),
+                "enclave key registration failed; worker stays up and will retry without \
+             leasing proof jobs"
+            );
+        });
+
+    // Race the (unbounded) retry against shutdown so a pod being rolled does not have to wait
+    // out a full backoff interval.
+    tokio::select! {
+        outcome = registration => match outcome {
+            Ok(outcome) => {
+                let label = match outcome {
+                    RegistrationOutcome::AlreadyRegistered => {
+                        info!("enclave key already registered on-chain");
+                        "already_registered"
+                    }
+                    RegistrationOutcome::Registered { tx_hash } => {
+                        info!(%tx_hash, "enclave key registered on-chain");
+                        "registered"
+                    }
+                };
+                world_chain_proof_metrics::increment_enclave_registration_attempts(label);
+                world_chain_proof_metrics::set_enclave_key_registered(true);
+                true
+            }
+            // Unreachable while the backoff is unbounded, but treat it the same as shutdown
+            // rather than leasing jobs with an unregistered key.
+            Err(error) => {
+                error!(?error, "enclave key registration gave up; not leasing proof jobs");
+                false
+            }
+        },
+        _ = tokio::signal::ctrl_c() => {
+            info!("received ctrl-c while retrying registration, shutting down");
+            false
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy, clap::ValueEnum)]
 enum Network {
     #[value(name = "worldchain")]
@@ -197,8 +273,10 @@ pub async fn run(args: WorkerArgs) -> Result<()> {
         args.pcr2.as_deref(),
     )?;
 
-    // Optionally self-register the enclave's generated signing key on-chain before leasing
-    // any jobs. Without a registered key the proofs this worker submits would not verify.
+    // Self-register the enclave's generated signing key on-chain before leasing any jobs:
+    // proofs signed by an unregistered key do not verify, so an unregistered worker must not
+    // take work. Registration is a *precondition*, not a fatal error — see
+    // [`register_with_retry`] for why this never aborts the process.
     if args.auto_register {
         let registry = args
             .registry
@@ -215,28 +293,27 @@ pub async fn run(args: WorkerArgs) -> Result<()> {
                  REGISTER_PRIVATE_KEY, or PRIVATE_KEY",
             )?;
 
+        // Publish the gauge before the first attempt so "never registered" is a visible zero
+        // rather than an absent series a threshold monitor would silently ignore.
+        world_chain_proof_metrics::set_enclave_key_registered(false);
         info!(
             registry = %registry,
             enclave_cid = args.enclave_cid,
-            "auto-register enabled; registering enclave key on-chain before starting"
+            "auto-register enabled; registering enclave key on-chain before leasing jobs"
         );
-        let outcome = register_enclave_key(RegisterParams {
+
+        let registered = register_with_retry(RegisterParams {
             enclave_cid: args.enclave_cid,
             enclave_port: args.enclave_port,
             expected_pcrs,
             l1_rpc_url,
             registry,
             private_key,
         })
-        .await
-        .context("auto-registration failed")?;
-        match outcome {
-            RegistrationOutcome::AlreadyRegistered => {
-                info!("enclave key already registered on-chain");
-            }
-            RegistrationOutcome::Registered { tx_hash } => {
-                info!(%tx_hash, "enclave key registered on-chain");
-            }
+        .await;
+        if !registered {
+            // Shutdown was requested while retrying; exit cleanly rather than starting up.
+            return Ok(());
         }
     }
 
@@ -296,3 +373,39 @@ pub async fn run(args: WorkerArgs) -> Result<()> {
     worker.await;
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use backon::BackoffBuilder;
+
+    /// The registration backoff must be unbounded: the whole point of this change is that a
+    /// worker keeps retrying (and stays exec-able) instead of crashlooping. A `with_max_times`
+    /// creeping in here would silently restore the give-up behaviour.
+    #[test]
+    fn registration_backoff_never_gives_up() {
+        let mut backoff = registration_backoff().build();
+        // Far more attempts than any bounded policy would allow.
+        for i in 0..10_000 {
+            assert!(
+                backoff.next().is_some(),
+                "backoff stopped yielding delays at attempt {i}"
+            );
+        }
+    }
+
+    /// Delays must stay within the configured ceiling so a stuck worker retries on a
+    /// predictable cadence rather than backing off unboundedly.
+    #[test]
+    fn registration_backoff_respects_the_delay_ceiling() {
+        let mut backoff = registration_backoff().build();
+        for _ in 0..256 {
+            let delay = backoff.next().expect("unbounded backoff yields a delay");
+            // `with_jitter` only ever adds to the base delay, bounded by the base itself.
+            assert!(
+                delay <= REGISTER_RETRY_MAX_DELAY * 2,
+                "delay {delay:?} exceeded the jittered ceiling"
+            );
+        }
+    }
+}