diff --git a/Cargo.lock b/Cargo.lock index 81d301a37..eccb61ffc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18853,6 +18853,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "async-trait", + "backon", "clap", "dotenvy", "hex", diff --git a/Justfile b/Justfile index 295aa134a..ecb422972 100644 --- a/Justfile +++ b/Justfile @@ -144,9 +144,11 @@ install *args='': # Phase 2 proof-deploy-system – Deploy proof system contracts # Phase 3a proof-certmanager-prewarm – Pre-warm CertManager with CA certs # Phase 3b proof-approve-pcrs – Approve PCR set on verifier +# Phase 3c proof-verify-pcrs – Assert the RUNNING enclave's PCR set is approved +# (drift check; safe to run any time) # Phase 4 proof-register-key – Register the enclave's generated key on-chain # (run separately; NOT part of proof-setup) -# Combined proof-setup – Run deploy phases 0a–3b in sequence (does NOT +# Combined proof-setup – Run deploy phases 0a–3c in sequence (does NOT # run Phase 4 — register the key afterwards with # proof-register-key, or let the worker # self-register via `nitro-worker run --auto-register`) @@ -562,6 +564,67 @@ proof-approve-pcrs env="alphanet": fi echo "PCR set approved." +# Phase 3c – Verify the RUNNING enclave's PCR set is approved on-chain. +# +# Every EIF rollout changes PCR0 (enclave image) and PCR2 (application) while PCR1 +# (kernel) stays put. Nothing re-approves the new measurements automatically, so the +# running enclave silently drops off the on-chain allowlist and every registerKey / TEE +# proof reverts. +# +# This measures the enclave that is actually running rather than trusting PCR0/1/2 from +# the shell — trusting the shell would reproduce exactly the drift this is meant to catch. +# +# Required: L1_RPC_URL. Optional: NITRO_ATTESTATION_VERIFIER (else read from the +# {{env}}-nitro.json deployment). +proof-verify-pcrs env="alphanet": + #!/usr/bin/env bash + set -euo pipefail + if [ ! -f "scripts/proof-envs/{{env}}.env" ]; then + echo "Error: unknown env '{{env}}' — create scripts/proof-envs/{{env}}.env to configure it" >&2 + exit 1 + fi + DEPLOYMENTS_FILE="pkg/contracts/deployments/{{env}}-nitro.json" + if [ -z "${NITRO_ATTESTATION_VERIFIER:-}" ] && [ -f "$DEPLOYMENTS_FILE" ]; then + NITRO_ATTESTATION_VERIFIER=$(jq -r '.nitroAttestationVerifier // empty' "$DEPLOYMENTS_FILE") + fi + : "${NITRO_ATTESTATION_VERIFIER:?NITRO_ATTESTATION_VERIFIER is required (set it or run proof-deploy-nitro first)}" + : "${L1_RPC_URL:?L1_RPC_URL is required}" + + echo "Measuring the running enclave…" >&2 + eval "$(just proof-get-pcrs {{env}})" + : "${PCR0:?proof-get-pcrs did not return PCR0}" + : "${PCR1:?proof-get-pcrs did not return PCR1}" + : "${PCR2:?proof-get-pcrs did not return PCR2}" + [[ "$PCR0" == 0x* ]] || PCR0="0x$PCR0" + [[ "$PCR1" == 0x* ]] || PCR1="0x$PCR1" + [[ "$PCR2" == 0x* ]] || PCR2="0x$PCR2" + + # The verifier stores keccak256 of each raw 48-byte PCR, not the PCR itself. + APPROVED=$(cast call "$NITRO_ATTESTATION_VERIFIER" \ + "isPCRSetApproved(bytes32,bytes32,bytes32)(bool)" \ + "$(cast keccak "$PCR0")" "$(cast keccak "$PCR1")" "$(cast keccak "$PCR2")" \ + --rpc-url "$L1_RPC_URL") + + if [ "$APPROVED" != "true" ]; then + echo "" >&2 + echo "ERROR: the running enclave's PCR set is NOT approved on ${NITRO_ATTESTATION_VERIFIER}." >&2 + echo "" >&2 + echo " running enclave:" >&2 + echo " PCR0=$PCR0" >&2 + echo " PCR1=$PCR1" >&2 + echo " PCR2=$PCR2" >&2 + if [ -f "$DEPLOYMENTS_FILE" ]; then + echo " approved in $DEPLOYMENTS_FILE:" >&2 + jq -r '.approvedPCRSets[]? | " PCR0=\(.pcr0)\n PCR1=\(.pcr1)\n PCR2=\(.pcr2)"' \ + "$DEPLOYMENTS_FILE" >&2 || true + fi + echo "" >&2 + echo "Until this is approved, registerKey and every TEE proof will revert." >&2 + echo "Fix: OWNER_KEY=... just proof-approve-pcrs {{env}}" >&2 + exit 1 + fi + echo "Running enclave's PCR set is approved on ${NITRO_ATTESTATION_VERIFIER}." + # Phase 4 – Register the enclave's generated signing key on-chain. # Execs into the running nitro-worker pod (which has vsock access to the # enclave) and runs `nitro-worker register`, which fetches a public-key @@ -704,7 +767,14 @@ proof-setup env="alphanet": echo "=== Step 3b: Approving PCR set ===" >&2 just dry_run={{dry_run}} proof-approve-pcrs {{env}} - echo "=== Deploy phases 0a-3b complete. ===" >&2 + # Verify rather than assume: re-measure the running enclave and confirm the allowlist + # actually accepts it. A dry run approves nothing, so there is nothing to verify. + if [ "{{dry_run}}" = "false" ]; then + echo "=== Step 3c: Verifying the running enclave's PCR set ===" >&2 + just proof-verify-pcrs {{env}} + fi + + echo "=== Deploy phases 0a-3c complete. ===" >&2 echo "The game is registered but not activated; run 'just proof-activate-system {{env}}' after readiness checks." >&2 echo "Next: register the enclave signing key (Phase 4) with 'just proof-register-key {{env}}'," >&2 echo " or run the worker with '--auto-register' so it self-registers on startup." >&2 diff --git a/pkg/contracts/deployments/alphanet-nitro.json b/pkg/contracts/deployments/alphanet-nitro.json index 31ac06195..803a09fd3 100644 --- a/pkg/contracts/deployments/alphanet-nitro.json +++ b/pkg/contracts/deployments/alphanet-nitro.json @@ -5,6 +5,12 @@ "nitroEnclaveKeyRegistry": "0x4b3FE81C5b59f469622dBcA5FF2b2B1ABBC808a4", "nitroProofVerifier": "0xaF6dbf1e90C47A4eDE64EF948b75800633Eb8839", "approvedPCRSets": [ + { + "verifier": "0x52AE16Ec4FBf14c0Ff9B06229C1cd2403b4A22d0", + "pcr0": "0x5354c41cc47717d3843e9aff37d5b3cbd35d03e219ecce5a1e5498773d4d06df2b22a022a96e542a292ff5c9f01f9a07", + "pcr1": "0x0343b056cd8485ca7890ddd833476d78460aed2aa161548e4e26bedf321726696257d623e8805f3f605946b3d8b0c6aa", + "pcr2": "0x44eb23fd7c4c12048b27a4fa3ed157c366603c5e1298f7aaeb184c4b4438040a283a5a112282c6439552faffbd56ec7c" + }, { "verifier": "0x52AE16Ec4FBf14c0Ff9B06229C1cd2403b4A22d0", "pcr0": "0xb94235a87364f955e068bc0493bf807c8c4bfa752893245a9117c90b444032187d6ae78b8eac89a527fd7b39878bc6f4", diff --git a/proofs/metrics/src/lib.rs b/proofs/metrics/src/lib.rs index d4f995093..6b9231b46 100644 --- a/proofs/metrics/src/lib.rs +++ b/proofs/metrics/src/lib.rs @@ -38,6 +38,10 @@ pub const METRICS_PROOF_JOBS_CLAIMED: &str = "proof_jobs.claimed"; pub const METRICS_PROOF_JOBS_COMPLETED: &str = "proof_jobs.completed"; /// End-to-end worker proof-job attempt duration. pub const METRICS_PROOF_JOB_DURATION_SECONDS: &str = "proof_job.duration_seconds"; +/// Whether this worker's enclave signing key is registered on-chain. +pub const METRICS_ENCLAVE_KEY_REGISTERED: &str = "enclave_key.registered"; +/// Enclave key registration attempts, by outcome. +pub const METRICS_ENCLAVE_REGISTRATION_ATTEMPTS: &str = "enclave_key.registration_attempts"; /// Registers shared metric descriptions. pub fn describe_metrics() { @@ -86,6 +90,31 @@ pub fn describe_metrics() { metrics::Unit::Seconds, "End-to-end worker proof-job attempt duration by backend and outcome." ); + metrics::describe_gauge!( + METRICS_ENCLAVE_KEY_REGISTERED, + metrics::Unit::Count, + "1 when this worker's enclave signing key is registered on-chain, 0 otherwise. \ + A worker holding 0 leases no proof jobs, because proofs signed by an unregistered \ + key do not verify." + ); + metrics::describe_counter!( + METRICS_ENCLAVE_REGISTRATION_ATTEMPTS, + metrics::Unit::Count, + "Enclave key registration attempts by outcome (registered, already_registered, failed)." + ); +} + +/// Sets the enclave-key registration gauge. +/// +/// Emitted eagerly at `0` on startup so the "never registered" case is a visible zero rather +/// than an absent series that a threshold monitor would silently ignore. +pub fn set_enclave_key_registered(registered: bool) { + metrics::gauge!(METRICS_ENCLAVE_KEY_REGISTERED).set(if registered { 1.0 } else { 0.0 }); +} + +/// Records an enclave key registration attempt and its outcome. +pub fn increment_enclave_registration_attempts(outcome: &'static str) { + metrics::counter!(METRICS_ENCLAVE_REGISTRATION_ATTEMPTS, "outcome" => outcome).increment(1); } /// Updates the latest finalized L2 block gauge. diff --git a/proofs/nitro/src/attestation.rs b/proofs/nitro/src/attestation.rs index 819fd408c..0642a11e4 100644 --- a/proofs/nitro/src/attestation.rs +++ b/proofs/nitro/src/attestation.rs @@ -506,14 +506,20 @@ pub fn leaf_cert_pubkey_xy(doc: &[u8]) -> Result<[u8; 96], AttestationError> { ) })?; - let verifying_key = extract_p384_key(&cert_der)?; + cert_pubkey_xy(&cert_der) +} + +/// Extracts any certificate's P-384 public key as the uncompressed `x || y` coordinate pair +/// (96 bytes, without the SEC1 `0x04` prefix). +pub fn cert_pubkey_xy(cert_der: &[u8]) -> Result<[u8; 96], AttestationError> { + let verifying_key = extract_p384_key(cert_der)?; let point = verifying_key.to_encoded_point(false); let bytes = point.as_bytes(); // Uncompressed SEC1 encoding is `0x04 || X (48) || Y (48)` = 97 bytes. if bytes.len() != 97 || bytes[0] != 0x04 { return Err(AttestationError::CertChain(format!( - "unexpected leaf public key encoding ({} bytes, prefix 0x{:02x})", + "unexpected public key encoding ({} bytes, prefix 0x{:02x})", bytes.len(), bytes.first().copied().unwrap_or(0) ))); diff --git a/proofs/nitro/src/lib.rs b/proofs/nitro/src/lib.rs index 0db9c2498..8d6e599c2 100644 --- a/proofs/nitro/src/lib.rs +++ b/proofs/nitro/src/lib.rs @@ -52,6 +52,10 @@ pub mod cose; /// See [`p384_hints::collect_hints`] for the primary entry point. pub mod p384_hints; +/// Pre-warming the on-chain `CertManager` cert cache so `registerKey` can verify the +/// attestation's certificate bundle. See [`prewarm::build_prewarm_plan`]. +pub mod prewarm; + /// On-chain enclave key registration (calldata builder + self-registration flow). #[cfg(all(feature = "enclave", target_os = "linux"))] pub mod register; diff --git a/proofs/nitro/src/prewarm.rs b/proofs/nitro/src/prewarm.rs new file mode 100644 index 000000000..7de116bef --- /dev/null +++ b/proofs/nitro/src/prewarm.rs @@ -0,0 +1,329 @@ +//! Pre-warming the on-chain [`CertManager`] certificate cache. +//! +//! `NitroValidator.validateAttestationWithHints` — the function behind +//! `NitroEnclaveKeyRegistry.registerKey` — re-walks the attestation's certificate bundle via +//! `verifyCachedCertBundle`, passing **empty** hint streams. That only succeeds on certificates +//! already present in the `CertManager` cache; an uncached certificate falls through to +//! signature verification against an empty hint stream and reverts with +//! `"inverse hint underflow"`, even when the attestation's own signature hints are valid. +//! +//! AWS rotates the enclave's leaf certificate roughly every three hours, so the cache goes cold +//! on its own. Rather than requiring an operator to run a pre-warm step inside that window, this +//! module lets the worker pre-warm its *own* chain: [`build_prewarm_plan`] turns an attestation +//! document into the ordered list of `verifyCACertWithHints` / `verifyClientCertWithHints` calls +//! needed to make `registerKey` succeed. +//! +//! The pinned AWS root CA is written into the cache by the `CertManager` constructor, so it is +//! never part of a plan — it only seeds the parent hash for the next certificate in the chain. + +use alloy_primitives::{B256, b256, keccak256}; +use anyhow::{Context, Result, anyhow, bail}; +use sha2::{Digest, Sha384}; + +use crate::{attestation::cert_pubkey_xy, p384_hints::collect_hints}; + +/// `keccak256` of the pinned AWS Nitro root CA certificate. +/// +/// Mirrors `CertManager.ROOT_CA_CERT_HASH`. The root is pre-cached in the `CertManager` +/// constructor and is keyed by this constant rather than by its TBS hash. +pub const ROOT_CA_CERT_HASH: B256 = + b256!("311d96fcd5c5e0ccf72ef548e2ea7d4c0cd53ad7c4cc49e67471aed41d61f185"); + +/// A certificate that must be verified into the `CertManager` cache before `registerKey` +/// can succeed, together with everything needed to submit it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ColdCert { + /// DER-encoded certificate. + pub cert: Vec, + /// Cache key of the parent certificate (`ROOT_CA_CERT_HASH` for the root's children). + pub parent_hash: B256, + /// This certificate's own cache key, used to check whether it is already cached. + pub cache_key: B256, + /// Off-chain modular-inverse hints for this certificate's P-384 signature. + pub hints: Vec, + /// `true` for CA certificates (`verifyCACertWithHints`), `false` for the end-entity leaf + /// (`verifyClientCertWithHints`). + pub is_ca: bool, +} + +/// Computes the `CertManager` cache key for a DER-encoded certificate. +/// +/// Mirrors `CertManager._certCacheKey`: the pinned root is keyed by [`ROOT_CA_CERT_HASH`]; +/// every other certificate is keyed by `keccak256` over its TBSCertificate element, header +/// included. Keying on the TBS rather than the whole certificate makes the key invariant to +/// ECDSA signature malleability. +/// +/// # Errors +/// +/// Returns an error if `der` is not a parseable X.509 certificate. +pub fn cert_cache_key(der: &[u8]) -> Result { + use x509_parser::prelude::{FromDer as _, X509Certificate}; + + if keccak256(der) == ROOT_CA_CERT_HASH { + return Ok(ROOT_CA_CERT_HASH); + } + + let (_, cert) = + X509Certificate::from_der(der).map_err(|e| anyhow!("X.509 parse error: {e:?}"))?; + Ok(keccak256(cert.tbs_certificate.as_ref())) +} + +/// Parses a DER-encoded X.509 certificate into `(sha384(tbs), r || s)`. +/// +/// The certificate signature covers the DER encoding of the TBSCertificate element (header +/// included); the signature itself is a DER `SEQUENCE { INTEGER r, INTEGER s }` which is +/// decoded here into the raw 96-byte `r || s` form the hint generator expects. +/// +/// # Errors +/// +/// Returns an error if the certificate or its signature cannot be parsed, or if either +/// signature component exceeds 384 bits. +pub fn parse_cert_signature(der: &[u8]) -> Result<(Vec, Vec)> { + use x509_parser::prelude::{FromDer as _, X509Certificate}; + + let (_, cert) = + X509Certificate::from_der(der).map_err(|e| anyhow!("X.509 parse error: {e:?}"))?; + + let hash = Sha384::digest(cert.tbs_certificate.as_ref()).to_vec(); + let sig = decode_ecdsa_der_sig(cert.signature_value.data.as_ref()) + .context("decoding certificate ECDSA signature")?; + + Ok((hash, sig)) +} + +/// Builds the ordered set of certificates that must be cached before `registerKey` succeeds. +/// +/// The returned entries are in dependency order: each one's `parent_hash` refers to a +/// certificate that either is already cached or appears earlier in the list. Callers should +/// skip entries whose `cache_key` is already present in the cache. +/// +/// # Errors +/// +/// Returns an error if the attestation document is malformed, carries no leaf certificate, or +/// if its `cabundle` does not begin with the pinned AWS root CA (AWS orders `cabundle` +/// root-first, and the on-chain walk depends on that ordering). +pub fn build_prewarm_plan(attestation_doc: &[u8]) -> Result> { + let parsed = crate::attestation::parse_attestation_doc(attestation_doc) + .map_err(|e| anyhow!("parsing attestation document: {e}"))?; + + let leaf = parsed.certificate.ok_or_else(|| { + anyhow!("attestation document missing required `certificate` field (no leaf cert)") + })?; + if parsed.cabundle.is_empty() { + bail!("attestation document has an empty `cabundle`; cannot build a pre-warm plan"); + } + + let mut plan = Vec::with_capacity(parsed.cabundle.len()); + let mut parent_hash = B256::ZERO; + // Public key of the most recently walked certificate, used to generate the next one's + // signature hints. `None` until the pinned root has been seen. + let mut parent_pubkey: Option<[u8; 96]> = None; + + for (i, cert) in parsed.cabundle.iter().enumerate() { + let cache_key = cert_cache_key(cert) + .with_context(|| format!("computing cache key for cabundle[{i}]"))?; + + // The root is pinned by the CertManager constructor: it is always cached, needs no + // hints, and costs no transaction. It only seeds the parent for the next certificate. + if cache_key == ROOT_CA_CERT_HASH { + parent_hash = cache_key; + parent_pubkey = Some( + cert_pubkey_xy(cert).map_err(|e| anyhow!("extracting root CA public key: {e}"))?, + ); + continue; + } + + let pubkey = parent_pubkey.ok_or_else(|| { + anyhow!( + "cabundle[{i}] has no verified parent — the pinned AWS root CA must be \ + cabundle[0], but it was not found there" + ) + })?; + + let (tbs_hash, sig) = parse_cert_signature(cert) + .with_context(|| format!("parsing cabundle[{i}] signature"))?; + let hints = collect_hints(&tbs_hash, &sig, &pubkey) + .with_context(|| format!("generating P-384 hints for cabundle[{i}]"))?; + + plan.push(ColdCert { + cert: cert.clone(), + parent_hash, + cache_key, + hints, + is_ca: true, + }); + + parent_hash = cache_key; + parent_pubkey = Some( + cert_pubkey_xy(cert) + .map_err(|e| anyhow!("extracting cabundle[{i}] public key: {e}"))?, + ); + } + + let pubkey = parent_pubkey.ok_or_else(|| { + anyhow!("cabundle does not contain the pinned AWS root CA; cannot verify the leaf") + })?; + let (tbs_hash, sig) = parse_cert_signature(&leaf).context("parsing leaf signature")?; + let hints = collect_hints(&tbs_hash, &sig, &pubkey) + .context("generating P-384 hints for the leaf certificate")?; + + plan.push(ColdCert { + cache_key: cert_cache_key(&leaf).context("computing cache key for the leaf cert")?, + cert: leaf, + parent_hash, + hints, + is_ca: false, + }); + + Ok(plan) +} + +/// Decodes the `notAfter` field out of a packed `CertManager.VerifiedCert` record. +/// +/// The contract stores records as `abi.encodePacked(ca, notAfter, maxPathLen, subjectHash, +/// pubKey)`, so `notAfter` is the big-endian `uint64` at bytes `1..9`. Returns `None` for an +/// empty (uncached) record or a record too short to carry the field. +pub fn packed_cert_not_after(packed: &[u8]) -> Option { + let bytes: [u8; 8] = packed.get(1..9)?.try_into().ok()?; + Some(u64::from_be_bytes(bytes)) +} + +// ─── DER helpers ───────────────────────────────────────────────────────────── + +/// Decodes a DER `SEQUENCE { INTEGER r, INTEGER s }` into raw `r || s`, each left-padded to +/// 48 bytes. +fn decode_ecdsa_der_sig(der: &[u8]) -> Result> { + if der.first() != Some(&0x30) { + bail!( + "expected SEQUENCE tag 0x30, got 0x{:02x}", + der.first().copied().unwrap_or(0) + ); + } + let mut pos = 1; + let (seq_len, consumed) = decode_der_length(&der[pos..])?; + pos += consumed; + let end = pos + .checked_add(seq_len) + .filter(|end| *end <= der.len()) + .ok_or_else(|| anyhow!("DER SEQUENCE length overflows input"))?; + + let (r_bytes, advanced) = decode_der_integer(&der[pos..end])?; + pos += advanced; + let (s_bytes, _) = decode_der_integer(&der[pos..end])?; + + let mut out = Vec::with_capacity(96); + out.extend_from_slice(&pad_to_48(&r_bytes)?); + out.extend_from_slice(&pad_to_48(&s_bytes)?); + Ok(out) +} + +fn decode_der_length(data: &[u8]) -> Result<(usize, usize)> { + let first = *data + .first() + .ok_or_else(|| anyhow!("unexpected end of DER length"))?; + if first < 0x80 { + return Ok((first as usize, 1)); + } + let num_bytes = (first & 0x7f) as usize; + if num_bytes == 0 || num_bytes > 4 || data.len() < 1 + num_bytes { + bail!("unsupported DER length encoding"); + } + let mut len = 0usize; + for &b in &data[1..=num_bytes] { + len = (len << 8) | b as usize; + } + Ok((len, 1 + num_bytes)) +} + +fn decode_der_integer(data: &[u8]) -> Result<(Vec, usize)> { + if data.first() != Some(&0x02) { + bail!( + "expected INTEGER tag 0x02, got 0x{:02x}", + data.first().copied().unwrap_or(0) + ); + } + let (len, header) = decode_der_length(&data[1..])?; + let start = 1 + header; + let bytes = data + .get(start..start + len) + .ok_or_else(|| anyhow!("DER INTEGER length overflows input"))?; + // DER prefixes a zero byte to keep the sign bit clear on positive integers. + let stripped = match bytes.first() { + Some(0x00) => &bytes[1..], + _ => bytes, + }; + Ok((stripped.to_vec(), start + len)) +} + +fn pad_to_48(bytes: &[u8]) -> Result> { + if bytes.len() > 48 { + bail!("integer exceeds 384 bits ({} bytes)", bytes.len()); + } + let mut out = vec![0u8; 48 - bytes.len()]; + out.extend_from_slice(bytes); + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The pinned root's cache key is the constant, not its TBS hash — matching + /// `CertManager._certCacheKey`'s short-circuit. + #[test] + fn root_ca_keys_to_the_pinned_constant() { + let root = include_bytes!("testdata/aws_root.der"); + assert_eq!(keccak256(root), ROOT_CA_CERT_HASH); + assert_eq!(cert_cache_key(root).unwrap(), ROOT_CA_CERT_HASH); + } + + /// Non-root certificates key on the TBSCertificate element, so the key differs from a + /// plain hash of the whole DER. + #[test] + fn non_root_keys_on_tbs_not_whole_cert() { + let cert = include_bytes!("testdata/aws_zonal.der"); + let key = cert_cache_key(cert).unwrap(); + assert_ne!(key, keccak256(cert)); + assert_ne!(key, ROOT_CA_CERT_HASH); + } + + #[test] + fn cert_signature_decodes_to_96_bytes() { + let cert = include_bytes!("testdata/aws_zonal.der"); + let (hash, sig) = parse_cert_signature(cert).unwrap(); + assert_eq!(hash.len(), 48, "SHA-384 digest"); + assert_eq!(sig.len(), 96, "r || s"); + } + + #[test] + fn rejects_malformed_certificate() { + assert!(cert_cache_key(&[0u8; 8]).is_err()); + assert!(parse_cert_signature(&[0u8; 8]).is_err()); + } + + #[test] + fn packed_not_after_reads_the_uint64_at_offset_one() { + // ca (1 byte) || notAfter (8 bytes) || rest + let mut packed = vec![0x01]; + packed.extend_from_slice(&1_787_419_975u64.to_be_bytes()); + packed.extend_from_slice(&[0u8; 40]); + assert_eq!(packed_cert_not_after(&packed), Some(1_787_419_975)); + + assert_eq!(packed_cert_not_after(&[]), None, "uncached record"); + assert_eq!(packed_cert_not_after(&[0x01, 0x00]), None, "truncated"); + } + + #[test] + fn der_integer_strips_the_sign_padding_byte() { + // INTEGER 0x00FF → 0xFF once the sign byte is stripped. + let (bytes, consumed) = decode_der_integer(&[0x02, 0x02, 0x00, 0xFF]).unwrap(); + assert_eq!(bytes, vec![0xFF]); + assert_eq!(consumed, 4); + } + + #[test] + fn der_length_rejects_overflowing_sequence() { + // SEQUENCE claiming 0x7F bytes of content but carrying none. + assert!(decode_ecdsa_der_sig(&[0x30, 0x7F]).is_err()); + } +} diff --git a/proofs/nitro/src/register.rs b/proofs/nitro/src/register.rs index 75e9f81ef..d90c3ff2f 100644 --- a/proofs/nitro/src/register.rs +++ b/proofs/nitro/src/register.rs @@ -21,7 +21,7 @@ use std::time::Duration; use alloy_network::EthereumWallet; use alloy_primitives::{Address, Bytes, TxHash, keccak256}; -use alloy_provider::ProviderBuilder; +use alloy_provider::{Provider, ProviderBuilder}; use alloy_signer_local::PrivateKeySigner; use alloy_sol_types::sol; use anyhow::{Context, Result, anyhow, bail}; @@ -29,6 +29,8 @@ use sha2::{Digest, Sha384}; use tracing::{info, warn}; use url::Url; +use crate::prewarm::{ColdCert, build_prewarm_plan, packed_cert_not_after}; + /// Max attempts for the `registerKey` submission. Retries let the flow survive transient /// RPC errors and funding-account nonce contention when several worker replicas share the /// same `REGISTER_PRIVATE_KEY`/`PRIVATE_KEY` and self-register simultaneously. @@ -134,9 +136,167 @@ sol! { external returns (address signer, bytes32 pcr0, bytes32 pcr1, bytes32 pcr2); function isSignerRegistered(address signer) external view returns (bool); + /// `NitroAttestationVerifier`, which is itself a `NitroValidator` and so exposes + /// `certManager()`. Used to discover the CertManager without extra configuration. + function verifier() external view returns (address); + } + + /// `NitroValidator`'s view of its CertManager. `NitroAttestationVerifier is NitroValidator`, + /// so this is callable on the address returned by `NitroEnclaveKeyRegistry.verifier()`. + #[sol(rpc)] + interface INitroValidator { + function certManager() external view returns (address); + } + + /// The subset of `CertManager` needed to pre-warm the attestation's certificate bundle. + #[sol(rpc)] + interface ICertManager { + /// Raw packed `VerifiedCert` record, or empty bytes when the cert is not cached. + function verified(bytes32 certHash) external view returns (bytes); + function verifyCACertWithHints(bytes cert, bytes32 parentCertHash, bytes signatureHints) + external + returns (bytes32); + function verifyClientCertWithHints(bytes cert, bytes32 parentCertHash, bytes signatureHints) + external; } } +/// Ensures every certificate in `plan` is present in the on-chain `CertManager` cache, +/// submitting a verification transaction for each one that is not. +/// +/// `registerKey` re-walks the attestation's certificate bundle with **empty** hints, so an +/// uncached certificate makes it revert with `"inverse hint underflow"` regardless of how good +/// the attestation's own hints are. AWS rotates the enclave leaf certificate roughly every three +/// hours, so this runs on every registration rather than as a one-off deploy step. +/// +/// Entries are submitted in order because each one's parent must be cached first. A certificate +/// cached by a peer replica between the check and the submit is treated as success, not an error. +/// +/// # Errors +/// +/// Returns an error if a certificate is cached but already expired (a new attestation is needed; +/// resubmitting cannot help), or if a verification transaction fails and the certificate is +/// still not cached afterwards. +async fn prewarm_cert_bundle( + provider: P, + cert_manager_address: Address, + plan: &[ColdCert], + now_secs: u64, +) -> Result { + let cert_manager = ICertManager::new(cert_manager_address, provider); + let mut submitted = 0usize; + + for (i, entry) in plan.iter().enumerate() { + let cached = cert_manager + .verified(entry.cache_key) + .call() + .await + .with_context(|| format!("checking CertManager cache for chain[{i}]"))?; + + if !cached.is_empty() { + // A cached-but-expired cert cannot be re-verified: `_verifyCert` short-circuits on + // the cache and reverts with "cert expired". Only a fresh attestation fixes it, so + // say so explicitly rather than letting registerKey fail opaquely later. + match packed_cert_not_after(&cached) { + Some(not_after) if not_after <= now_secs => bail!( + "chain[{i}] (cache key {}) is cached but expired at {not_after} (now {now_secs}); \ + the enclave needs a fresh attestation before it can register", + entry.cache_key + ), + _ => {} + } + continue; + } + + info!( + target: "world_chain::nitro", + cert_manager = %cert_manager_address, + cache_key = %entry.cache_key, + parent = %entry.parent_hash, + is_ca = entry.is_ca, + hint_bytes = entry.hints.len(), + "pre-warming CertManager with uncached certificate" + ); + + let cert = Bytes::from(entry.cert.clone()); + let hints = Bytes::from(entry.hints.clone()); + let sent = if entry.is_ca { + cert_manager + .verifyCACertWithHints(cert, entry.parent_hash, hints) + .send() + .await + } else { + cert_manager + .verifyClientCertWithHints(cert, entry.parent_hash, hints) + .send() + .await + }; + + let outcome = match sent { + Ok(pending) => { + let tx_hash = *pending.tx_hash(); + pending + .get_receipt() + .await + .map(|receipt| (tx_hash, receipt.status())) + .map_err(|err| anyhow!("awaiting receipt for tx {tx_hash}: {err}")) + } + Err(err) => Err(anyhow!("submitting cert verification: {err}")), + }; + + match outcome { + Ok((tx_hash, true)) => { + submitted += 1; + info!( + target: "world_chain::nitro", + %tx_hash, + cache_key = %entry.cache_key, + "certificate verified into the CertManager cache" + ); + } + // A revert (or a send failure) is only fatal if the cert is still uncached — a peer + // replica pre-warming the same chain concurrently is a benign race. + Ok((tx_hash, false)) => { + let now_cached = cert_manager + .verified(entry.cache_key) + .call() + .await + .map(|v| !v.is_empty()) + .unwrap_or(false); + if !now_cached { + bail!( + "cert verification tx {tx_hash} reverted and chain[{i}] is still uncached" + ); + } + warn!( + target: "world_chain::nitro", + %tx_hash, + "cert verification reverted but the cert is now cached; treating as success" + ); + } + Err(err) => { + let now_cached = cert_manager + .verified(entry.cache_key) + .call() + .await + .map(|v| !v.is_empty()) + .unwrap_or(false); + if !now_cached { + return Err(err) + .with_context(|| format!("pre-warming CertManager for chain[{i}] failed")); + } + warn!( + target: "world_chain::nitro", + error = %err, + "cert verification failed but the cert is now cached; treating as success" + ); + } + } + } + + Ok(submitted) +} + /// Inputs for [`register_enclave_key`]. #[derive(Clone, Debug)] pub struct RegisterParams { @@ -175,12 +335,16 @@ pub enum RegistrationOutcome { /// registration wins the race and the tx reverts with `SignerAlreadyRegistered`) this returns /// [`RegistrationOutcome::AlreadyRegistered`] instead of erroring. /// +/// The attestation's certificate chain is pre-warmed into the on-chain `CertManager` first +/// (see [`prewarm_cert_bundle`]), so registration recovers on its own after AWS rotates the +/// enclave's leaf certificate — no operator-run pre-warm step is required. +/// /// # Errors /// -/// Returns an error if the enclave is unreachable, the RPC/key are invalid, the -/// `registerKey` transaction reverts for a reason other than `SignerAlreadyRegistered` -/// (e.g. PCR set not approved, CertManager not pre-warmed), or the key is still not -/// registered after a confirmed transaction. +/// Returns an error if the enclave is unreachable, the RPC/key are invalid, the certificate +/// chain cannot be pre-warmed, the `registerKey` transaction reverts for a reason other than +/// `SignerAlreadyRegistered` (e.g. PCR set not approved), or the key is still not registered +/// after a confirmed transaction. pub async fn register_enclave_key(params: RegisterParams) -> Result { // 0. Validate the string inputs up front with clear messages. `.parse()` below // would also reject these, but an explicit empty-string check gives operators a @@ -219,9 +383,42 @@ pub async fn register_enclave_key(params: RegisterParams) -> Result ExponentialBuilder { + ExponentialBuilder::default() + .with_min_delay(REGISTER_RETRY_INITIAL_DELAY) + .with_max_delay(REGISTER_RETRY_MAX_DELAY) + .with_jitter() + .without_max_times() +} + +/// Registers the enclave key on-chain, retrying until it succeeds or shutdown is requested. +/// Returns `false` only when shutdown won. +/// +/// This deliberately never aborts the process. Every way registration can fail — PCR set not +/// yet approved on-chain, registration key unfunded, L1 unreachable, certificate chain not yet +/// verifiable — is a condition an operator resolves *while the worker is running*. +async fn register_with_retry(params: RegisterParams) -> bool { + let attempt = || { + let params = params.clone(); + async move { register_enclave_key(params).await } + }; + + let registration = attempt + .retry(registration_backoff()) + .notify(|error, delay| { + world_chain_proof_metrics::increment_enclave_registration_attempts("failed"); + error!( + ?error, + retry_in_secs = delay.as_secs(), + "enclave key registration failed; worker stays up and will retry without \ + leasing proof jobs" + ); + }); + + // Race the (unbounded) retry against shutdown so a pod being rolled does not have to wait + // out a full backoff interval. + tokio::select! { + outcome = registration => match outcome { + Ok(outcome) => { + let label = match outcome { + RegistrationOutcome::AlreadyRegistered => { + info!("enclave key already registered on-chain"); + "already_registered" + } + RegistrationOutcome::Registered { tx_hash } => { + info!(%tx_hash, "enclave key registered on-chain"); + "registered" + } + }; + world_chain_proof_metrics::increment_enclave_registration_attempts(label); + world_chain_proof_metrics::set_enclave_key_registered(true); + true + } + // Unreachable while the backoff is unbounded, but treat it the same as shutdown + // rather than leasing jobs with an unregistered key. + Err(error) => { + error!(?error, "enclave key registration gave up; not leasing proof jobs"); + false + } + }, + _ = tokio::signal::ctrl_c() => { + info!("received ctrl-c while retrying registration, shutting down"); + false + } + } +} + #[derive(Debug, Clone, Copy, clap::ValueEnum)] enum Network { #[value(name = "worldchain")] @@ -197,8 +273,10 @@ pub async fn run(args: WorkerArgs) -> Result<()> { args.pcr2.as_deref(), )?; - // Optionally self-register the enclave's generated signing key on-chain before leasing - // any jobs. Without a registered key the proofs this worker submits would not verify. + // Self-register the enclave's generated signing key on-chain before leasing any jobs: + // proofs signed by an unregistered key do not verify, so an unregistered worker must not + // take work. Registration is a *precondition*, not a fatal error — see + // [`register_with_retry`] for why this never aborts the process. if args.auto_register { let registry = args .registry @@ -215,12 +293,16 @@ pub async fn run(args: WorkerArgs) -> Result<()> { REGISTER_PRIVATE_KEY, or PRIVATE_KEY", )?; + // Publish the gauge before the first attempt so "never registered" is a visible zero + // rather than an absent series a threshold monitor would silently ignore. + world_chain_proof_metrics::set_enclave_key_registered(false); info!( registry = %registry, enclave_cid = args.enclave_cid, - "auto-register enabled; registering enclave key on-chain before starting" + "auto-register enabled; registering enclave key on-chain before leasing jobs" ); - let outcome = register_enclave_key(RegisterParams { + + let registered = register_with_retry(RegisterParams { enclave_cid: args.enclave_cid, enclave_port: args.enclave_port, expected_pcrs, @@ -228,15 +310,10 @@ pub async fn run(args: WorkerArgs) -> Result<()> { registry, private_key, }) - .await - .context("auto-registration failed")?; - match outcome { - RegistrationOutcome::AlreadyRegistered => { - info!("enclave key already registered on-chain"); - } - RegistrationOutcome::Registered { tx_hash } => { - info!(%tx_hash, "enclave key registered on-chain"); - } + .await; + if !registered { + // Shutdown was requested while retrying; exit cleanly rather than starting up. + return Ok(()); } } @@ -296,3 +373,39 @@ pub async fn run(args: WorkerArgs) -> Result<()> { worker.await; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use backon::BackoffBuilder; + + /// The registration backoff must be unbounded: the whole point of this change is that a + /// worker keeps retrying (and stays exec-able) instead of crashlooping. A `with_max_times` + /// creeping in here would silently restore the give-up behaviour. + #[test] + fn registration_backoff_never_gives_up() { + let mut backoff = registration_backoff().build(); + // Far more attempts than any bounded policy would allow. + for i in 0..10_000 { + assert!( + backoff.next().is_some(), + "backoff stopped yielding delays at attempt {i}" + ); + } + } + + /// Delays must stay within the configured ceiling so a stuck worker retries on a + /// predictable cadence rather than backing off unboundedly. + #[test] + fn registration_backoff_respects_the_delay_ceiling() { + let mut backoff = registration_backoff().build(); + for _ in 0..256 { + let delay = backoff.next().expect("unbounded backoff yields a delay"); + // `with_jitter` only ever adds to the base delay, bounded by the base itself. + assert!( + delay <= REGISTER_RETRY_MAX_DELAY * 2, + "delay {delay:?} exceeded the jittered ceiling" + ); + } + } +}