feat: implement multi-RPC failover for source chains

Sequential failover across multiple RPC endpoints with exponential
backoff. Replaces single SOURCE_RPC_URL with comma-delimited
SOURCE_RPC_URLS. Retries indefinitely on network failures with
graceful shutdown support.

Author: rube-de

paymaster-relayer/.env.example

@@ -14,19 +14,22 @@ ROFL_ADAPTER_ADDRESS=0xYourROFLAdapterOnSapphire
 # ===========================================
 # BASE SEPOLIA (Chain ID: 84532) -> relayer-base
 # ===========================================
-BASE_SOURCE_RPC_URL=https://base-sepolia.g.alchemy.com/v2/YOUR_KEY
+BASE_SOURCE_RPC_URLS=https://base-sepolia.g.alchemy.com/v2/YOUR_KEY
+# Add failover RPCs with commas: https://url1,https://url2,https://url3
 BASE_VAULT_ADDRESS=0xYourPaymasterVaultOnBase
 
 # ===========================================
 # ETHEREUM SEPOLIA (Chain ID: 11155111) -> relayer-eth
 # ===========================================
-ETH_SOURCE_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/YOUR_KEY
+ETH_SOURCE_RPC_URLS=https://eth-sepolia.g.alchemy.com/v2/YOUR_KEY
+# Add failover RPCs with commas: https://url1,https://url2,https://url3
 ETH_VAULT_ADDRESS=0xYourPaymasterVaultOnEthereum
 
 # ===========================================
 # ARBITRUM SEPOLIA (Chain ID: 421614) -> relayer-arb
 # ===========================================
-ARB_SOURCE_RPC_URL=https://arb-sepolia.g.alchemy.com/v2/YOUR_KEY
+ARB_SOURCE_RPC_URLS=https://arb-sepolia.g.alchemy.com/v2/YOUR_KEY
+# Add failover RPCs with commas: https://url1,https://url2,https://url3
 ARB_VAULT_ADDRESS=0xYourPaymasterVaultOnArbitrum
 
 # ===========================================

paymaster-relayer/README.md

@@ -42,7 +42,7 @@ ROFL deployment in one go.
 
 | Variable | Required | Description |
 |----------|----------|-------------|
-| `SOURCE_RPC_URL` | Yes | Source chain RPC endpoint (HTTP) |
+| `SOURCE_RPC_URLS` | Yes | Source chain RPC endpoints (comma-separated for failover) |
 | `PAYMASTER_VAULT_ADDRESS` | Yes | PaymasterVault contract address on source chain |
 | `TARGET_RPC_URL` | Yes | Sapphire RPC endpoint |
 | `PAYMASTER_PROXY_ADDRESS` | Yes | CrossChainPaymaster proxy address on Sapphire |

paymaster-relayer/compose.local.yaml

@@ -5,7 +5,7 @@ x-relayer-common: &relayer-common
   build:
     context: .
     dockerfile: Dockerfile
-  image: ghcr.io/oasisprotocol/rofl-paymaster-relayer:multichain-test
+  image: ghcr.io/oasisprotocol/rofl-paymaster-relayer:multirpc-test
   platform: linux/amd64
   entrypoint: /bin/sh -c 'uv run python -m paymaster_relayer --local'
   restart: on-failure
@@ -31,7 +31,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: base
-      SOURCE_RPC_URL: ${BASE_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${BASE_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${BASE_VAULT_ADDRESS}
       POLLING_INTERVAL: ${BASE_POLLING_INTERVAL:-6}
 
@@ -43,7 +43,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: ethereum
-      SOURCE_RPC_URL: ${ETH_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ETH_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ETH_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ETH_POLLING_INTERVAL:-12}
 
@@ -55,6 +55,6 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: arbitrum
-      SOURCE_RPC_URL: ${ARB_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ARB_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ARB_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ARB_POLLING_INTERVAL:-3}

paymaster-relayer/compose.yaml

@@ -31,7 +31,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: base
-      SOURCE_RPC_URL: ${BASE_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${BASE_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${BASE_VAULT_ADDRESS}
       POLLING_INTERVAL: ${BASE_POLLING_INTERVAL:-6}
 
@@ -43,7 +43,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: ethereum
-      SOURCE_RPC_URL: ${ETH_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ETH_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ETH_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ETH_POLLING_INTERVAL:-12}
 
@@ -55,6 +55,6 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: arbitrum
-      SOURCE_RPC_URL: ${ARB_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ARB_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ARB_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ARB_POLLING_INTERVAL:-3}

paymaster-relayer/paymaster_relayer/__main__.py

@@ -19,7 +19,9 @@ async def main():
     )
     args = parser.parse_args()
 
-    logger.info(f"=== Paymaster Relayer Starting {'(LOCAL MODE)' if args.local else ''} ===")
+    logger.info(
+        f"=== Paymaster Relayer Starting {'(LOCAL MODE)' if args.local else ''} ==="
+    )
 
     relayer = None
 
@@ -29,11 +31,19 @@ async def main():
     except ValueError as e:
         logger.error(f"Configuration error: {e}")
         logger.error("Required environment variables:")
-        logger.error("  - SOURCE_RPC_URL: Source chain RPC endpoint (e.g., Ethereum)")
+        logger.error(
+            "  - SOURCE_RPC_URLS: Source chain RPC endpoints (comma-separated)"
+        )
         logger.error("  - TARGET_RPC_URL: Target chain RPC endpoint (e.g., Sapphire)")
-        logger.error("  - PAYMASTER_VAULT_ADDRESS: PaymasterVault contract address (source)")
-        logger.error("  - PAYMASTER_PROXY_ADDRESS: CrossChainPaymaster contract address (target)")
-        logger.error("  - ROFL_ADAPTER_ADDRESS: ROFLAdapter contract address (target, HashStored)")
+        logger.error(
+            "  - PAYMASTER_VAULT_ADDRESS: PaymasterVault contract address (source)"
+        )
+        logger.error(
+            "  - PAYMASTER_PROXY_ADDRESS: CrossChainPaymaster contract address (target)"
+        )
+        logger.error(
+            "  - ROFL_ADAPTER_ADDRESS: ROFLAdapter contract address (target, HashStored)"
+        )
         if args.local:
             logger.error("  - PRIVATE_KEY: Private key for signing transactions")
         sys.exit(1)

paymaster-relayer/paymaster_relayer/config.py

@@ -7,45 +7,37 @@
 """
 
 import os
-import re
 from dataclasses import dataclass, field
 
+from .utils.multi_rpc_provider import sanitize_url
 
-def parse_indexed_rpc_urls(prefix: str) -> list[str]:
+
+def parse_rpc_urls() -> list[str]:
     """
-    Parse indexed environment variables like PREFIX_1, PREFIX_2, etc.
+    Parse SOURCE_RPC_URLS env var into a list of RPC endpoint URLs.
 
-    Args:
-        prefix: The base name without the index (e.g., "SOURCE_RPC_URL")
+    Splits on commas, strips whitespace, and filters empty entries.
 
     Returns:
-        List of URLs in ascending index order
+        List of non-empty, trimmed URLs
 
     Raises:
-        ValueError: If no indexed environment variables are found
+        ValueError: If SOURCE_RPC_URLS is missing or contains no valid URLs
     """
-    pattern = re.compile(rf"^{re.escape(prefix)}_(\d+)$")
-    indexed_urls: list[tuple[int, str]] = []
-
-    for key, value in os.environ.items():
-        match = pattern.match(key)
-        if match:
-            index = int(match.group(1))
-            indexed_urls.append((index, value))
+    raw = os.environ.get("SOURCE_RPC_URLS", "")
+    urls = [url.strip() for url in raw.split(",") if url.strip()]
 
-    if not indexed_urls:
-        raise ValueError(f"No {prefix}_N environment variables found")
+    if not urls:
+        raise ValueError("SOURCE_RPC_URLS environment variable is missing or empty")
 
-    # Sort by index and return just the URLs
-    indexed_urls.sort(key=lambda x: x[0])
-    return [url for _, url in indexed_urls]
+    return urls
 
 
 @dataclass(frozen=True, slots=True)
 class SourceChainConfig:
     """Configuration for the source chain (e.g., Base/Sepolia)."""
 
-    rpc_url: str
+    rpc_urls: list[str]
     paymaster_vault_address: str
 
 
@@ -68,7 +60,9 @@ class MonitoringConfig:
     retry_count: int = 3
     lookback_blocks: int = 9
     process_batch_size: int = 10  # max events to process in one batch
-    max_block_range: int = 10  # max blocks per get_logs request (Alchemy free tier limit)
+    max_block_range: int = (
+        10  # max blocks per get_logs request (Alchemy free tier limit)
+    )
 
     def __post_init__(self) -> None:
         """Validate monitoring configuration."""
@@ -100,7 +94,6 @@ def __post_init__(self) -> None:
                 f"Lookback blocks too high (max 1000), got {self.lookback_blocks}"
             )
 
-
         # Validate batch size
         if self.process_batch_size <= 0:
             raise ValueError(
@@ -121,6 +114,7 @@ def __post_init__(self) -> None:
                 f"Max block range too large (max 10000), got {self.max_block_range}"
             )
 
+
 @dataclass(frozen=True, slots=True)
 class RelayerConfig:
     """Main configuration class for the ROFL Relayer."""
@@ -141,13 +135,14 @@ def from_env(cls, local_mode: bool = False) -> "RelayerConfig":
         Raises:
             ValueError: If required environment variables are missing
         """
-        # Source chain configuration
-        source_rpc_url = os.environ.get("SOURCE_RPC_URL")
-        if not source_rpc_url:
+        # Source chain configuration - parse comma-delimited RPC URLs
+        try:
+            source_rpc_urls = parse_rpc_urls()
+        except ValueError:
             raise ValueError(
-                "SOURCE_RPC_URL environment variable is required. "
-                "Example: https://ethereum-sepolia.publicnode.com"
-            )
+                "SOURCE_RPC_URLS environment variable is required (comma-separated). "
+                "Example: SOURCE_RPC_URLS=https://rpc1.example.com,https://rpc2.example.com"
+            ) from None
 
         paymaster_vault_address = os.environ.get("PAYMASTER_VAULT_ADDRESS")
         if not paymaster_vault_address:
@@ -204,7 +199,7 @@ def from_env(cls, local_mode: bool = False) -> "RelayerConfig":
 
         # Create configuration objects
         source_chain = SourceChainConfig(
-            rpc_url=source_rpc_url,
+            rpc_urls=source_rpc_urls,
             paymaster_vault_address=paymaster_vault_address,
         )
 
@@ -228,11 +223,13 @@ def log_config(self) -> None:
         print(f"Mode: {'LOCAL' if self.local_mode else 'ROFL'}")
 
         print("\n[Source Chain]")
-        print(f"  RPC URL: {self.source_chain.rpc_url}")
+        print(f"  RPC URLs ({len(self.source_chain.rpc_urls)} configured):")
+        for i, url in enumerate(self.source_chain.rpc_urls, 1):
+            print(f"    [{i}] {sanitize_url(url)}")
         print(f"  PaymasterVault: {self.source_chain.paymaster_vault_address}")
 
         print("\n[Target Chain]")
-        print(f"  RPC URL: {self.target_chain.rpc_url}")
+        print(f"  RPC URL: {sanitize_url(self.target_chain.rpc_url)}")
         print(f"  CrossChainPaymaster: {self.target_chain.paymaster_address}")
         print(f"  ROFLAdapter: {self.target_chain.rofl_adapter_address}")
         print(

paymaster-relayer/paymaster_relayer/event_processor.py

@@ -182,7 +182,9 @@ async def process_hash_stored(self, event: EventData) -> tuple[int, str] | None:
                 self.stored_hashes.popitem(last=False)
             self.stored_hashes[block_id] = block_hash
 
-            logger.info(f"Hash stored - Chain {domain} Block {block_id}: {block_hash[:10]}...")
+            logger.info(
+                f"Hash stored - Chain {domain} Block {block_id}: {block_hash[:10]}..."
+            )
 
             matching_payments: list[PaymentEvent] = self.pending_payments.get(
                 block_id, []
@@ -264,7 +266,9 @@ async def process_matched_payment(self, payment_event: PaymentEvent) -> bool:
         # This prevents race between HashStored handler and retry task
         if not self._remove_from_pending(payment_event):
             # Already removed by another task - skip to avoid duplicate submission
-            logger.debug(f"Payment {payment_event.tx_hash[:10]}... already being processed")
+            logger.debug(
+                f"Payment {payment_event.tx_hash[:10]}... already being processed"
+            )
             return False
 
         paymaster_address = self.config.target_chain.paymaster_address