Merge pull request #23 from oasisprotocol/rube/19-rpc-can-fail

Rube/19 rpc can fail

Author: rube-de

paymaster-relayer/.env.example

@@ -14,19 +14,22 @@ ROFL_ADAPTER_ADDRESS=0xYourROFLAdapterOnSapphire
 # ===========================================
 # BASE SEPOLIA (Chain ID: 84532) -> relayer-base
 # ===========================================
-BASE_SOURCE_RPC_URL=https://base-sepolia.g.alchemy.com/v2/YOUR_KEY
+BASE_SOURCE_RPC_URLS=https://base-sepolia.g.alchemy.com/v2/YOUR_KEY
+# Add failover RPCs with commas: https://url1,https://url2,https://url3
 BASE_VAULT_ADDRESS=0xYourPaymasterVaultOnBase
 
 # ===========================================
 # ETHEREUM SEPOLIA (Chain ID: 11155111) -> relayer-eth
 # ===========================================
-ETH_SOURCE_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/YOUR_KEY
+ETH_SOURCE_RPC_URLS=https://eth-sepolia.g.alchemy.com/v2/YOUR_KEY
+# Add failover RPCs with commas: https://url1,https://url2,https://url3
 ETH_VAULT_ADDRESS=0xYourPaymasterVaultOnEthereum
 
 # ===========================================
 # ARBITRUM SEPOLIA (Chain ID: 421614) -> relayer-arb
 # ===========================================
-ARB_SOURCE_RPC_URL=https://arb-sepolia.g.alchemy.com/v2/YOUR_KEY
+ARB_SOURCE_RPC_URLS=https://arb-sepolia.g.alchemy.com/v2/YOUR_KEY
+# Add failover RPCs with commas: https://url1,https://url2,https://url3
 ARB_VAULT_ADDRESS=0xYourPaymasterVaultOnArbitrum
 
 # ===========================================

paymaster-relayer/README.md

@@ -42,7 +42,7 @@ ROFL deployment in one go.
 
 | Variable | Required | Description |
 |----------|----------|-------------|
-| `SOURCE_RPC_URL` | Yes | Source chain RPC endpoint (HTTP) |
+| `SOURCE_RPC_URLS` | Yes | Source chain RPC endpoints (comma-separated for failover) |
 | `PAYMASTER_VAULT_ADDRESS` | Yes | PaymasterVault contract address on source chain |
 | `TARGET_RPC_URL` | Yes | Sapphire RPC endpoint |
 | `PAYMASTER_PROXY_ADDRESS` | Yes | CrossChainPaymaster proxy address on Sapphire |

paymaster-relayer/compose.local.yaml

@@ -5,7 +5,7 @@ x-relayer-common: &relayer-common
   build:
     context: .
     dockerfile: Dockerfile
-  image: ghcr.io/oasisprotocol/rofl-paymaster-relayer:multichain-test
+  image: ghcr.io/oasisprotocol/rofl-paymaster-relayer:multirpc-test
   platform: linux/amd64
   entrypoint: /bin/sh -c 'uv run python -m paymaster_relayer --local'
   restart: on-failure
@@ -31,7 +31,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: base
-      SOURCE_RPC_URL: ${BASE_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${BASE_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${BASE_VAULT_ADDRESS}
       POLLING_INTERVAL: ${BASE_POLLING_INTERVAL:-6}
 
@@ -43,7 +43,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: ethereum
-      SOURCE_RPC_URL: ${ETH_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ETH_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ETH_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ETH_POLLING_INTERVAL:-12}
 
@@ -55,6 +55,6 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: arbitrum
-      SOURCE_RPC_URL: ${ARB_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ARB_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ARB_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ARB_POLLING_INTERVAL:-3}

paymaster-relayer/compose.yaml

@@ -31,7 +31,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: base
-      SOURCE_RPC_URL: ${BASE_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${BASE_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${BASE_VAULT_ADDRESS}
       POLLING_INTERVAL: ${BASE_POLLING_INTERVAL:-6}
 
@@ -43,7 +43,7 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: ethereum
-      SOURCE_RPC_URL: ${ETH_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ETH_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ETH_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ETH_POLLING_INTERVAL:-12}
 
@@ -55,6 +55,6 @@ services:
     environment:
       <<: *common-env
       CHAIN_NAME: arbitrum
-      SOURCE_RPC_URL: ${ARB_SOURCE_RPC_URL}
+      SOURCE_RPC_URLS: ${ARB_SOURCE_RPC_URLS}
       PAYMASTER_VAULT_ADDRESS: ${ARB_VAULT_ADDRESS}
       POLLING_INTERVAL: ${ARB_POLLING_INTERVAL:-3}

paymaster-relayer/paymaster_relayer/__main__.py

@@ -19,7 +19,9 @@ async def main():
     )
     args = parser.parse_args()
 
-    logger.info(f"=== Paymaster Relayer Starting {'(LOCAL MODE)' if args.local else ''} ===")
+    logger.info(
+        f"=== Paymaster Relayer Starting {'(LOCAL MODE)' if args.local else ''} ==="
+    )
 
     relayer = None
 
@@ -29,11 +31,19 @@ async def main():
     except ValueError as e:
         logger.error(f"Configuration error: {e}")
         logger.error("Required environment variables:")
-        logger.error("  - SOURCE_RPC_URL: Source chain RPC endpoint (e.g., Ethereum)")
+        logger.error(
+            "  - SOURCE_RPC_URLS: Source chain RPC endpoints (comma-separated)"
+        )
         logger.error("  - TARGET_RPC_URL: Target chain RPC endpoint (e.g., Sapphire)")
-        logger.error("  - PAYMASTER_VAULT_ADDRESS: PaymasterVault contract address (source)")
-        logger.error("  - PAYMASTER_PROXY_ADDRESS: CrossChainPaymaster contract address (target)")
-        logger.error("  - ROFL_ADAPTER_ADDRESS: ROFLAdapter contract address (target, HashStored)")
+        logger.error(
+            "  - PAYMASTER_VAULT_ADDRESS: PaymasterVault contract address (source)"
+        )
+        logger.error(
+            "  - PAYMASTER_PROXY_ADDRESS: CrossChainPaymaster contract address (target)"
+        )
+        logger.error(
+            "  - ROFL_ADAPTER_ADDRESS: ROFLAdapter contract address (target, HashStored)"
+        )
         if args.local:
             logger.error("  - PRIVATE_KEY: Private key for signing transactions")
         sys.exit(1)

paymaster-relayer/paymaster_relayer/config.py

@@ -9,12 +9,35 @@
 import os
 from dataclasses import dataclass, field
 
+from .utils.multi_rpc_provider import sanitize_url
+
+
+def parse_rpc_urls() -> list[str]:
+    """
+    Parse SOURCE_RPC_URLS env var into a list of RPC endpoint URLs.
+
+    Splits on commas, strips whitespace, and filters empty entries.
+
+    Returns:
+        List of non-empty, trimmed URLs
+
+    Raises:
+        ValueError: If SOURCE_RPC_URLS is missing or contains no valid URLs
+    """
+    raw = os.environ.get("SOURCE_RPC_URLS", "")
+    urls = [url.strip() for url in raw.split(",") if url.strip()]
+
+    if not urls:
+        raise ValueError("SOURCE_RPC_URLS environment variable is missing or empty")
+
+    return urls
+
 
 @dataclass(frozen=True, slots=True)
 class SourceChainConfig:
     """Configuration for the source chain (e.g., Base/Sepolia)."""
 
-    rpc_url: str
+    rpc_urls: list[str]
     paymaster_vault_address: str
 
 
@@ -37,7 +60,9 @@ class MonitoringConfig:
     retry_count: int = 3
     lookback_blocks: int = 9
     process_batch_size: int = 10  # max events to process in one batch
-    max_block_range: int = 10  # max blocks per get_logs request (Alchemy free tier limit)
+    max_block_range: int = (
+        10  # max blocks per get_logs request (Alchemy free tier limit)
+    )
 
     def __post_init__(self) -> None:
         """Validate monitoring configuration."""
@@ -69,7 +94,6 @@ def __post_init__(self) -> None:
                 f"Lookback blocks too high (max 1000), got {self.lookback_blocks}"
             )
 
-
         # Validate batch size
         if self.process_batch_size <= 0:
             raise ValueError(
@@ -90,6 +114,7 @@ def __post_init__(self) -> None:
                 f"Max block range too large (max 10000), got {self.max_block_range}"
             )
 
+
 @dataclass(frozen=True, slots=True)
 class RelayerConfig:
     """Main configuration class for the ROFL Relayer."""
@@ -110,13 +135,14 @@ def from_env(cls, local_mode: bool = False) -> "RelayerConfig":
         Raises:
             ValueError: If required environment variables are missing
         """
-        # Source chain configuration
-        source_rpc_url = os.environ.get("SOURCE_RPC_URL")
-        if not source_rpc_url:
+        # Source chain configuration - parse comma-delimited RPC URLs
+        try:
+            source_rpc_urls = parse_rpc_urls()
+        except ValueError:
             raise ValueError(
-                "SOURCE_RPC_URL environment variable is required. "
-                "Example: https://ethereum-sepolia.publicnode.com"
-            )
+                "SOURCE_RPC_URLS environment variable is required (comma-separated). "
+                "Example: SOURCE_RPC_URLS=https://rpc1.example.com,https://rpc2.example.com"
+            ) from None
 
         paymaster_vault_address = os.environ.get("PAYMASTER_VAULT_ADDRESS")
         if not paymaster_vault_address:
@@ -173,7 +199,7 @@ def from_env(cls, local_mode: bool = False) -> "RelayerConfig":
 
         # Create configuration objects
         source_chain = SourceChainConfig(
-            rpc_url=source_rpc_url,
+            rpc_urls=source_rpc_urls,
             paymaster_vault_address=paymaster_vault_address,
         )
 
@@ -197,11 +223,13 @@ def log_config(self) -> None:
         print(f"Mode: {'LOCAL' if self.local_mode else 'ROFL'}")
 
         print("\n[Source Chain]")
-        print(f"  RPC URL: {self.source_chain.rpc_url}")
+        print(f"  RPC URLs ({len(self.source_chain.rpc_urls)} configured):")
+        for i, url in enumerate(self.source_chain.rpc_urls, 1):
+            print(f"    [{i}] {sanitize_url(url)}")
         print(f"  PaymasterVault: {self.source_chain.paymaster_vault_address}")
 
         print("\n[Target Chain]")
-        print(f"  RPC URL: {self.target_chain.rpc_url}")
+        print(f"  RPC URL: {sanitize_url(self.target_chain.rpc_url)}")
         print(f"  CrossChainPaymaster: {self.target_chain.paymaster_address}")
         print(f"  ROFLAdapter: {self.target_chain.rofl_adapter_address}")
         print(

paymaster-relayer/paymaster_relayer/event_processor.py

@@ -182,7 +182,9 @@ async def process_hash_stored(self, event: EventData) -> tuple[int, str] | None:
                 self.stored_hashes.popitem(last=False)
             self.stored_hashes[block_id] = block_hash
 
-            logger.info(f"Hash stored - Chain {domain} Block {block_id}: {block_hash[:10]}...")
+            logger.info(
+                f"Hash stored - Chain {domain} Block {block_id}: {block_hash[:10]}..."
+            )
 
             matching_payments: list[PaymentEvent] = self.pending_payments.get(
                 block_id, []
@@ -264,7 +266,9 @@ async def process_matched_payment(self, payment_event: PaymentEvent) -> bool:
         # This prevents race between HashStored handler and retry task
         if not self._remove_from_pending(payment_event):
             # Already removed by another task - skip to avoid duplicate submission
-            logger.debug(f"Payment {payment_event.tx_hash[:10]}... already being processed")
+            logger.debug(
+                f"Payment {payment_event.tx_hash[:10]}... already being processed"
+            )
             return False
 
         paymaster_address = self.config.target_chain.paymaster_address

paymaster-relayer/paymaster_relayer/proof_manager.py

@@ -20,6 +20,7 @@
 
 if TYPE_CHECKING:
     from .utils.contract_utility import ContractUtility
+    from .utils.multi_rpc_provider import MultiRpcProvider
     from .utils.rofl_utility import ROFLUtility
 
 logger = logging.getLogger(__name__)
@@ -38,19 +39,19 @@ class ProofManager:
 
     def __init__(
         self,
-        w3_source: Web3,
+        source_provider: "MultiRpcProvider",
         contract_util: "ContractUtility",
         rofl_util: "ROFLUtility | None" = None,
     ):
         """
         Initialize the ProofManager.
 
         Args:
-            w3_source: Web3 instance for the source chain
+            source_provider: MultiRpcProvider for source chain with failover
             contract_util: Utility for contract interactions
             rofl_util: ROFL utility for transaction submission (optional)
         """
-        self.w3_source = w3_source
+        self.source_provider = source_provider
         self.contract_util = contract_util
         self.rofl_util = rofl_util
 
@@ -72,7 +73,9 @@ def _get_transaction_local_index(self, payment_event: PaymentEvent) -> int:
         Returns:
             Transaction-local index (position within transaction's logs)
         """
-        receipt = self.w3_source.eth.get_transaction_receipt(HexStr(payment_event.tx_hash))
+        receipt = self.source_provider.execute_with_failover(
+            lambda w3: w3.eth.get_transaction_receipt(HexStr(payment_event.tx_hash))
+        )
         if not receipt or "logs" not in receipt:
             logger.warning(f"No logs found in transaction {payment_event.tx_hash}")
             return 0
@@ -90,7 +93,9 @@ def _get_transaction_local_index(self, payment_event: PaymentEvent) -> int:
                 return i
 
         # If not found (shouldn't happen), default to 0
-        logger.warning("PaymentInitiated not found in transaction logs, defaulting to index 0")
+        logger.warning(
+            "PaymentInitiated not found in transaction logs, defaulting to index 0"
+        )
         return 0
 
     async def generate_proof(self, payment_event: PaymentEvent) -> list[Any]:
@@ -109,18 +114,28 @@ async def generate_proof(self, payment_event: PaymentEvent) -> list[Any]:
             ValueError: If receipt or block not found, or proof generation fails
         """
         # Calculate transaction-local log index from event content
-        log_index = self._get_transaction_local_index(payment_event)
+        log_index = await asyncio.to_thread(
+            self._get_transaction_local_index, payment_event
+        )
         logger.info(
             f"Generating proof for tx {payment_event.tx_hash}, transaction-local log index {log_index}"
         )
 
         # 1. Fetch receipt and block
-        receipt = self.w3_source.eth.get_transaction_receipt(HexStr(payment_event.tx_hash))
+        receipt = await asyncio.to_thread(
+            self.source_provider.execute_with_failover,
+            lambda w3: w3.eth.get_transaction_receipt(HexStr(payment_event.tx_hash)),
+        )
         if not receipt:
-            raise ValueError(f"Transaction receipt not found for {payment_event.tx_hash}")
+            raise ValueError(
+                f"Transaction receipt not found for {payment_event.tx_hash}"
+            )
 
         block_number = receipt["blockNumber"]
-        block = self.w3_source.eth.get_block(block_number, full_transactions=True)
+        block = await asyncio.to_thread(
+            self.source_provider.execute_with_failover,
+            lambda w3: w3.eth.get_block(block_number, full_transactions=True),
+        )
         if not block:
             raise ValueError(f"Block not found for block number {block_number}")
 
@@ -129,7 +144,7 @@ async def generate_proof(self, payment_event: PaymentEvent) -> list[Any]:
         )
 
         # 2. Get all receipts in block
-        receipts = self._get_block_receipts(block_number)
+        receipts = await asyncio.to_thread(self._get_block_receipts, block_number)
 
         logger.info(f"Fetched {len(receipts)} receipts from block")
 
@@ -166,7 +181,10 @@ async def generate_proof(self, payment_event: PaymentEvent) -> list[Any]:
         # 6. Encode block header
         encoded_block_header = BlockchainEncoder.encode_block_header(block)
 
-        chain_id = int(self.w3_source.eth.chain_id)
+        chain_id = await asyncio.to_thread(
+            self.source_provider.execute_with_failover,
+            lambda w3: int(w3.eth.chain_id),
+        )
 
         # 7. Create proof structure for Hashi
         proof = [
@@ -185,7 +203,9 @@ async def generate_proof(self, payment_event: PaymentEvent) -> list[Any]:
         )
         return proof
 
-    async def submit_proof(self, proof: list[Any], paymaster_address: str) -> str | None:
+    async def submit_proof(
+        self, proof: list[Any], paymaster_address: str
+    ) -> str | None:
         """
         Submit proof to CrossChainPaymaster contract.
 
@@ -314,7 +334,9 @@ def _get_block_receipts(self, block_number: int) -> list[TxReceipt]:
             ValueError: If block receipts cannot be fetched
         """
         try:
-            receipts = self.w3_source.eth.get_block_receipts(block_number)
+            receipts = self.source_provider.execute_with_failover(
+                lambda w3: w3.eth.get_block_receipts(block_number)
+            )
         except Exception as e:
             logger.error(f"Failed to fetch receipts for block {block_number}: {e}")
             raise ValueError(