Support Nimbus Verified Proxy

.github/workflows/build-clients.yml

@@ -22,6 +22,12 @@ jobs:
             test_cl: false
             test_el: false
             test_vc: false
+          - env: |-
+              COMPOSE_FILE=nimbus-vp.yml
+              NIMVP_DOCKERFILE=Dockerfile.source
+            test_cl: false
+            test_el: false
+            test_vc: false
           - env: |-
               COMPOSE_FILE=prysm.yml:geth.yml:mev-boost.yml
               MEV_DOCKERFILE=Dockerfile.source

.github/workflows/test-nimbus-proxy.yml

@@ -0,0 +1,47 @@
+name: Test Nimbus Verified Proxy
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, labeled, unlabeled]
+    branches: [main]
+
+jobs:
+  test-nimbus-proxy:
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'test-nimbus-proxy') ||
+      contains(github.event.pull_request.labels.*.name, 'test-all') ||
+      github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Set up Docker buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Create .env file
+        run: cp default.env .env
+      - name: Set Nimbus Verified Proxy
+        run: |
+          source ./.github/helper.sh
+          NETWORK=mainnet
+          var=NETWORK
+          set_value_in_env
+          COMPOSE_FILE=nimbus-vp.yml
+          var=COMPOSE_FILE
+          set_value_in_env
+          RPC_URL=wss://ethereum-rpc.publicnode.com
+          var=RPC_URL
+          set_value_in_env
+          CL_NODE=https://ethereum.operationsolarstorm.org/
+          var=CL_NODE
+          set_value_in_env
+      - name: Start Nimbus Verified Proxy
+        run: ./ethd up
+      - name: Pause for 30 seconds
+        run: sleep 30
+      - name: Test Nimbus Verified Proxy
+        run: ./.github/check-service.sh rpc-proxy

default.env

@@ -83,6 +83,8 @@ EL_HOST=rpc
 EL_LB=rpc-lb
 EL_WS_HOST=ws
 EL_WS_LB=ws-lb
+PROXY_RPC_HOST=rpc
+PROXY_WS_HOST=ws
 CL_HOST=cl
 CL_LB=cl-lb
 VC_HOST=vc
@@ -155,6 +157,10 @@ EL_RPC_PORT=8545
 # Note that for Erigon, this needs to match EL_RPC_PORT *if* you use traefik, and only then
 # Do not change it for Erigon and el-shared.yml
 EL_WS_PORT=8546
+# Ports to use for the verified proxy. They can be the same as the EL ports, depending on whether
+# you are mapping either to host.
+PROXY_RPC_PORT=48545
+PROXY_WS_PORT=48546
 # Erigon's torrent port. Don't make this 42070, as it will fail
 ERIGON_TORRENT_PORT=42069
 # Erigon's third P2P port
@@ -246,6 +252,8 @@ PG_ALIAS=${NETWORK}-postgres
 CL_ALIAS=${NETWORK}-consensus
 EL_ALIAS=${NETWORK}-execution
 MEV_ALIAS=${NETWORK}-mev
+RPC_PROXY_ALIAS=${NETWORK}-rpc-proxy
+WS_PROXY_ALIAS=${NETWORK}-ws-proxy
 # MEV-boost address. This would only be changed for Vouch setups
 MEV_NODE=http://${MEV_ALIAS}:18550
 # Web3signer address - match service name or alias, or it can be remote
@@ -258,6 +266,8 @@ OBOL_CL_NODE=http://${NETWORK}-consensus:${CL_REST_PORT}
 EL_RPC_NODE=http://${NETWORK}-execution:${EL_RPC_PORT}
 # Execution client address (WS) for SSV Anchor
 EL_WS_NODE=ws://${NETWORK}-execution:${EL_WS_PORT}
+# RPC provider when using a verified proxy, use wss:// here
+RPC_URL=wss://eth-${NETWORK}.g.alchemy.com/v2/<ApiKey>
 
 # You can set specific version targets and choose binary or compiled from source builds below,
 # via "Dockerfile.binary" or "Dockerfile.source"
@@ -435,6 +445,13 @@ ETHREX_DOCKER_TAG=latest
 ETHREX_DOCKER_REPO=ghcr.io/lambdaclass/ethrex
 ETHREX_DOCKERFILE=Dockerfile.binary
 
+# Nimbus verified proxy
+# SRC build target can be a tag, a branch, or a pr as "pr-ID"
+NIMVP_SRC_BUILD_TARGET=master
+NIMVP_SRC_REPO=https://github.com/status-im/nimbus-eth1
+NIMVP_DOCKER_TAG=latest
+NIMVP_DOCKER_REPO=statusim/nimbus-verified-proxy
+NIMVP_DOCKERFILE=Dockerfile.source
 
 # staking-deposit-cli
 # SRC build target can be a tag, a branch, or a pr as "pr-ID"
@@ -456,4 +473,4 @@ NODE_EXPORTER_IGNORE_MOUNT_REGEX='^/(dev|proc|sys|run|var/snap/.+|var/lib/docker
 DOCKER_ROOT=/var/lib/docker
 
 # Used by ethd update - please do not adjust
-ENV_VERSION=50
+ENV_VERSION=51

ethd

@@ -5437,6 +5437,14 @@ version() {
       ;;
   esac
 
+  # RPC proxy version
+  case "${__value}" in
+    *nimbus-vp.yml*)
+      __docompose exec rpc-proxy nimbus_verified_proxy --version
+      echo
+      ;;
+  esac
+
   # Grafana version
   case "${__value}" in
     *grafana.yml*)

nimbus-vp.yml

@@ -0,0 +1,51 @@
+x-logging: &logging
+  logging:
+    driver: json-file
+    options:
+      max-size: 100m
+      max-file: "3"
+      tag: '{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}'
+
+services:
+  rpc-proxy:
+    restart: "unless-stopped"
+    build:
+      context: ./nimbus-vp
+      dockerfile: ${NIMVP_DOCKERFILE}
+      args:
+        - BUILD_TARGET=${NIMVP_SRC_BUILD_TARGET:-master}
+        - SRC_REPO=${NIMVP_SRC_REPO:-https://github.com/status-im/nimbus-eth1}
+        - DOCKER_TAG=${NIMVP_DOCKER_TAG:-latest}
+        - DOCKER_REPO=${NIMVP_DOCKER_REPO:-statusim/nimbus-verified-proxy}
+    stop_grace_period: 30s
+    image: nimbus-vp:local
+    pull_policy: never
+    user: user
+    environment:
+      - CL_NODE=${CL_NODE}
+    volumes:
+      - nimbus-vp-data:/var/lib/nimbus
+      - /etc/localtime:/etc/localtime:ro
+    networks:
+      default:
+        aliases:
+          # This allows multiple Eth Docker stacks all connected to the same bridge network
+          - ${RPC_PROXY_ALIAS:-default-rpc-proxy}
+    <<: *logging
+    entrypoint:
+      - docker-entrypoint.sh
+      - nimbus_verified_proxy
+      - --data-dir=/var/lib/nimbus
+      - --network=${NETWORK}
+      - --execution-api-url=${RPC_URL}
+      - --listen-url=http://0.0.0.0:${PROXY_RPC_PORT:-48545}
+      - --listen-url=ws://0.0.0.0:${PROXY_WS_PORT:-48546}
+      - --beacon-api-url=${CL_NODE}
+      - --log-level=${LOG_LEVEL}
+
+volumes:
+  nimbus-vp-data:
+
+networks:
+  default:
+    enable_ipv6: ${IPV6:-false}

nimbus-vp/Dockerfile.binary

@@ -0,0 +1,46 @@
+# hadolint global ignore=DL3007,DL3008,DL3059
+ARG DOCKER_TAG=latest
+ARG DOCKER_REPO=statusim/nimbus-verified-proxy
+
+FROM ${DOCKER_REPO}:${DOCKER_TAG}
+
+# Included here to avoid build-time complaints
+ARG BUILD_TARGET
+ARG SRC_REPO
+
+ARG USER=user
+ARG UID=11001
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y --no-install-recommends \
+  ca-certificates \
+  tzdata \
+  gosu \
+  adduser \
+  bash \
+  curl \
+  jq \
+  && gosu nobody true \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# See https://stackoverflow.com/a/55757473/12429735RUN
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/usr/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    "${USER}"
+
+RUN mkdir -p /var/lib/nimbus && chown -R ${USER}:${USER} /var/lib/nimbus && chmod 700 /var/lib/nimbus
+
+RUN cp /home/user/nimbus-verified-proxy/build/nimbus_verified_proxy /usr/local/bin/ && chown ${USER}:${USER} /usr/local/bin/nimbus_verified_proxy
+# Cannot assume buildkit, hence no chmod
+COPY --chown=${USER}:${USER} ./docker-entrypoint.sh /usr/local/bin/
+# Belt and suspenders
+RUN chmod -R 755 /usr/local/bin/*
+
+USER ${USER}
+
+ENTRYPOINT ["nimbus_verified_proxy"]

nimbus-vp/Dockerfile.source

@@ -0,0 +1,76 @@
+# hadolint global ignore=DL3007,DL3008,DL3059,DL4006
+# Build Nimbus verified proxy in a stock debian container
+FROM debian:trixie-slim AS builder
+
+# Included here to avoid build-time complaints
+ARG DOCKER_TAG
+ARG DOCKER_REPO
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y --no-install-recommends \
+  build-essential \
+  ca-certificates \
+  git
+
+ARG BUILD_TARGET
+ARG SRC_REPO
+
+WORKDIR /usr/src
+
+ARG SRC_DIR=nimbus-eth1
+RUN bash -eo pipefail <<'EOF'
+  git clone "$SRC_REPO" "$SRC_DIR"
+  cd "$SRC_DIR"
+  git config advice.detachedHead false
+  git fetch --all --tags
+  CLEANED=$(echo "$BUILD_TARGET" | sed 's/\$\$(/$(/g')
+  TARGET=$(eval echo "$CLEANED")
+  if [[ "$TARGET" =~ pr-.+ ]]; then
+    git fetch origin pull/$(echo "$TARGET" | cut -d '-' -f 2)/head:build-pr
+    git checkout build-pr
+  else
+    git checkout "$TARGET"
+  fi
+  make -j$(nproc) update
+  make -j$(nproc) nimbus_verified_proxy
+EOF
+
+
+# Pull all binaries into a second stage deploy debian container
+FROM debian:trixie-slim
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y --no-install-recommends \
+  ca-certificates \
+  tzdata \
+  gosu \
+  adduser \
+  bash \
+  curl \
+  jq \
+  && gosu nobody true \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+ARG USER=user
+ARG UID=11001
+
+# See https://stackoverflow.com/a/55757473/12429735RUN
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/usr/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    "${USER}"
+
+RUN mkdir -p /var/lib/nimbus && chown -R ${USER}:${USER} /var/lib/nimbus && chmod 700 /var/lib/nimbus
+
+# Cannot assume buildkit, hence no chmod
+COPY --from=builder --chown=${USER}:${USER} /usr/src/nimbus-eth1/build/nimbus_verified_proxy /usr/local/bin/
+COPY --chown=${USER}:${USER} ./docker-entrypoint.sh /usr/local/bin/
+# Belt and suspenders
+RUN chmod -R 755 /usr/local/bin/*
+
+USER ${USER}
+
+ENTRYPOINT ["nimbus_verified_proxy"]

nimbus-vp/docker-entrypoint.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$(id -u)" -eq 0 ]]; then
+  chown -R user:user /var/lib/nimbus
+  exec gosu user docker-entrypoint.sh "$@"
+fi
+
+
+while true; do
+  if curl -s -m 5 "${CL_NODE}" &> /dev/null; then
+    echo "Consensus Layer node is up, fetching trusted block root"
+    break
+  else
+    echo "Waiting for Consensus Layer node to be reachable..."
+    sleep 5
+  fi
+done
+
+set +e
+  root=$(curl -s -f -m 30 "${CL_NODE}/eth/v1/beacon/headers/finalized" | jq -r '.data.root')
+  exitstatus=$?
+set -e
+
+if [[ "${exitstatus}" -ne 0 ]]; then
+  echo "Failed to fetch trusted block root from ${CL_NODE}"
+  echo "Please verify it's reachable"
+  sleep 30
+  exit 1
+fi
+
+__trusted_root="--trusted-block-root=${root}"
+i=0
+# Verified proxy can get "stuck" if light client bootstrap isn't ready. Check for it here
+while true; do
+  if curl -s -f -m 30 "${CL_NODE}/eth/v1/beacon/light_client/bootstrap/${root}" &> /dev/null; then
+    echo "Consensus Layer node has light client bootstrap available, starting Nimbus Verified Proxy"
+    break
+  else
+    ((++i))
+    if [[ "$i" -eq 4 ]]; then
+      echo "Failed to get light client bootstrap data four times in a row. Waiting for epoch switchover to try with fresh block root"
+      secs=370  # Plus the 15 we already waited, 385. Epoch is 384
+      while [ $secs -gt 0 ]; do
+       echo "Waiting for $secs seconds"
+       sleep 10
+       ((secs -= 10)) || true  # To protect against "falsy" evaluation when secs==10, in some version of bash
+      done
+      exit 1
+    else
+      echo "Waiting for Consensus Layer node to have light client bootstrap data..."
+      sleep 5
+    fi
+  fi
+done
+
+# Word splitting is desired for the command line parameters
+# shellcheck disable=SC2086
+exec "$@" ${__trusted_root}

rpc-proxy-shared.yml

@@ -0,0 +1,6 @@
+# To be used in conjunction with nimbus-vp.yml
+services:
+  rpc-proxy:
+    ports:
+      - ${SHARE_IP:-}:${PROXY_RPC_PORT}:${PROXY_RPC_PORT:-48545}/tcp
+      - ${SHARE_IP:-}:${PROXY_WS_PORT}:${PROXY_WS_PORT:-48546}/tcp

rpc-proxy-traefik.yml

@@ -0,0 +1,15 @@
+# To be used in conjunction with nimbus-vp.yml
+services:
+  rpc-proxy:
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.${PROXY_RPC_HOST:-rpc}.service=${PROXY_RPC_HOST:-rpc}
+      - traefik.http.routers.${PROXY_RPC_HOST:-rpc}.entrypoints=websecure
+      - traefik.http.routers.${PROXY_RPC_HOST:-rpc}.rule=Host(`${PROXY_RPC_HOST:-rpc}.${DOMAIN}`)
+      - traefik.http.routers.${PROXY_RPC_HOST:-rpc}.tls.certresolver=letsencrypt
+      - traefik.http.services.${PROXY_RPC_HOST:-rpc}.loadbalancer.server.port=${PROXY_RPC_PORT:-48545}
+      - traefik.http.routers.${PROXY_WS_HOST:-ws}.service=${PROXY_WS_HOST:-ws}
+      - traefik.http.routers.${PROXY_WS_HOST:-ws}.entrypoints=websecure
+      - traefik.http.routers.${PROXY_WS_HOST:-ws}.rule=Host(`${PROXY_WS_HOST:-ws}.${DOMAIN}`)
+      - traefik.http.routers.${PROXY_WS_HOST:-ws}.tls.certresolver=letsencrypt
+      - traefik.http.services.${PROXY_WS_HOST:-ws}.loadbalancer.server.port=${PROXY_WS_PORT:-48546}