diff --git a/scripts/tests/external-rpc-checks/.env b/scripts/tests/external-rpc-checks/.env index c4eef04913a..84ddafc20d0 100644 --- a/scripts/tests/external-rpc-checks/.env +++ b/scripts/tests/external-rpc-checks/.env @@ -3,4 +3,3 @@ FOREST_IMAGE=ghcr.io/chainsafe/forest:edge-fat FOREST_RPC_PORT=2345 FOREST_HEALTHZ_RPC_PORT=2346 FOREST_CHAIN=calibnet -FOREST_CHECK_EPOCHS=1000 diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml index 35e9c5d6cd2..ef54472d46f 100644 --- a/scripts/tests/external-rpc-checks/docker-compose.yaml +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -18,7 +18,6 @@ services: entrypoint: ["ruby", "/resolve.rb"] environment: - FOREST_CHAIN=${FOREST_CHAIN} - - FOREST_CHECK_EPOCHS=${FOREST_CHECK_EPOCHS} init: <<: *forest-service @@ -33,6 +32,7 @@ services: environment: - FOREST_PATH=/data - FOREST_CHAIN_INDEXER_ENABLED=1 + - FULLNODE_API_INFO=/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http command: - --chain=${FOREST_CHAIN} - --encrypt-keystore=false @@ -47,6 +47,7 @@ services: "forest-cli", "healthcheck", "live", + "--wait", "--healthcheck-port", "${FOREST_HEALTHZ_RPC_PORT}", ] diff --git a/scripts/tests/external-rpc-checks/resolve.rb b/scripts/tests/external-rpc-checks/resolve.rb index 1efd8f5a86e..de5121039af 100644 --- a/scripts/tests/external-rpc-checks/resolve.rb +++ b/scripts/tests/external-rpc-checks/resolve.rb @@ -1,32 +1,17 @@ # frozen_string_literal: true -# Picks the ${FOREST_CHAIN} snapshot published DAYS_AGO days ago (UTC), by the date in its name, -# and records under /data what the rest of the harness needs: its URL, its head epoch, and the -# chain plus the ${FOREST_CHECK_EPOCHS}-epoch range the checks cover. -# Runs in the checks image, which has Ruby, so the Forest image needs no curl. +# Picks the newest ${FOREST_CHAIN} snapshot whose head lies within the first 2000 epochs of +# the UTC day: it holds the state the backfill needs to reach the day's start, +# and the node syncs the rest. Writes its URL, head epoch and the day's epoch range under /data. require 'date' require 'json' require 'net/http' -# Compose passes an unset host variable through as an empty string, which `ENV.fetch` accepts. -def env(name) - value = ENV[name].to_s - abort "#{name} is not set" if value.empty? - - value -end - -def env_int(name) - Integer(env(name)) -rescue ArgumentError => e - abort e.message -end - -chain = env('FOREST_CHAIN') -epochs = env_int('FOREST_CHECK_EPOCHS') -days_ago = env_int('DAYS_AGO') -LIST = URI("https://forest-archive.chainsafe.dev/list/#{chain}/latest-v2?format=json") +SECONDS_IN_EPOCH = 30 +EPOCHS_IN_DAY = 2880 +SNAPSHOT_DEPTH = 2000 +GENESIS = { 'calibnet' => 1_667_326_380, 'mainnet' => 1_598_306_400 }.freeze def fetch(uri, attempts: 3) reason = nil @@ -42,13 +27,27 @@ def fetch(uri, attempts: 3) abort "failed to fetch #{uri} after #{attempts} attempts: #{reason}" end -day = (Time.now.utc.to_date - days_ago).iso8601 -urls = JSON.parse(fetch(LIST))['items'].map { |item| item['url'] } -url = urls.find { |candidate| candidate.include?("_#{day}_") } -abort "no #{chain} snapshot published for #{day}" if url.nil? -epoch = Integer(url[/_height_(\d+)/, 1]) +chain = ENV.fetch('FOREST_CHAIN') +days_ago = Integer(ENV.fetch('DAYS_AGO')) +genesis = GENESIS.fetch(chain) { abort "no genesis timestamp for #{chain}" } + +day = Time.now.utc.to_date - days_ago +start = (Time.utc(day.year, day.month, day.day).to_i - genesis) / SECONDS_IN_EPOCH +finish = start + EPOCHS_IN_DAY - 1 + +list = URI("https://forest-archive.chainsafe.dev/list/#{chain}/latest-v2?format=json") +snapshots = JSON.parse(fetch(list))['items'].filter_map do |item| + height = item['url'][/_height_(\d+)/, 1] + [item['url'], Integer(height)] if height +end +reachable = start...start + SNAPSHOT_DEPTH +url, epoch = snapshots.select { |_, height| reachable.cover?(height) }.max_by(&:last) +if url.nil? + warn "no #{chain} snapshot heads within the first #{SNAPSHOT_DEPTH} epochs of #{day.iso8601}" + exit 2 +end File.write('/data/snapshot-url', "#{url}\n") File.write('/data/snapshot-epoch', "#{epoch}\n") -File.write('/data/check-target', "#{chain} #{epoch - epochs} #{epoch - 1}\n") -puts "#{chain} snapshot for #{day}: #{url}" +File.write('/data/check-target', "#{chain} #{start} #{finish}\n") +puts "#{chain} #{day.iso8601} is epochs #{start}..#{finish}; snapshot at #{epoch}: #{url}" diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh index 340f7ff5ca3..24e4df838eb 100755 --- a/scripts/tests/external-rpc-checks/setup.sh +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -10,36 +10,59 @@ pushd "${PARENT_PATH}" # This should not be needed in GH. It is useful for running locally. docker compose down --remove-orphans --volumes -# The dataset publishes each UTC day's archives some time the next morning, so -# yesterday's may not exist yet. Try yesterday's snapshot first, and the day -# before if the dataset has no data for it. -for days_ago in 1 2; do - docker compose run --rm --env DAYS_AGO="${days_ago}" resolve - CHECK_TARGET="$(docker compose run --rm --no-TTY --entrypoint cat resolve /data/check-target)" +# Resolves the snapshot and probes the dataset for DAYS_AGO. +# Sets CHAIN, START and END on success. +# Returns 2 on failure +check_day() { + local status=0 + docker compose run --rm --env DAYS_AGO="$1" resolve || status=$? + case ${status} in + 0) ;; + 2) + echo "no snapshot covers the day $1 day(s) ago" + return 2 + ;; + *) + echo "resolve failed for the day $1 day(s) ago with status ${status}" + exit "${status}" + ;; + esac + + CHECK_TARGET="$(docker compose run --rm --entrypoint cat resolve /data/check-target)" read -r CHAIN START END <<< "${CHECK_TARGET}" [[ -n ${CHAIN} && ${START} =~ ^[0-9]+$ && ${END} =~ ^[0-9]+$ ]] || { echo "resolve did not report a usable chain and epoch range: ${CHECK_TARGET}" exit 1 } - probe=0 - docker compose run --rm rpc-checks --probe --network "${CHAIN}" "${START}" "${END}" || probe=$? - case ${probe} in - 0) break ;; - 2) echo "the dataset has not published that day yet" ;; - *) exit "${probe}" ;; + docker compose run --rm rpc-checks --probe --network "${CHAIN}" "${START}" "${END}" || status=$? + case ${status} in + 0) ;; + 2) + echo "the dataset has not published the day $1 day(s) ago yet" + return 2 + ;; + *) + echo "the dataset probe failed for ${CHAIN} ${START}..${END}" + exit "${status}" + ;; esac -done -[[ ${probe} -eq 0 ]] || { - echo "the dataset has published neither yesterday nor the day before; raise it with its maintainer" - exit 1 } +# Tries to fetch latest available archive dataset (from yesterday) +# if not available lookup an archive dataset before yesterday +# if nothing is available then fail +check_day 1 || check_day 2 + # Imports the snapshot and back-fills the index. docker compose run --rm init docker compose up --detach --wait forest +# Syncs the rest till reach END. +docker compose exec forest \ + timeout 30m forest-cli sync wait --target "$((END + 1))" > /dev/null + docker compose run --rm verify "${START}" "${END}" docker compose run --rm rpc-checks "${START}" "${END}" diff --git a/scripts/tests/external-rpc-checks/verify.rb b/scripts/tests/external-rpc-checks/verify.rb index 567ba308c91..57987db1c5f 100644 --- a/scripts/tests/external-rpc-checks/verify.rb +++ b/scripts/tests/external-rpc-checks/verify.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -# Confirms the back-fill indexed the epochs the checks query. Runs in a container -# so the node's RPC port never has to be published. +# Confirms the node indexed the epochs the checks query. require 'json' require 'net/http'