diff --git a/.github/MINER_TESTS_ISSUE_TEMPLATE.md b/.github/MINER_TESTS_ISSUE_TEMPLATE.md new file mode 100644 index 000000000000..d1fdb84a7e1c --- /dev/null +++ b/.github/MINER_TESTS_ISSUE_TEMPLATE.md @@ -0,0 +1,8 @@ +--- +title: "[automated] Miner integration tests failure @ {{ date | date('D/M/YY HH:mm') }}" +labels: ["Bug"] +--- + +## Description + +Miner integration tests failed. Please [check the logs]({{ env.WORKFLOW_URL }}) for more information. diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index 9c1bc7139264..940dbec12d67 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -488,12 +488,7 @@ jobs: export DEVNET_DIR=${{ matrix.dir }} source ./scripts/devnet/test_harness.sh devnet_test_env_init - forest-dev tests mpool - forest-dev tests wallet - forest-dev devnet eth-gas - forest-dev devnet eth-skip-sender - forest-dev devnet state-call - forest-dev devnet eth-pending-tx + devnet_integration_tests timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} - name: Dump docker logs if: always() diff --git a/.github/workflows/miner-tests.yml b/.github/workflows/miner-tests.yml new file mode 100644 index 000000000000..dce6f87f7675 --- /dev/null +++ b/.github/workflows/miner-tests.yml @@ -0,0 +1,160 @@ +name: Miner integration tests +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +on: + workflow_dispatch: + pull_request: + branches: + - main + types: [opened, synchronize, reopened, labeled, unlabeled] + schedule: + - cron: "0 13 * * *" # Runs every day at 13:00 UTC + +env: + CI: 1 + CACHE_TIMEOUT_MINUTES: 5 + SCRIPT_TIMEOUT_MINUTES: 30 + FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters + SHELL_IMAGE: busybox + +jobs: + # Pull requests only run the tests with the `Miner` or `Release` label. + build: + name: Build Forest + if: |- + ${{ github.event_name != 'pull_request' + || contains(github.event.pull_request.labels.*.name, 'Miner') + || contains(github.event.pull_request.labels.*.name, 'Release') }} + runs-on: ubuntu-24.04 + env: + RUSTC_WRAPPER: sccache + CC: sccache clang + CXX: sccache clang++ + AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + steps: + - name: Configure SCCache variables + run: | + # External PRs do not have access to 'vars' or 'secrets'. + if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then + { + echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" + echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" + echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" + } >> "$GITHUB_ENV" + fi + - uses: rui314/setup-mold@v1 + with: + make-default: false + - name: Checkout Sources + uses: actions/checkout@v7 + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.11 + timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} + continue-on-error: true + - uses: actions/setup-go@v7 + with: + go-version-file: "go.work" + cache-dependency-path: "**/go.sum" + - uses: jdx/mise-action@v5 + - name: Install Forest + env: + # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times + RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=mold" + run: mise run install --slim quick + - uses: actions/upload-artifact@v7 + with: + name: "forest-${{ runner.os }}" + path: | + ~/.cargo/bin/forest* + if-no-files-found: error + + miner-tests: + name: Miner integration tests (${{ matrix.group }}) + needs: build + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + # The `forest-dev devnet miner --group` values. + group: [onboarding] + env: + # We use a custom Dockerfile for CI to speed up the build process. + FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile + MINER_TEST_GROUP: ${{ matrix.group }} + steps: + - uses: actions/cache@v6 + id: cache + with: + path: "${{ env.FIL_PROOFS_PARAMETER_CACHE }}" + key: proof-params-2k + - name: Load cache into volume + if: steps.cache.outputs.cache-hit == 'true' + run: | + docker volume create devnet-forest-miner_filecoin-proofs + docker run --rm \ + -v devnet-forest-miner_filecoin-proofs:/proofs \ + -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \ + "$SHELL_IMAGE" \ + sh -c "cp /cache/* /proofs" + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: "forest-${{ runner.os }}" + - name: Devnet setup + run: ./scripts/devnet-forest-miner/setup.sh + timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} + - name: Devnet check + run: ./scripts/devnet-forest-miner/check.sh + timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} + - name: Add forest binaries to PATH + run: | + chmod +x "$GITHUB_WORKSPACE"/forest* + echo "$GITHUB_WORKSPACE" >> "$GITHUB_PATH" + - name: Miner tests + run: | + set -euo pipefail + export DEVNET_DIR=scripts/devnet-forest-miner + source ./scripts/devnet/test_harness.sh + devnet_test_env_init + forest-dev devnet miner --group "$MINER_TEST_GROUP" + timeout-minutes: 120 + - name: Dump docker logs + if: always() + uses: jwalton/gh-docker-logs@v2 + - name: Prepare cache folder for uploading + if: steps.cache.outputs.cache-hit != 'true' + run: | + docker run --rm \ + -v devnet-forest-miner_filecoin-proofs:/proofs \ + -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \ + "$SHELL_IMAGE" \ + sh -c "cp /proofs/* /cache" + sudo chmod -R 755 "$FIL_PROOFS_PARAMETER_CACHE" + + # One issue per failed scheduled run, including a failed build. + report-failure: + name: Report failure + permissions: + contents: read + issues: write + needs: [build, miner-tests] + if: ${{ failure() && github.event_name == 'schedule' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - name: Set WORKFLOW_URL + run: | + export WORKFLOW_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "${WORKFLOW_URL}" + echo "WORKFLOW_URL=${WORKFLOW_URL}" >> "$GITHUB_ENV" + - uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + filename: .github/MINER_TESTS_ISSUE_TEMPLATE.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b65c479c8199..8776b69a909c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,16 +27,22 @@ ### Breaking +- [#7759](https://github.com/ChainSafe/forest/pull/7759): A devnet refuses to start when a `FOREST_*_HEIGHT` variable is not an integer or the enabled upgrades are not in increasing order, matching Lotus. Previously Forest used the default height or re-sorted the upgrades. + ### Added - [#7781](https://github.com/ChainSafe/forest/pull/7781): Actor bundles are now verified against their CIDs, and missing or tampered blocks are replaced. ### Changed +- [#7759](https://github.com/ChainSafe/forest/pull/7759): Devnet defaults match Lotus v1.38.0 2k builds: the genesis network version defaults to 28, the upgrade heights follow Lotus, and the minimum consensus power is 2048. + ### Removed ### Fixed +- [#7759](https://github.com/ChainSafe/forest/pull/7759): An upgrade with a negative epoch is disabled, and an enabled devnet upgrade keeps its own network version when that version is below the genesis version, matching Lotus. + ## Forest v0.39.0 "Vijayadashami" Non-mandatory release for all node operators. It includes fixes and improvements, notably around the Ethereum and state RPC methods. Note that this release contains breaking changes, so please read the changelog carefully before upgrading. diff --git a/docs/docs/users/guides/offline_forest.md b/docs/docs/users/guides/offline_forest.md index 2a6a1e21a84e..4ebe205dec4a 100644 --- a/docs/docs/users/guides/offline_forest.md +++ b/docs/docs/users/guides/offline_forest.md @@ -117,7 +117,7 @@ Sample output: "result": { "NetworkName": "localnet-55c7758d-c91a-41eb-94a2-718cb4601bc5", "BlockDelaySecs": 4, - "ConsensusMinerMinPower": "2040", + "ConsensusMinerMinPower": "2048", "SupportedProofTypes": [ 0, 1 diff --git a/docs/docs/users/reference/cli.sh b/docs/docs/users/reference/cli.sh index 41322712234f..21adc2c56fa1 100755 --- a/docs/docs/users/reference/cli.sh +++ b/docs/docs/users/reference/cli.sh @@ -219,4 +219,5 @@ generate_markdown_section "forest-dev" "devnet" generate_markdown_section "forest-dev" "devnet eth-gas" generate_markdown_section "forest-dev" "devnet eth-skip-sender" generate_markdown_section "forest-dev" "devnet state-call" +generate_markdown_section "forest-dev" "devnet miner" generate_markdown_section "forest-dev" "devnet eth-pending-tx" diff --git a/mise.toml b/mise.toml index 6035506e810c..5179f3aef08d 100644 --- a/mise.toml +++ b/mise.toml @@ -244,11 +244,15 @@ forest-dev tests wallet [tasks."test:devnet"] description = "Run the integration test suites against a local docker devnet." -run = "scripts/devnet/run_integration_tests.sh scripts/devnet" +run = "scripts/devnet/with_devnet.sh scripts/devnet devnet_integration_tests" [tasks."test:devnet:forest-miner"] description = "Run the integration test suites against the Forest-produced local docker devnet." -run = "scripts/devnet/run_integration_tests.sh scripts/devnet-forest-miner" +run = "scripts/devnet/with_devnet.sh scripts/devnet-forest-miner devnet_integration_tests" + +[tasks."test:devnet:miner"] +description = "Run the Lotus Miner integration tests against the Forest-produced local docker devnet." +run = "scripts/devnet/with_devnet.sh scripts/devnet-forest-miner forest-dev devnet miner" [tasks."codecov:nextest"] description = "Run codecov with nextest" diff --git a/scripts/devnet-forest-miner/docker-compose.yml b/scripts/devnet-forest-miner/docker-compose.yml index fb9782b0ed5b..2a0962580ac1 100644 --- a/scripts/devnet-forest-miner/docker-compose.yml +++ b/scripts/devnet-forest-miner/docker-compose.yml @@ -195,7 +195,7 @@ services: environment: <<: *lotus-heights FIL_PROOFS_PARAMETER_CACHE: ${FIL_PROOFS_PARAMETER_CACHE} - LOTUS_MINER_API_LISTENADDRESS: /ip4/0.0.0.0/tcp/${MINER_RPC_PORT} + LOTUS_API_LISTENADDRESS: /ip4/0.0.0.0/tcp/${MINER_RPC_PORT}/http env_file: - lotus-miner.env networks: diff --git a/scripts/devnet/test_harness.sh b/scripts/devnet/test_harness.sh index c134f2bd6466..62b2f1e6f68b 100644 --- a/scripts/devnet/test_harness.sh +++ b/scripts/devnet/test_harness.sh @@ -1,6 +1,5 @@ #!/bin/bash -# Sourced (not executed) helpers for the devnet test suites (wallet, mpool, -# eth-gas, eth-skip-sender) on the docker devnet. Run after the devnet is up (`setup.sh`) and synced (`check.sh`). +# Sourced (not executed) helpers for the devnet test suites. Run after the devnet is up (`setup.sh`) and synced (`check.sh`). # # The genesis key is the Lotus miner's default wallet, so using it as the test # sender causes nonce contention. We fund a dedicated wallet instead. @@ -61,3 +60,13 @@ function devnet_test_env_init { ${FOREST_CLI_PATH} chain head echo " balance=${balance}" } + +# The integration suites shared by both devnets. Run after `devnet_test_env_init`. +function devnet_integration_tests { + forest-dev tests mpool + forest-dev tests wallet + forest-dev devnet eth-gas + forest-dev devnet eth-skip-sender + forest-dev devnet state-call + forest-dev devnet eth-pending-tx +} diff --git a/scripts/devnet/run_integration_tests.sh b/scripts/devnet/with_devnet.sh similarity index 51% rename from scripts/devnet/run_integration_tests.sh rename to scripts/devnet/with_devnet.sh index e8c6e8ad6582..6a21af3be833 100755 --- a/scripts/devnet/run_integration_tests.sh +++ b/scripts/devnet/with_devnet.sh @@ -1,9 +1,13 @@ #!/bin/bash -# Bring up the devnet in $1, run the integration suites against it over Forest's -# RPC, and tear it down on exit. Shared by both the lotus- and forest-produced devnets. +# Bring up the devnet in $1, run the remaining arguments against it, and tear it down on exit. set -euo pipefail -DEVNET_DIR="${1:?usage: run_integration_tests.sh }" +if [[ $# -lt 2 ]]; then + echo "usage: with_devnet.sh " >&2 + exit 1 +fi +DEVNET_DIR="$1" +shift export DEVNET_DIR REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) @@ -18,9 +22,4 @@ popd >/dev/null source ./scripts/devnet/test_harness.sh devnet_test_env_init -forest-dev tests mpool -forest-dev tests wallet -forest-dev devnet eth-gas -forest-dev devnet eth-skip-sender -forest-dev devnet state-call -forest-dev devnet eth-pending-tx +"$@" diff --git a/src/dev/subcommands/devnet_cmd.rs b/src/dev/subcommands/devnet_cmd.rs index 67760436a9bf..9898ae06f51c 100644 --- a/src/dev/subcommands/devnet_cmd.rs +++ b/src/dev/subcommands/devnet_cmd.rs @@ -9,6 +9,7 @@ mod eth_gas; mod eth_pending_tx; mod eth_skip_sender; +mod miner; mod state_call; use crate::dev::subcommands::tests_cmd::helpers::{docker, forest_client, lotus_client}; @@ -21,6 +22,7 @@ pub enum DevnetCommand { EthGas(eth_gas::EthGasTestCommand), EthPendingTx(eth_pending_tx::EthPendingTxTestCommand), EthSkipSender(eth_skip_sender::EthSkipSenderTestCommand), + Miner(miner::MinerTestCommand), StateCall(state_call::StateCallTestCommand), } @@ -31,6 +33,7 @@ impl DevnetCommand { Self::EthGas(cmd) => cmd.run().await, Self::EthPendingTx(cmd) => cmd.run().await, Self::EthSkipSender(cmd) => cmd.run().await, + Self::Miner(cmd) => cmd.run().await, Self::StateCall(cmd) => cmd.run().await, } } diff --git a/src/dev/subcommands/devnet_cmd/miner.rs b/src/dev/subcommands/devnet_cmd/miner.rs new file mode 100644 index 000000000000..1461d8167f0b --- /dev/null +++ b/src/dev/subcommands/devnet_cmd/miner.rs @@ -0,0 +1,358 @@ +// Copyright 2019-2026 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +//! Lotus Miner integration tests, ported from Lotus `itests`. They need +//! `scripts/devnet-forest-miner`, where Lotus Miner seals and mines through Forest's RPC. + +mod onboarding; + +use crate::dev::subcommands::tests_cmd::helpers::*; +use crate::rpc::prelude::*; +use crate::rpc::types::{ApiTipsetKey, SectorOnChainInfo}; +use crate::rpc::{ApiPaths, Client, DEFAULT_REQUEST_TIMEOUT, Request, humanize_rpc_error}; +use crate::shim::actors::miner::MinerInfo; +use crate::shim::address::Address; +use anyhow::{Context as _, bail, ensure}; +use clap::ValueEnum as _; +use itertools::Itertools as _; +use libtest_mimic::{Arguments, Trial}; +use serde::Deserialize; +use serde::de::{DeserializeOwned, IgnoredAny}; +use serde_json::json; +use std::marker::PhantomData; +use std::str::FromStr as _; + +/// Lotus Miner integration tests on Forest (`scripts/devnet-forest-miner` only) +#[derive(Debug, clap::Args)] +pub struct MinerTestCommand { + /// Run only this group; all groups run by default + #[arg(long)] + group: Option, + /// Run only the tests whose name contains this string + filter: Option, +} + +/// Tests that share one devnet. CI runs each group as a separate job on its own devnet. +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +enum Group { + Onboarding, +} + +impl MinerTestCommand { + pub async fn run(self) -> anyhow::Result<()> { + let groups = match self.group { + Some(group) => vec![group], + None => Group::value_variants().to_vec(), + }; + let args = Arguments { + filter: self.filter, + test_threads: Some(1), + ..Default::default() + }; + libtest_mimic::run(&args, groups.into_iter().flat_map(tests).collect()).exit(); + } +} + +fn tests(group: Group) -> Vec { + match group { + Group::Onboarding => onboarding::tests(), + } +} + +fn genesis_miner() -> anyhow::Result
{ + let raw = harness_env("MINER_ACTOR_ADDRESS")?; + Address::from_str(&raw).with_context(|| format!("invalid MINER_ACTOR_ADDRESS `{raw}`")) +} + +async fn miner_info(client: &Client, miner: Address) -> anyhow::Result { + Ok(client + .call(StateMinerInfo::request((miner, ApiTipsetKey(None)))?) + .await?) +} + +/// Lotus also marks every descendant of a rejected block bad, so checking Forest's head catches +/// an earlier split. +async fn ensure_lotus_accepts_forest_head(forest: &Client, lotus: &Client) -> anyhow::Result<()> { + let head = forest.call(ChainHead::request(())?).await?; + for cid in head.cids() { + let reason = lotus.call(SyncCheckBad::request((cid,))?).await?; + ensure!( + reason.is_empty(), + "Lotus rejected Forest block {cid} at epoch {}: {reason}", + head.epoch() + ); + } + Ok(()) +} + +async fn sector_info( + client: &Client, + miner: Address, + sector: u64, +) -> anyhow::Result> { + Ok(client + .call(StateSectorGetInfo::request(( + miner, + sector, + ApiTipsetKey(None), + ))?) + .await?) +} + +/// The Lotus Miner API, which Forest's RPC does not define. +struct MinerApi(Client); + +impl MinerApi { + /// Connects with an admin token, which flushing sealing batches needs. + fn new() -> anyhow::Result { + let port = harness_env("MINER_RPC_PORT")?; + let token = lotus_miner_exec(&["auth", "create-token", "--perm", "admin"])?; + let mut url: url::Url = format!("http://127.0.0.1:{port}/").parse()?; + url.set_password(Some(token.as_str())) + .map_err(|()| anyhow::anyhow!("cannot set the Lotus Miner token on {url}"))?; + Ok(Self(Client::from_url(url))) + } + + async fn call( + &self, + method: &'static str, + params: serde_json::Value, + ) -> anyhow::Result { + let request = Request:: { + method_name: method.into(), + params, + result_type: PhantomData, + api_path: ApiPaths::V0, + timeout: *DEFAULT_REQUEST_TIMEOUT, + }; + let result = self + .0 + .call(request) + .await + .with_context(|| format!("Lotus Miner {method}")) + .map_err(humanize_rpc_error)?; + serde_json::from_value(result) + .with_context(|| format!("decoding the Lotus Miner {method} result")) + } + + /// Starts sealing a new CC sector and returns its number. + async fn pledge_sector(&self) -> anyhow::Result { + let id: SectorId = self.call("Filecoin.PledgeSector", json!([])).await?; + Ok(id.number) + } + + async fn sectors_status(&self, sector: u64) -> anyhow::Result { + self.call("Filecoin.SectorsStatus", json!([sector, false])) + .await + } + + /// Sends the queued pre-commit and commit batches now; Lotus Miner otherwise holds them for + /// up to a day. + async fn flush_sealing_batches(&self) -> anyhow::Result<()> { + for method in [ + "Filecoin.SectorPreCommitFlush", + "Filecoin.SectorCommitFlush", + ] { + self.call::(method, json!([])).await?; + } + Ok(()) + } +} + +/// Lotus `abi.SectorID`. +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct SectorId { + number: u64, +} + +/// The fields of Lotus `api.SectorInfo` these tests read. +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct SectorStatus { + state: SectorState, + last_err: String, + log: Vec, +} + +/// Lotus `api.SectorState`. Every state is listed, so one added by a newer Lotus fails to decode. +#[derive(Debug, Deserialize)] +enum SectorState { + /// A new sector, before Lotus processes its first event. + #[serde(rename = "")] + Undefined, + Empty, + WaitDeals, + AddPiece, + Packing, + GetTicket, + PreCommit1, + PreCommit2, + PreCommitting, + PreCommitWait, + SubmitPreCommitBatch, + PreCommitBatchWait, + WaitSeed, + Committing, + CommitFinalize, + CommitFinalizeFailed, + SubmitCommit, + CommitWait, + SubmitCommitAggregate, + CommitAggregateWait, + FinalizeSector, + Proving, + Available, + + SnapDealsWaitDeals, + SnapDealsAddPiece, + SnapDealsPacking, + UpdateReplica, + ProveReplicaUpdate, + SubmitReplicaUpdate, + WaitMutable, + ReplicaUpdateWait, + FinalizeReplicaUpdate, + UpdateActivating, + ReleaseSectorKey, + + ReceiveSector, + + FailedUnrecoverable, + AddPieceFailed, + SealPreCommit1Failed, + SealPreCommit2Failed, + PreCommitFailed, + ComputeProofFailed, + RemoteCommitFailed, + CommitFailed, + PackingFailed, + FinalizeFailed, + DealsExpired, + RecoverDealIDs, + + SnapDealsAddPieceFailed, + SnapDealsDealsExpired, + SnapDealsRecoverDealIDs, + AbortUpgrade, + ReplicaUpdateFailed, + ReleaseSectorKeyFailed, + FinalizeReplicaUpdateFailed, + + Faulty, + FaultReported, + FaultedFinal, + + Terminating, + TerminateWait, + TerminateFinality, + TerminateFailed, + + Removing, + RemoveFailed, + Removed, +} + +/// Lotus `api.SectorLog`. +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct SectorLog { + kind: String, + message: String, +} + +async fn wait_until_proving(miner_api: &MinerApi, sector: u64) -> anyhow::Result<()> { + let waited = poll(&format!("sector {sector} to reach Proving"), || async { + miner_api.flush_sealing_batches().await?; + let status = miner_api.sectors_status(sector).await?; + eprintln!("sector {sector} state: {:?}", status.state); + match status.state { + SectorState::Proving | SectorState::Available => Ok(Some(())), + // Lotus never moves a sector out of these towards `Proving`. + SectorState::FailedUnrecoverable + | SectorState::PackingFailed + | SectorState::DealsExpired + | SectorState::Faulty + | SectorState::FaultReported + | SectorState::FaultedFinal + | SectorState::Terminating + | SectorState::TerminateWait + | SectorState::TerminateFinality + | SectorState::TerminateFailed + | SectorState::Removing + | SectorState::RemoveFailed + | SectorState::Removed => { + bail!( + "sector {sector} reached {:?}, which never leads to Proving", + status.state + ) + } + // Lotus can still move a sector out of these towards `Proving`. + SectorState::Undefined + | SectorState::Empty + | SectorState::WaitDeals + | SectorState::AddPiece + | SectorState::Packing + | SectorState::GetTicket + | SectorState::PreCommit1 + | SectorState::PreCommit2 + | SectorState::PreCommitting + | SectorState::PreCommitWait + | SectorState::SubmitPreCommitBatch + | SectorState::PreCommitBatchWait + | SectorState::WaitSeed + | SectorState::Committing + | SectorState::CommitFinalize + | SectorState::CommitFinalizeFailed + | SectorState::SubmitCommit + | SectorState::CommitWait + | SectorState::SubmitCommitAggregate + | SectorState::CommitAggregateWait + | SectorState::FinalizeSector + | SectorState::SnapDealsWaitDeals + | SectorState::SnapDealsAddPiece + | SectorState::SnapDealsPacking + | SectorState::UpdateReplica + | SectorState::ProveReplicaUpdate + | SectorState::SubmitReplicaUpdate + | SectorState::WaitMutable + | SectorState::ReplicaUpdateWait + | SectorState::FinalizeReplicaUpdate + | SectorState::UpdateActivating + | SectorState::ReleaseSectorKey + | SectorState::ReceiveSector + | SectorState::AddPieceFailed + | SectorState::SealPreCommit1Failed + | SectorState::SealPreCommit2Failed + | SectorState::PreCommitFailed + | SectorState::ComputeProofFailed + | SectorState::RemoteCommitFailed + | SectorState::CommitFailed + | SectorState::FinalizeFailed + | SectorState::RecoverDealIDs + | SectorState::SnapDealsAddPieceFailed + | SectorState::SnapDealsDealsExpired + | SectorState::SnapDealsRecoverDealIDs + | SectorState::AbortUpgrade + | SectorState::ReplicaUpdateFailed + | SectorState::ReleaseSectorKeyFailed + | SectorState::FinalizeReplicaUpdateFailed => Ok(None), + } + }) + .await; + let Err(e) = waited else { + return Ok(()); + }; + let Ok(status) = miner_api.sectors_status(sector).await else { + return Err(e); + }; + let log = status + .log + .iter() + .map(|l| format!("{} {}", l.kind, l.message)) + .join("\n"); + Err(e.context(format!( + "sector {sector} is in {:?}, last error: {:?}\nsector log:\n{log}", + status.state, status.last_err + ))) +} diff --git a/src/dev/subcommands/devnet_cmd/miner/onboarding.rs b/src/dev/subcommands/devnet_cmd/miner/onboarding.rs new file mode 100644 index 000000000000..aa04752291a3 --- /dev/null +++ b/src/dev/subcommands/devnet_cmd/miner/onboarding.rs @@ -0,0 +1,163 @@ +// Copyright 2019-2026 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +//! Lotus Miner creating a miner and pledging a sector through Forest's RPC. Mirrors Lotus +//! `testNonGenesisMiner` and `TestPledgeSectors` (single sector, real 2 KiB proofs). + +use super::{ + MinerApi, ensure_lotus_accepts_forest_head, genesis_miner, miner_info, sector_info, + wait_until_proving, +}; +use crate::dev::subcommands::tests_cmd::helpers::*; +use crate::key_management::generate_key; +use crate::rpc::Client; +use crate::rpc::prelude::*; +use crate::rpc::types::ApiTipsetKey; +use crate::shim::address::Address; +use crate::shim::crypto::SignatureType; +use crate::shim::sector::SectorSize; +use ahash::HashSet; +use anyhow::{Context as _, bail, ensure}; +use libtest_mimic::Trial; +use std::str::FromStr as _; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub(super) fn tests() -> Vec { + vec![ + trial("onboarding::create_miner", || block_on(create_miner())), + trial("onboarding::pledge_sector", || block_on(pledge_sector())), + ] +} + +async fn list_miners(client: &Client) -> anyhow::Result> { + Ok(client + .call(StateListMiners::request((ApiTipsetKey(None),))?) + .await? + .into_iter() + .collect()) +} + +async fn create_miner() -> anyhow::Result<()> { + let forest = forest_client()?; + let lotus = lotus_client()?; + let owner_key = Address::from_str(&FOREST_TEST_PRELOADED_ADDRESS) + .context("parsing FOREST_TEST_PRELOADED_ADDRESS")?; + let owner_id = forest + .call(StateLookupID::request((owner_key, ApiTipsetKey(None)))?) + .await?; + // The miner actor requires a BLS worker; the harness wallet is secp256k1. + let worker_key = generate_key(SignatureType::Bls)?.address; + + // Lotus trails Forest; query both at Lotus's head so the tipset exists on Forest. + let lotus_head = lotus.call(ChainHead::request(())?).await?; + let at_lotus_head = ApiTipsetKey(Some(lotus_head.key().clone())); + let deposit = StateMinerCreationDeposit::request((at_lotus_head.clone(),))?; + let forest_deposit = forest.call(deposit.clone()).await?; + let lotus_deposit = lotus.call(deposit).await?; + ensure!( + forest_deposit == lotus_deposit, + "StateMinerCreationDeposit at epoch {}: forest {forest_deposit}, lotus {lotus_deposit}", + lotus_head.epoch() + ); + // The deposit is the per-byte cap here, so a supply disagreement does not change it. + let supply = StateVMCirculatingSupplyInternal::request((at_lotus_head,))?; + let forest_supply = forest.call(supply.clone()).await?; + let lotus_supply = lotus.call(supply).await?; + ensure!( + forest_supply == lotus_supply, + "StateVMCirculatingSupplyInternal at epoch {}:\nforest: {forest_supply:?}\nlotus: {lotus_supply:?}", + lotus_head.epoch() + ); + + let before = list_miners(&forest).await?; + let repo = format!( + "/tmp/forest-onboarding-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() + ); + let init = init_miner(&repo, owner_key, worker_key); + let _ = docker(&["exec", "lotus-miner", "rm", "-rf", &repo]); + init?; + + let created: Vec
= list_miners(&forest) + .await? + .difference(&before) + .copied() + .collect(); + let [miner] = created.as_slice() else { + bail!("expected exactly one new miner after `lotus-miner init`, found {created:?}"); + }; + + let worker_id = forest + .call(StateLookupID::request((worker_key, ApiTipsetKey(None)))?) + .await?; + let info = miner_info(&forest, *miner).await?; + ensure!( + info.owner == owner_id && info.worker == worker_id, + "miner {miner}: owner {} and worker {}, expected {owner_id} and {worker_id}", + info.owner, + info.worker + ); + ensure!( + info.sector_size == SectorSize::_2KiB, + "miner {miner}: sector size {:?}, expected 2 KiB", + info.sector_size + ); + ensure!( + !info.peer_id.is_empty(), + "miner {miner}: no peer ID recorded" + ); + + poll(&format!("lotus StateGetActor {miner}"), || async { + ensure_lotus_accepts_forest_head(&forest, &lotus).await?; + get_actor(&lotus, *miner).await + }) + .await?; + let lotus_info = miner_info(&lotus, *miner).await?; + ensure!( + lotus_info == info, + "StateMinerInfo for {miner} differs:\nforest: {info:?}\nlotus: {lotus_info:?}" + ); + Ok(()) +} + +fn init_miner(repo: &str, owner: Address, worker: Address) -> anyhow::Result<()> { + let port = harness_env("FOREST_RPC_PORT")?; + let data_dir = harness_env("FOREST_DATA_DIR")?; + let script = format!( + "export LOTUS_MINER_PATH='{repo}' \ + FULLNODE_API_INFO=\"$(cat '{data_dir}/token.jwt'):/dns/forest/tcp/{port}/http\" && \ + exec lotus-miner init --owner='{owner}' --worker='{worker}' --sector-size=2KiB \ + --no-local-storage --nosync" + ); + docker(&["exec", "lotus-miner", "sh", "-c", &script]) + .context("`lotus-miner init` against Forest failed")?; + Ok(()) +} + +/// Pledges one CC sector; the chain verifies the seal proof on prove-commit. +async fn pledge_sector() -> anyhow::Result<()> { + let forest = forest_client()?; + let lotus = lotus_client()?; + let miner_api = MinerApi::new()?; + let miner = genesis_miner()?; + let sector = miner_api.pledge_sector().await?; + + wait_until_proving(&miner_api, sector).await?; + + let forest_info = sector_info(&forest, miner, sector) + .await? + .with_context(|| format!("sector {sector} is Proving, but Forest has no info for it"))?; + let lotus_info = poll( + &format!("lotus StateSectorGetInfo {miner} {sector}"), + || async { + ensure_lotus_accepts_forest_head(&forest, &lotus).await?; + sector_info(&lotus, miner, sector).await + }, + ) + .await?; + ensure!( + forest_info == lotus_info, + "StateSectorGetInfo for sector {sector} differs:\nforest: {forest_info:?}\nlotus: {lotus_info:?}" + ); + Ok(()) +} diff --git a/src/dev/subcommands/tests_cmd/helpers.rs b/src/dev/subcommands/tests_cmd/helpers.rs index ef99337bb419..8083bb7e1413 100644 --- a/src/dev/subcommands/tests_cmd/helpers.rs +++ b/src/dev/subcommands/tests_cmd/helpers.rs @@ -405,10 +405,14 @@ fn api() -> anyhow::Result<&'static (String, String)> { .map_err(|e| anyhow::anyhow!("FULLNODE_API_INFO unavailable: {e}")) } +/// Reads a variable that sourcing the devnet test harness sets. +pub fn harness_env(name: &str) -> anyhow::Result { + std::env::var(name).with_context(|| format!("{name} not set; source the devnet test harness")) +} + /// Typed client for the Lotus node on the docker devnet, whose read methods need no token. pub fn lotus_client() -> anyhow::Result { - let port = std::env::var("LOTUS_RPC_PORT") - .context("LOTUS_RPC_PORT not set; source the devnet test harness")?; + let port = harness_env("LOTUS_RPC_PORT")?; Ok(crate::rpc::Client::from_url( format!("http://127.0.0.1:{port}/").parse()?, )) @@ -430,6 +434,13 @@ pub fn lotus_exec(args: &[&str]) -> anyhow::Result { docker(&full) } +/// Runs `lotus-miner ` inside the `lotus-miner` container on the docker devnet. +pub fn lotus_miner_exec(args: &[&str]) -> anyhow::Result { + let mut full = vec!["exec", "lotus-miner", "lotus-miner"]; + full.extend_from_slice(args); + docker(&full) +} + /// POST a JSON-RPC v1 request and return the `result` field, or `None` if /// the server responded without one. pub async fn rpc_call_opt(method: &str, params: Value) -> anyhow::Result> { diff --git a/src/networks/devnet/mod.rs b/src/networks/devnet/mod.rs index 815d56a07544..ad99ce1a25c4 100644 --- a/src/networks/devnet/mod.rs +++ b/src/networks/devnet/mod.rs @@ -3,7 +3,9 @@ use cid::Cid; use indexmap::IndexMap; +use itertools::Itertools as _; use std::sync::LazyLock; +use strum::IntoEnumIterator as _; use crate::{eth::EthChainId, make_height, shim::version::NetworkVersion}; @@ -25,46 +27,46 @@ pub static GENESIS_NETWORK_VERSION: LazyLock = LazyLock::new(|| .expect("Invalid genesis network version"), ) } else { - NetworkVersion::V21 + NetworkVersion::V28 } }); /// Height epochs. -/// Environment variable names follow -/// +/// Defaults follow +/// . pub static HEIGHT_INFOS: LazyLock> = LazyLock::new(|| { - IndexMap::from_iter([ + let height_infos: IndexMap = IndexMap::from_iter([ make_height!( Breeze, - get_upgrade_height_from_env("FOREST_BREEZE_HEIGHT").unwrap_or(-50) + get_upgrade_height_from_env("FOREST_BREEZE_HEIGHT").unwrap_or(-1) ), make_height!( Smoke, - get_upgrade_height_from_env("FOREST_SMOKE_HEIGHT").unwrap_or(-2) + get_upgrade_height_from_env("FOREST_SMOKE_HEIGHT").unwrap_or(-1) ), make_height!( Ignition, - get_upgrade_height_from_env("FOREST_IGNITION_HEIGHT").unwrap_or(-3) + get_upgrade_height_from_env("FOREST_IGNITION_HEIGHT").unwrap_or(-2) ), make_height!( Refuel, - get_upgrade_height_from_env("FOREST_REFUEL_HEIGHT").unwrap_or(-4) + get_upgrade_height_from_env("FOREST_REFUEL_HEIGHT").unwrap_or(-3) ), make_height!( - Assembly, - get_upgrade_height_from_env("FOREST_ASSEMBLY_HEIGHT").unwrap_or(-5) + Tape, + get_upgrade_height_from_env("FOREST_TAPE_HEIGHT").unwrap_or(-4) ), make_height!( - Tape, - get_upgrade_height_from_env("FOREST_TAPE_HEIGHT").unwrap_or(-6) + Assembly, + get_upgrade_height_from_env("FOREST_ASSEMBLY_HEIGHT").unwrap_or(-5) ), make_height!( Liftoff, - get_upgrade_height_from_env("FOREST_LIFTOFF_HEIGHT").unwrap_or(-7) + get_upgrade_height_from_env("FOREST_LIFTOFF_HEIGHT").unwrap_or(-6) ), make_height!( Kumquat, - get_upgrade_height_from_env("FOREST_KUMQUAT_HEIGHT").unwrap_or(-8) + get_upgrade_height_from_env("FOREST_KUMQUAT_HEIGHT").unwrap_or(-7) ), make_height!( Calico, @@ -75,12 +77,12 @@ pub static HEIGHT_INFOS: LazyLock> = LazyLock::new( get_upgrade_height_from_env("FOREST_PERSIAN_HEIGHT").unwrap_or(-10) ), make_height!( - Claus, - get_upgrade_height_from_env("FOREST_CLAUS_HEIGHT").unwrap_or(-11) + Orange, + get_upgrade_height_from_env("FOREST_ORANGE_HEIGHT").unwrap_or(-11) ), make_height!( - Orange, - get_upgrade_height_from_env("FOREST_ORANGE_HEIGHT").unwrap_or(-12) + Claus, + get_upgrade_height_from_env("FOREST_CLAUS_HEIGHT").unwrap_or(-12) ), make_height!( Trust, @@ -131,50 +133,66 @@ pub static HEIGHT_INFOS: LazyLock> = LazyLock::new( ), make_height!( Watermelon, - get_upgrade_height_from_env("FOREST_WATERMELON_HEIGHT").unwrap_or(-1), + get_upgrade_height_from_env("FOREST_WATERMELON_HEIGHT").unwrap_or(-24), get_bundle_cid("v12.0.0") ), make_height!( Dragon, - get_upgrade_height_from_env("FOREST_DRAGON_HEIGHT").unwrap_or(20), + get_upgrade_height_from_env("FOREST_DRAGON_HEIGHT").unwrap_or(-25), get_bundle_cid("v13.0.0") ), - make_height!(Phoenix, i64::MIN), + make_height!( + Phoenix, + get_upgrade_height_from_env("FOREST_PHOENIX_HEIGHT").unwrap_or(-26) + ), make_height!( Waffle, - get_upgrade_height_from_env("FOREST_WAFFLE_HEIGHT").unwrap_or(9999999999), + get_upgrade_height_from_env("FOREST_WAFFLE_HEIGHT").unwrap_or(-27), get_bundle_cid("v14.0.0-rc.1") ), make_height!( TukTuk, - get_upgrade_height_from_env("FOREST_TUKTUK_HEIGHT").unwrap_or(9999999999), + get_upgrade_height_from_env("FOREST_TUKTUK_HEIGHT").unwrap_or(-28), get_bundle_cid("v15.0.0-rc1") ), make_height!( Teep, - get_upgrade_height_from_env("FOREST_TEEP_HEIGHT").unwrap_or(9999999999), + get_upgrade_height_from_env("FOREST_TEEP_HEIGHT").unwrap_or(-29), get_bundle_cid("v16.0.1") ), make_height!( Tock, - get_upgrade_height_from_env("FOREST_TOCK_HEIGHT").unwrap_or(9999999999) + get_upgrade_height_from_env("FOREST_TOCK_HEIGHT").unwrap_or(-30) ), make_height!( GoldenWeek, - get_upgrade_height_from_env("FOREST_GOLDEN_WEEK_HEIGHT").unwrap_or(9999999999), + get_upgrade_height_from_env("FOREST_GOLDEN_WEEK_HEIGHT").unwrap_or(-31), get_bundle_cid("v17.0.0") ), make_height!( FireHorse, - get_upgrade_height_from_env("FOREST_FIREHORSE_HEIGHT").unwrap_or(9999999999), + get_upgrade_height_from_env("FOREST_FIREHORSE_HEIGHT").unwrap_or(-32), get_bundle_cid("v18.0.0") ), make_height!( Solstice, - get_upgrade_height_from_env("FOREST_SOLSTICE_HEIGHT").unwrap_or(9999999999), + get_upgrade_height_from_env("FOREST_SOLSTICE_HEIGHT").unwrap_or(200), get_bundle_cid("v19.0.1") ), - ]) + ]); + for ((prev, prev_info), (next, next_info)) in Height::iter() + .filter_map(|height| Some((height, height_infos.get(&height)?))) + .filter(|(_, info)| info.epoch >= 0) + .tuple_windows() + { + assert!( + prev_info.epoch < next_info.epoch, + "upgrade heights must be strictly increasing: {prev} at {}, followed by {next} at {}", + prev_info.epoch, + next_info.epoch + ); + } + height_infos }); fn get_bundle_cid(version: &str) -> Cid { @@ -197,7 +215,7 @@ pub(super) static DRAND_SCHEDULE: LazyLock<[DrandPoint<'static>; 1]> = LazyLock: macro_rules! make_devnet_policy { (v11) => { fil_actors_shared::v11::runtime::Policy { - minimum_consensus_power: 2040.into(), + minimum_consensus_power: 2048.into(), minimum_verified_allocation_size: 256.into(), pre_commit_challenge_delay: 10, valid_pre_commit_proof_type: { @@ -221,7 +239,7 @@ macro_rules! make_devnet_policy { }; ($version:tt) => { fil_actors_shared::$version::runtime::Policy { - minimum_consensus_power: 2040.into(), + minimum_consensus_power: 2048.into(), minimum_verified_allocation_size: 256.into(), pre_commit_challenge_delay: 10, valid_pre_commit_proof_type: { diff --git a/src/networks/mod.rs b/src/networks/mod.rs index 417fe90fbbeb..3ecac5792fdc 100644 --- a/src/networks/mod.rs +++ b/src/networks/mod.rs @@ -9,7 +9,6 @@ use libp2p::Multiaddr; use num_traits::Zero; use serde::{Deserialize, Serialize}; use strum::{Display, EnumIter, IntoEnumIterator}; -use tracing::warn; use crate::beacon::{BeaconPoint, BeaconSchedule, DrandBeacon, DrandConfig}; use crate::db::SettingsStore; @@ -440,10 +439,16 @@ impl ChainConfig { } } - fn network_height(&self, epoch: ChainEpoch) -> Option { + /// Upgrades from the latest down. A negative epoch disables the upgrade. + fn enabled_heights_rev(&self) -> impl Iterator { self.height_infos .iter() .rev() + .filter(|(_, info)| info.epoch >= 0) + } + + fn network_height(&self, epoch: ChainEpoch) -> Option { + self.enabled_heights_rev() .find(|(_, info)| epoch > info.epoch) .map(|(height, _)| *height) } @@ -454,9 +459,7 @@ impl ChainConfig { epoch: ChainEpoch, ) -> Option> { if let Some((height, info, manifest_cid)) = self - .height_infos - .iter() - .rev() + .enabled_heights_rev() .filter_map(|(height, info)| info.bundle.map(|bundle| (*height, info, bundle))) .find(|(_, info, _)| epoch > info.epoch) { @@ -476,7 +479,6 @@ impl ChainConfig { self.network_height(epoch) .map(NetworkVersion::from) .unwrap_or_else(|| self.genesis_network_version()) - .max(self.genesis_network) } /// Returns the network version revision at the given epoch for distinguishing network upgrades @@ -616,14 +618,12 @@ fn get_upgrade_epoch_by_height<'a>( } fn get_upgrade_height_from_env(env_var_key: &str) -> Option { - if let Ok(value) = std::env::var(env_var_key) { - if let Ok(epoch) = value.parse() { - return Some(epoch); - } else { - warn!("Failed to parse {env_var_key}={value}, value should be an integer"); - } - } - None + let value = std::env::var(env_var_key).ok()?; + Some( + value + .parse() + .unwrap_or_else(|e| panic!("failed to parse {env_var_key}={value}: {e}")), + ) } #[macro_export] @@ -766,10 +766,10 @@ mod tests { } #[test] + #[should_panic(expected = "failed to parse FOREST_TEST_VAR_3=foo")] fn test_get_upgrade_height_invalid_env_var() { unsafe { std::env::set_var("FOREST_TEST_VAR_3", "foo") }; - let epoch = get_upgrade_height_from_env("FOREST_TEST_VAR_3"); - assert_eq!(epoch, None); + get_upgrade_height_from_env("FOREST_TEST_VAR_3"); } #[test] @@ -833,6 +833,15 @@ mod tests { ChainConfig::butterflynet(); } + #[test] + fn network_version_keeps_an_upgrade_below_genesis() { + let mut cfg = ChainConfig::devnet(); + cfg.genesis_network = NetworkVersion::V28; + cfg.height_infos.get_mut(&Height::Lightning).unwrap().epoch = 3; + assert_eq!(cfg.network_version(3), NetworkVersion::V28); + assert_eq!(cfg.network_version(4), NetworkVersion::V19); + } + #[test] fn network_version() { let cfg = ChainConfig::calibnet(); diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index b484aa4cbd0d..6140d45d271c 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -468,7 +468,7 @@ use openrpc_types::{self, ParamStructure}; pub const DEFAULT_PORT: u16 = 2345; /// Request timeout read from environment variables -static DEFAULT_REQUEST_TIMEOUT: LazyLock = LazyLock::new(|| { +pub(crate) static DEFAULT_REQUEST_TIMEOUT: LazyLock = LazyLock::new(|| { env::var("FOREST_RPC_DEFAULT_TIMEOUT") .ok() .and_then(|it| Duration::from_secs(it.parse().ok()?).into()) diff --git a/src/state_manager/state_computation.rs b/src/state_manager/state_computation.rs index 4b8364dadad8..29006bdd8ef8 100644 --- a/src/state_manager/state_computation.rs +++ b/src/state_manager/state_computation.rs @@ -556,18 +556,17 @@ impl<'a> TipsetExecutor<'a> { where F: FnMut(MessageCallbackCtx<'_>) -> anyhow::Result<()>, { - use crate::shim::clock::EPOCH_DURATION_SECONDS; - let mut parent_state = *self.tipset.parent_state(); let parent_epoch = self .chain_index .load_required_tipset(self.tipset.parents())? .epoch(); let epoch = self.tipset.epoch(); + let block_delay_secs = u64::from(self.chain_config.block_delay_secs); for epoch_i in parent_epoch..epoch { if epoch_i > parent_epoch { - let timestamp = genesis_timestamp + ((EPOCH_DURATION_SECONDS * epoch_i) as u64); + let timestamp = genesis_timestamp + (block_delay_secs * epoch_i as u64); parent_state = with_fvm_stack(|| -> anyhow::Result { let mut vm = self.create_vm(parent_state, epoch_i, timestamp, null_epoch_trace)?; @@ -647,9 +646,11 @@ impl<'a> TipsetExecutor<'a> { /// messages in epoch 10, we have to run cron for epoch 9. However, running /// 'cron' requires the timestamp of the youngest block in the tipset (which /// doesn't exist because there are no blocks in the tipset). Lotus dictates that -/// the timestamp of a null tipset is `30s * epoch` after the genesis timestamp. -/// So, in the above example, if the genesis block was mined at time `X`, the -/// null tipset for epoch 9 will have timestamp `X + 30 * 9`. +/// the timestamp of a null tipset is `BlockDelaySecs * epoch` after the genesis +/// timestamp (30 seconds on mainnet/calibnet, 4 seconds on 2k/devnet). So, in +/// the above example, if the genesis block was mined at time `X` with a +/// 30-second block delay, the null tipset for epoch 9 will have timestamp +/// `X + 30 * 9`. /// /// # Migrations ///