Devnet additional check for miner integration

.github/MINER_TESTS_ISSUE_TEMPLATE.md

@@ -0,0 +1,8 @@
+---
+title: "[automated] Miner integration tests failure @ {{ date | date('D/M/YY HH:mm') }}"
+labels: ["Bug"]
+---
+
+## Description
+
+Miner integration tests failed. Please [check the logs]({{ env.WORKFLOW_URL }}) for more information.

.github/workflows/forest.yml

@@ -488,12 +488,7 @@ jobs:
           export DEVNET_DIR=${{ matrix.dir }}
           source ./scripts/devnet/test_harness.sh
           devnet_test_env_init
-          forest-dev tests mpool
-          forest-dev tests wallet
-          forest-dev devnet eth-gas
-          forest-dev devnet eth-skip-sender
-          forest-dev devnet state-call
-          forest-dev devnet eth-pending-tx
+          devnet_integration_tests
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
       - name: Dump docker logs
         if: always()

.github/workflows/miner-tests.yml

@@ -0,0 +1,160 @@
+name: Miner integration tests
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+  schedule:
+    - cron: "0 13 * * *" # Runs every day at 13:00 UTC
+
+env:
+  CI: 1
+  CACHE_TIMEOUT_MINUTES: 5
+  SCRIPT_TIMEOUT_MINUTES: 30
+  FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters
+  SHELL_IMAGE: busybox
+
+jobs:
+  # Pull requests only run the tests with the `Miner` or `Release` label.
+  build:
+    name: Build Forest
+    if: |-
+      ${{ github.event_name != 'pull_request'
+      || contains(github.event.pull_request.labels.*.name, 'Miner')
+      || contains(github.event.pull_request.labels.*.name, 'Release') }}
+    runs-on: ubuntu-24.04
+    env:
+      RUSTC_WRAPPER: sccache
+      CC: sccache clang
+      CXX: sccache clang++
+      AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+      AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+    steps:
+      - name: Configure SCCache variables
+        run: |
+          # External PRs do not have access to 'vars' or 'secrets'.
+          if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then
+            {
+              echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}"
+              echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}"
+              echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}"
+            } >> "$GITHUB_ENV"
+          fi
+      - uses: rui314/setup-mold@v1
+        with:
+          make-default: false
+      - name: Checkout Sources
+        uses: actions/checkout@v7
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+        continue-on-error: true
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: "go.work"
+          cache-dependency-path: "**/go.sum"
+      - uses: jdx/mise-action@v5
+      - name: Install Forest
+        env:
+          # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
+          RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=mold"
+        run: mise run install --slim quick
+      - uses: actions/upload-artifact@v7
+        with:
+          name: "forest-${{ runner.os }}"
+          path: |
+            ~/.cargo/bin/forest*
+          if-no-files-found: error
+
+  miner-tests:
+    name: Miner integration tests (${{ matrix.group }})
+    needs: build
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # The `forest-dev devnet miner --group` values.
+        group: [onboarding]
+    env:
+      # We use a custom Dockerfile for CI to speed up the build process.
+      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
+      MINER_TEST_GROUP: ${{ matrix.group }}
+    steps:
+      - uses: actions/cache@v6
+        id: cache
+        with:
+          path: "${{ env.FIL_PROOFS_PARAMETER_CACHE }}"
+          key: proof-params-2k
+      - name: Load cache into volume
+        if: steps.cache.outputs.cache-hit == 'true'
+        run: |
+          docker volume create devnet-forest-miner_filecoin-proofs
+          docker run --rm \
+            -v devnet-forest-miner_filecoin-proofs:/proofs \
+            -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \
+            "$SHELL_IMAGE" \
+            sh -c "cp /cache/* /proofs"
+      - uses: actions/checkout@v7
+      - uses: actions/download-artifact@v8
+        with:
+          name: "forest-${{ runner.os }}"
+      - name: Devnet setup
+        run: ./scripts/devnet-forest-miner/setup.sh
+        timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
+      - name: Devnet check
+        run: ./scripts/devnet-forest-miner/check.sh
+        timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
+      - name: Add forest binaries to PATH
+        run: |
+          chmod +x "$GITHUB_WORKSPACE"/forest*
+          echo "$GITHUB_WORKSPACE" >> "$GITHUB_PATH"
+      - name: Miner tests
+        run: |
+          set -euo pipefail
+          export DEVNET_DIR=scripts/devnet-forest-miner
+          source ./scripts/devnet/test_harness.sh
+          devnet_test_env_init
+          forest-dev devnet miner --group "$MINER_TEST_GROUP"
+        timeout-minutes: 120
+      - name: Dump docker logs
+        if: always()
+        uses: jwalton/gh-docker-logs@v2
+      - name: Prepare cache folder for uploading
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          docker run --rm \
+            -v devnet-forest-miner_filecoin-proofs:/proofs \
+            -v "$FIL_PROOFS_PARAMETER_CACHE":/cache \
+            "$SHELL_IMAGE" \
+            sh -c "cp /proofs/* /cache"
+          sudo chmod -R 755 "$FIL_PROOFS_PARAMETER_CACHE"
+
+  # One issue per failed scheduled run, including a failed build.
+  report-failure:
+    name: Report failure
+    permissions:
+      contents: read
+      issues: write
+    needs: [build, miner-tests]
+    if: ${{ failure() && github.event_name == 'schedule' }}
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set WORKFLOW_URL
+        run: |
+          export WORKFLOW_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          echo "${WORKFLOW_URL}"
+          echo "WORKFLOW_URL=${WORKFLOW_URL}" >> "$GITHUB_ENV"
+      - uses: JasonEtco/create-an-issue@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          filename: .github/MINER_TESTS_ISSUE_TEMPLATE.md

CHANGELOG.md

@@ -27,16 +27,22 @@
 
 ### Breaking
 
+- [#7759](https://github.com/ChainSafe/forest/pull/7759): A devnet refuses to start when a `FOREST_*_HEIGHT` variable is not an integer or the enabled upgrades are not in increasing order, matching Lotus. Previously Forest used the default height or re-sorted the upgrades.
+
 ### Added
 
 - [#7781](https://github.com/ChainSafe/forest/pull/7781): Actor bundles are now verified against their CIDs, and missing or tampered blocks are replaced.
 
 ### Changed
 
+- [#7759](https://github.com/ChainSafe/forest/pull/7759): Devnet defaults match Lotus v1.38.0 2k builds: the genesis network version defaults to 28, the upgrade heights follow Lotus, and the minimum consensus power is 2048.
+
 ### Removed
 
 ### Fixed
 
+- [#7759](https://github.com/ChainSafe/forest/pull/7759): An upgrade with a negative epoch is disabled, and an enabled devnet upgrade keeps its own network version when that version is below the genesis version, matching Lotus.
+
 ## Forest v0.39.0 "Vijayadashami"
 
 Non-mandatory release for all node operators. It includes fixes and improvements, notably around the Ethereum and state RPC methods. Note that this release contains breaking changes, so please read the changelog carefully before upgrading.

docs/docs/users/guides/offline_forest.md

@@ -117,7 +117,7 @@ Sample output:
   "result": {
     "NetworkName": "localnet-55c7758d-c91a-41eb-94a2-718cb4601bc5",
     "BlockDelaySecs": 4,
-    "ConsensusMinerMinPower": "2040",
+    "ConsensusMinerMinPower": "2048",
     "SupportedProofTypes": [
       0,
       1

docs/docs/users/reference/cli.sh

@@ -219,4 +219,5 @@ generate_markdown_section "forest-dev" "devnet"
 generate_markdown_section "forest-dev" "devnet eth-gas"
 generate_markdown_section "forest-dev" "devnet eth-skip-sender"
 generate_markdown_section "forest-dev" "devnet state-call"
+generate_markdown_section "forest-dev" "devnet miner"
 generate_markdown_section "forest-dev" "devnet eth-pending-tx"

mise.toml

@@ -244,11 +244,15 @@ forest-dev tests wallet
 
 [tasks."test:devnet"]
 description = "Run the integration test suites against a local docker devnet."
-run = "scripts/devnet/run_integration_tests.sh scripts/devnet"
+run = "scripts/devnet/with_devnet.sh scripts/devnet devnet_integration_tests"
 
 [tasks."test:devnet:forest-miner"]
 description = "Run the integration test suites against the Forest-produced local docker devnet."
-run = "scripts/devnet/run_integration_tests.sh scripts/devnet-forest-miner"
+run = "scripts/devnet/with_devnet.sh scripts/devnet-forest-miner devnet_integration_tests"
+
+[tasks."test:devnet:miner"]
+description = "Run the Lotus Miner integration tests against the Forest-produced local docker devnet."
+run = "scripts/devnet/with_devnet.sh scripts/devnet-forest-miner forest-dev devnet miner"
 
 [tasks."codecov:nextest"]
 description = "Run codecov with nextest"

scripts/devnet-forest-miner/docker-compose.yml

@@ -195,7 +195,7 @@ services:
     environment:
       <<: *lotus-heights
       FIL_PROOFS_PARAMETER_CACHE: ${FIL_PROOFS_PARAMETER_CACHE}
-      LOTUS_MINER_API_LISTENADDRESS: /ip4/0.0.0.0/tcp/${MINER_RPC_PORT}
+      LOTUS_API_LISTENADDRESS: /ip4/0.0.0.0/tcp/${MINER_RPC_PORT}/http
     env_file:
       - lotus-miner.env
     networks:

scripts/devnet/test_harness.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-# Sourced (not executed) helpers for the devnet test suites (wallet, mpool,
-# eth-gas, eth-skip-sender) on the docker devnet. Run after the devnet is up (`setup.sh`) and synced (`check.sh`).
+# Sourced (not executed) helpers for the devnet test suites. Run after the devnet is up (`setup.sh`) and synced (`check.sh`).
 #
 # The genesis key is the Lotus miner's default wallet, so using it as the test
 # sender causes nonce contention. We fund a dedicated wallet instead.
@@ -61,3 +60,13 @@ function devnet_test_env_init {
   ${FOREST_CLI_PATH} chain head
   echo "  balance=${balance}"
 }
+
+# The integration suites shared by both devnets. Run after `devnet_test_env_init`.
+function devnet_integration_tests {
+  forest-dev tests mpool
+  forest-dev tests wallet
+  forest-dev devnet eth-gas
+  forest-dev devnet eth-skip-sender
+  forest-dev devnet state-call
+  forest-dev devnet eth-pending-tx
+}

scripts/devnet/run_integration_tests.sh → scripts/devnet/with_devnet.sh

@@ -1,9 +1,13 @@
 #!/bin/bash
-# Bring up the devnet in $1, run the integration suites against it over Forest's
-# RPC, and tear it down on exit. Shared by both the lotus- and forest-produced devnets.
+# Bring up the devnet in $1, run the remaining arguments against it, and tear it down on exit.
 set -euo pipefail
 
-DEVNET_DIR="${1:?usage: run_integration_tests.sh <devnet-dir>}"
+if [[ $# -lt 2 ]]; then
+  echo "usage: with_devnet.sh <devnet-dir> <command...>" >&2
+  exit 1
+fi
+DEVNET_DIR="$1"
+shift
 export DEVNET_DIR
 
 REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
@@ -18,9 +22,4 @@ popd >/dev/null
 
 source ./scripts/devnet/test_harness.sh
 devnet_test_env_init
-forest-dev tests mpool
-forest-dev tests wallet
-forest-dev devnet eth-gas
-forest-dev devnet eth-skip-sender
-forest-dev devnet state-call
-forest-dev devnet eth-pending-tx
+"$@"

src/dev/subcommands/devnet_cmd.rs

@@ -9,6 +9,7 @@
 mod eth_gas;
 mod eth_pending_tx;
 mod eth_skip_sender;
+mod miner;
 mod state_call;
 
 use crate::dev::subcommands::tests_cmd::helpers::{docker, forest_client, lotus_client};
@@ -21,6 +22,7 @@ pub enum DevnetCommand {
     EthGas(eth_gas::EthGasTestCommand),
     EthPendingTx(eth_pending_tx::EthPendingTxTestCommand),
     EthSkipSender(eth_skip_sender::EthSkipSenderTestCommand),
+    Miner(miner::MinerTestCommand),
     StateCall(state_call::StateCallTestCommand),
 }
 
@@ -31,6 +33,7 @@ impl DevnetCommand {
             Self::EthGas(cmd) => cmd.run().await,
             Self::EthPendingTx(cmd) => cmd.run().await,
             Self::EthSkipSender(cmd) => cmd.run().await,
+            Self::Miner(cmd) => cmd.run().await,
             Self::StateCall(cmd) => cmd.run().await,
         }
     }