fix: --ipv6-prefix for routed mode, stale state cleanup, snapshot restore #1548

Workflow file for this run

	name: CI

	on:
	workflow_dispatch:
	inputs:
	job:
	description: 'Which job to run'
	type: choice
	default: 'both'
	options:
	- both
	- host
	- container
	pull_request:
	branches: [main]
	paths-ignore:
	- 'kernel/**'
	- 'scripts/claude-assistant/**'
	- '.github/workflows/claude*.yml'
	- '.claude/**'
	- '*.md'
	- 'docs/**'
	push:
	branches: [main]
	paths-ignore:
	- 'kernel/**' # When kernel changes, wait for Build Kernels workflow
	- 'scripts/claude-assistant/**'
	- '.github/workflows/claude*.yml'
	- '.claude/**'
	- '*.md'
	- 'docs/**'
	workflow_run:
	workflows: ["Build Kernels"]
	types: [completed]
	branches: [main]

	# Cancel in-progress runs when a new revision is pushed
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	env:
	CARGO_TERM_COLOR: always
	FUSE_BACKEND_RS: ${{ github.workspace }}/fuse-backend-rs
	FUSER: ${{ github.workspace }}/fuser
	# Cache keys for cargo tools (update versions here when bumping)
	CACHE_KEY_LINT: cargo-tools-audit-0.22.0-deny-0.18.9
	CACHE_KEY_HOST: cargo-tools-nextest-0.9.115-audit-0.22.0-deny-0.18.9

	jobs:
	# Skip expensive CI jobs on main pushes when the tree SHA already passed on a PR.
	# When a PR merges via fast-forward or rebase, the merge commit has the same tree
	# SHA as the PR tip. Re-running all tests on self-hosted runners is wasteful.
	skip-check:
	name: Skip Check
	runs-on: ubuntu-latest
	outputs:
	skip: ${{ steps.check.outputs.skip }}
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 20
	- name: Check if tree SHA already passed CI
	id: check
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	if [ "${{ github.event_name }}" != "push" ]; then
	echo "skip=false" >> "$GITHUB_OUTPUT"
	echo "Not a push event, running CI"
	exit 0
	fi

	TREE_SHA=$(git rev-parse HEAD^{tree})
	echo "Current tree SHA: $TREE_SHA"

	# Check recent successful CI runs on PRs for the same tree SHA
	# Look at the last 20 commits on main to find PR branch tips
	SKIP=false
	for sha in $(git log --format=%H -20 HEAD^ 2>/dev/null \|\| true); do
	COMMIT_TREE=$(git rev-parse "$sha^{tree}" 2>/dev/null \|\| true)
	if [ "$COMMIT_TREE" = "$TREE_SHA" ] && [ "$sha" != "$(git rev-parse HEAD)" ]; then
	# Found a commit with the same tree — check if it passed CI
	STATUS=$(gh api "repos/${{ github.repository }}/commits/$sha/status" --jq '.state' 2>/dev/null \|\| echo "unknown")
	echo "Commit $sha has same tree SHA, CI status: $STATUS"
	if [ "$STATUS" = "success" ]; then
	SKIP=true
	echo "Tree SHA $TREE_SHA already passed CI on commit $sha"
	break
	fi
	fi
	done

	echo "skip=$SKIP" >> "$GITHUB_OUTPUT"

	# Runner 0: Lint (default GitHub runner, no KVM needed)
	# Fast checks: fmt, clippy, audit, deny
	lint:
	name: Lint
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	path: fcvm
	- uses: ./fcvm/.github/actions/checkout-deps
	- name: Install Rust
	uses: dtolnay/rust-toolchain@1.93.0
	- name: Get dependency SHAs
	id: deps
	run: \|
	echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
	echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: fcvm -> target
	shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
	- name: Install build dependencies
	run: sudo apt-get update && sudo apt-get install -y libfuse3-dev libclang-dev clang
	- name: Cache cargo tools
	uses: actions/cache@v5
	with:
	path: ~/.cargo/bin
	key: ${{ env.CACHE_KEY_LINT }}-${{ runner.os }}-${{ runner.arch }}
	- name: Install cargo tools
	run: \|
	which cargo-audit \|\| cargo install cargo-audit@0.22.0 --locked
	which cargo-deny \|\| cargo install cargo-deny@0.18.9 --locked
	- name: Check formatting
	working-directory: fcvm
	run: cargo fmt -p fcvm -p fuse-pipe -p fc-agent --check
	- name: Clippy
	working-directory: fcvm
	run: cargo clippy --all-targets -- -D warnings
	- name: Audit
	working-directory: fcvm
	run: cargo audit
	- name: Deny
	working-directory: fcvm
	run: cargo deny check

	# Runner 0b: Packaging (default GitHub runner, no KVM needed)
	# Verifies cargo install works without source tree access
	packaging:
	name: Packaging
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	path: fcvm
	- uses: ./fcvm/.github/actions/checkout-deps
	- name: Install Rust
	uses: dtolnay/rust-toolchain@1.93.0
	- name: Get dependency SHAs
	id: deps
	run: \|
	echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
	echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: fcvm -> target
	shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
	- name: Install build dependencies
	run: sudo apt-get update && sudo apt-get install -y libfuse3-dev libclang-dev clang
	- name: Build release
	working-directory: fcvm
	run: cargo build --release -p fcvm
	- name: Test packaging
	working-directory: fcvm
	run: ./scripts/test-packaging.sh ./target/release/fcvm

	# Runner 0c: fc-mock (default GitHub runner, no KVM needed)
	# Tests fc-mock (Firecracker mock) which runs containers via podman
	# instead of real Firecracker VMs. No KVM or btrfs required.
	fc-mock:
	name: fc-mock
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	with:
	path: fcvm
	- uses: ./fcvm/.github/actions/checkout-deps
	- name: Install Rust
	uses: dtolnay/rust-toolchain@1.93.0
	- name: Get dependency SHAs
	id: deps
	run: \|
	echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
	echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: fcvm -> target
	shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
	- name: Install build dependencies
	run: sudo apt-get update && sudo apt-get install -y libfuse3-dev libclang-dev clang podman
	- name: Setup podman
	run: \|
	# Disable AppArmor restriction on unprivileged user namespaces
	# (needed for rootless podman on Ubuntu 24.04+)
	sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null \|\| true
	- name: Cache cargo tools
	uses: actions/cache@v5
	with:
	path: ~/.cargo/bin
	key: cargo-nextest-0.9.115-${{ runner.os }}-${{ runner.arch }}
	- name: Install cargo-nextest
	run: which cargo-nextest \|\| cargo install cargo-nextest@0.9.115 --locked
	- name: Build fc-mock
	working-directory: fcvm
	run: \|
	cargo build --release -p fc-mock
	sudo install -m 755 target/release/fc-mock /usr/local/bin/fc-mock
	- name: Test fc-mock
	working-directory: fcvm
	env:
	FCVM_FIRECRACKER_BIN: /usr/local/bin/fc-mock
	RUST_LOG: "fcvm=debug,health-monitor=info,fuser=warn,fuse_backend_rs=warn,passthrough=warn"
	run: \|
	cargo nextest run --release -p fcvm --profile fc-mock \
	--features privileged-tests \
	-E 'package(fcvm) & (test(/fc_mock/) \| test(/state_manager/) \| test(/health_monitor/) \| test(/no_sudo/)) & not test(=test_fc_mock_sanity) & not test(=test_fc_mock_container_launch)'

	# Runner 1a: Host (bare metal with KVM)
	# Runs: test-unit → test-fast (quick tests, no privileged)
	# Self-hosted runners with nested virtualization (ARM64: c7g.metal, x86: c5.metal)
	host:
	name: Host-${{ matrix.arch }}
	needs: [skip-check]
	if: ${{ github.event.inputs.job != 'container' && needs.skip-check.outputs.skip != 'true' }}
	runs-on: [self-hosted, Linux, '${{ matrix.arch }}']
	strategy:
	fail-fast: false
	matrix:
	arch: [arm64, x64]
	steps:
	# Fix ownership of root-owned files from previous test runs (sudo cargo test)
	- name: Fix workspace permissions (pre-checkout)
	run: \|
	sudo chown -R $USER:$USER ${{ github.workspace }}/fcvm/target 2>/dev/null \|\| true
	# Fix cargo advisory-db permissions (can become read-only from previous runs)
	sudo chmod -R u+w ~/.cargo/advisory-db* 2>/dev/null \|\| true
	sudo chown -R $USER:$USER ~/.cargo/advisory-db* 2>/dev/null \|\| true
	- uses: actions/checkout@v6
	with:
	path: fcvm
	- uses: ./fcvm/.github/actions/checkout-deps
	- name: Install Rust
	run: \|
	curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused --location --silent --show-error --fail https://sh.rustup.rs \| sh -s -- --default-toolchain none -y
	echo "$HOME/.cargo/bin" >> $GITHUB_PATH
	# rust-toolchain.toml handles version + components; just trigger install
	rustup show active-toolchain \|\| true
	- name: Get dependency SHAs
	id: deps
	run: \|
	echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
	echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: fcvm -> target
	cache-on-failure: "true"
	shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
	iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted \
	podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
	libseccomp-dev nfs-kernel-server \
	flex bison bc libelf-dev libssl-dev
	- name: Build passt from source
	working-directory: fcvm
	run: ./scripts/build-passt.sh
	- name: Install Firecracker
	run: \|
	ARCH=$(uname -m)
	FC_VERSION="1.14.0"
	curl -L -o /tmp/firecracker.tgz \
	"https://github.com/firecracker-microvm/firecracker/releases/download/v${FC_VERSION}/firecracker-v${FC_VERSION}-${ARCH}.tgz"
	sudo tar -xzf /tmp/firecracker.tgz -C /usr/local/bin --strip-components=1 \
	"release-v${FC_VERSION}-${ARCH}/firecracker-v${FC_VERSION}-${ARCH}" \
	"release-v${FC_VERSION}-${ARCH}/jailer-v${FC_VERSION}-${ARCH}"
	sudo mv "/usr/local/bin/firecracker-v${FC_VERSION}-${ARCH}" /usr/local/bin/firecracker
	sudo mv "/usr/local/bin/jailer-v${FC_VERSION}-${ARCH}" /usr/local/bin/jailer
	- name: Restore cargo tools cache
	id: cache-cargo-tools
	uses: actions/cache/restore@v5
	with:
	path: ~/.cargo/bin
	key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}
	- name: Install cargo tools
	# Skip if already cached
	# Create symlinks in /usr/local/bin so sudo can find cargo tools
	run: \|
	which cargo-nextest \|\| cargo install cargo-nextest@0.9.115 --locked
	which cargo-audit \|\| cargo install cargo-audit@0.22.0 --locked
	which cargo-deny \|\| cargo install cargo-deny@0.18.9 --locked
	sudo ln -sf $HOME/.cargo/bin/cargo /usr/local/bin/
	sudo ln -sf $HOME/.cargo/bin/rustc /usr/local/bin/
	sudo ln -sf $HOME/.cargo/bin/cargo-nextest /usr/local/bin/
	sudo ln -sf $HOME/.cargo/bin/cargo-audit /usr/local/bin/
	sudo ln -sf $HOME/.cargo/bin/cargo-deny /usr/local/bin/
	- name: Setup KVM and networking
	run: \|
	# Print kernel version and check for nested virtualization support
	echo "=== Host Kernel Info ==="
	uname -a
	cat /proc/cmdline
	echo ""
	echo "=== KVM Capabilities ==="
	ARCH=$(uname -m)
	if [ "$ARCH" = "aarch64" ]; then
	cat /sys/module/kvm/parameters/mode 2>/dev/null \|\| echo "kvm mode param not available"
	# Check if nested virt is available (NV2 on ARM64)
	if grep -q "kvm-arm.mode=nested" /proc/cmdline; then
	echo "✓ Nested virtualization ENABLED (kvm-arm.mode=nested)"
	else
	echo "⚠ Nested virtualization NOT enabled in kernel cmdline"
	echo " Nested virtualization tests will fail - add kvm-arm.mode=nested to GRUB_CMDLINE_LINUX"
	fi
	else
	# x86: Check for Intel VT-x or AMD-V nested virtualization
	cat /sys/module/kvm_intel/parameters/nested 2>/dev/null && echo "Intel VT-x nested support" \|\| true
	cat /sys/module/kvm_amd/parameters/nested 2>/dev/null && echo "AMD-V nested support" \|\| true
	if grep -E "^Y\|^1" /sys/module/kvm_intel/parameters/nested 2>/dev/null \|\| \
	grep -E "^Y\|^1" /sys/module/kvm_amd/parameters/nested 2>/dev/null; then
	echo "✓ Nested virtualization ENABLED"
	else
	echo "⚠ Nested virtualization may not be enabled"
	fi
	fi
	echo ""
	sudo chmod 666 /dev/kvm
	sudo mkdir -p /var/run/netns
	if [ ! -e /dev/userfaultfd ]; then
	sudo mknod /dev/userfaultfd c 10 126
	fi
	sudo chmod 666 /dev/userfaultfd
	sudo sysctl -w vm.unprivileged_userfaultfd=1
	# Disable AppArmor restriction on unprivileged user namespaces (needed for rootless networking on newer Ubuntu)
	sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null \|\| true
	# Enable IP forwarding for all interfaces (including future ones like podman0)
	# This fixes podman container networking - without this, containers can't reach the internet
	sudo sysctl -w net.ipv4.conf.all.forwarding=1
	sudo sysctl -w net.ipv4.conf.default.forwarding=1
	# Enable IPv6 forwarding for routed networking mode (veth + IPv6 routing)
	sudo sysctl -w net.ipv6.conf.all.forwarding=1
	sudo sysctl -w net.ipv6.conf.default.forwarding=1
	# Keep accepting Router Advertisements with forwarding enabled (otherwise default route expires).
	# all/default: ensures new interfaces (pasta's virtual devices) accept RAs despite forwarding=1.
	# Per-interface: ensures the host's primary interface keeps its default route.
	sudo sysctl -w net.ipv6.conf.all.accept_ra=2
	sudo sysctl -w net.ipv6.conf.default.accept_ra=2
	DEFAULT_IFACE=$(ip route show default \| awk '/dev/{print $5; exit}')
	sudo sysctl -w "net.ipv6.conf.${DEFAULT_IFACE}.accept_ra=2" 2>/dev/null \|\| true
	# Enable FUSE allow_other for tests
	echo "user_allow_other" \| sudo tee /etc/fuse.conf
	# Move podman storage to btrfs (more space for large container images)
	mkdir -p ~/.config/containers
	printf '[storage]\ndriver = "overlay"\ngraphroot = "/mnt/fcvm-btrfs/containers/storage"\n' > ~/.config/containers/storage.conf
	# Reset podman state if corrupted from previous runs
	podman system migrate \|\| true
	- name: Create test log directory
	run: sudo rm -rf /tmp/fcvm-test-logs && mkdir -p /tmp/fcvm-test-logs
	- name: Clean test data
	working-directory: fcvm
	run: make clean-test-data \|\| true
	- name: test-unit
	working-directory: fcvm
	run: make test-unit
	- name: setup-fcvm
	working-directory: fcvm
	run: make setup-fcvm
	- name: Build btrfs kernel
	working-directory: fcvm
	run: \|
	# Regenerate config for root (setup-fcvm wrote to CI user's config dir)
	sudo ./target/release/fcvm setup --generate-config --force
	# Build btrfs kernel locally (needed for test_localhost_rootless tests)
	sudo ./target/release/fcvm setup --kernel-profile btrfs --build-kernels
	- name: Refresh KVM permissions
	run: sudo chmod 666 /dev/kvm
	- name: test-packaging-e2e
	working-directory: fcvm
	run: ./scripts/test-packaging-e2e.sh ./target/release/fcvm
	- name: test-fast
	working-directory: fcvm
	run: make test-fast
	- name: test-serve-sdk
	working-directory: fcvm
	run: \|
	# SDK test requires computesdk package (sibling repo)
	# On CI, check if it's available; on dev machines, always available
	COMPUTESDK_DIR="${{ github.workspace }}/computesdk"
	if [ ! -d "$COMPUTESDK_DIR/packages/computesdk" ]; then
	echo "⚠ Skipping SDK E2E test (computesdk not checked out)"
	echo " To run locally: cd tests && npm install && cd .. && npx tsx tests/test_serve_sdk.ts"
	exit 0
	fi
	# Install Node.js if not available
	which node \|\| { echo "⚠ Skipping SDK test (Node.js not available)"; exit 0; }
	cd tests && npm install && cd ..
	npx tsx tests/test_serve_sdk.ts
	- name: Upload test logs
	if: always()
	uses: actions/upload-artifact@v6
	with:
	name: test-logs-host-${{ matrix.arch }}
	path: /tmp/fcvm-test-logs/
	if-no-files-found: ignore
	retention-days: 14
	- name: Save cargo tools cache
	if: always() && steps.cache-cargo-tools.outputs.cache-hit != 'true'
	uses: actions/cache/save@v5
	with:
	path: ~/.cargo/bin
	key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}

	# Runner 1b: Host-Root (bare metal with KVM)
	# Runs test-root with matrix for snapshot testing modes and architectures.
	# IMPORTANT: Both snapshot mode entries run on the SAME self-hosted runner to test
	# snapshot persistence across runs.
	#
	# Matrix modes:
	# - SnapshotDisabled: FCVM_NO_SNAPSHOT=1, runs once
	# Tests the code path where snapshot feature is explicitly disabled.
	# Verifies --no-snapshot flag and FCVM_NO_SNAPSHOT env var work correctly.
	#
	# - SnapshotEnabled: No env var, runs TWICE on same runner
	# Run 1: Snapshot miss - creates snapshots for container images
	# Run 2: Snapshot hit - reuses snapshots from Run 1 (should be faster)
	# This validates the full snapshot lifecycle: create -> persist -> restore
	host-root:
	name: Host-Root-${{ matrix.arch }}-${{ matrix.mode }}
	needs: [skip-check]
	if: ${{ github.event.inputs.job != 'container' && needs.skip-check.outputs.skip != 'true' }}
	runs-on: [self-hosted, Linux, '${{ matrix.arch }}']
	strategy:
	fail-fast: false
	matrix:
	arch: [arm64, x64]
	mode: [SnapshotDisabled, SnapshotEnabled]
	include:
	- mode: SnapshotDisabled
	fcvm_no_snapshot: "1"
	test_runs: "1"
	- mode: SnapshotEnabled
	fcvm_no_snapshot: ""
	test_runs: "2"
	env:
	FCVM_NO_SNAPSHOT: ${{ matrix.fcvm_no_snapshot }}
	steps:
	# Fix ownership of root-owned files from previous test runs (sudo cargo test)
	- name: Fix workspace permissions (pre-checkout)
	run: \|
	sudo chown -R $USER:$USER ${{ github.workspace }}/fcvm/target 2>/dev/null \|\| true
	# Fix cargo advisory-db permissions (can become read-only from previous runs)
	sudo chmod -R u+w ~/.cargo/advisory-db* 2>/dev/null \|\| true
	sudo chown -R $USER:$USER ~/.cargo/advisory-db* 2>/dev/null \|\| true
	- uses: actions/checkout@v6
	with:
	path: fcvm
	- uses: ./fcvm/.github/actions/checkout-deps
	- name: Install Rust
	run: \|
	curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused --location --silent --show-error --fail https://sh.rustup.rs \| sh -s -- --default-toolchain none -y
	echo "$HOME/.cargo/bin" >> $GITHUB_PATH
	# rust-toolchain.toml handles version + components; just trigger install
	rustup show active-toolchain \|\| true
	- name: Get dependency SHAs
	id: deps
	run: \|
	echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
	echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: fcvm -> target
	cache-on-failure: "true"
	shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
	iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted \
	podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
	libseccomp-dev nfs-kernel-server \
	flex bison bc libelf-dev libssl-dev
	- name: Build passt from source
	working-directory: fcvm
	run: ./scripts/build-passt.sh
	- name: Install Firecracker
	run: \|
	ARCH=$(uname -m)
	FC_VERSION="1.14.0"
	curl -L -o /tmp/firecracker.tgz \
	"https://github.com/firecracker-microvm/firecracker/releases/download/v${FC_VERSION}/firecracker-v${FC_VERSION}-${ARCH}.tgz"
	sudo tar -xzf /tmp/firecracker.tgz -C /usr/local/bin --strip-components=1 \
	"release-v${FC_VERSION}-${ARCH}/firecracker-v${FC_VERSION}-${ARCH}" \
	"release-v${FC_VERSION}-${ARCH}/jailer-v${FC_VERSION}-${ARCH}"
	sudo mv "/usr/local/bin/firecracker-v${FC_VERSION}-${ARCH}" /usr/local/bin/firecracker
	sudo mv "/usr/local/bin/jailer-v${FC_VERSION}-${ARCH}" /usr/local/bin/jailer
	- name: Restore cargo tools cache
	id: cache-cargo-tools
	uses: actions/cache/restore@v5
	with:
	path: ~/.cargo/bin
	key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}
	- name: Install cargo tools
	run: \|
	which cargo-nextest \|\| cargo install cargo-nextest@0.9.115 --locked
	sudo ln -sf $HOME/.cargo/bin/cargo /usr/local/bin/
	sudo ln -sf $HOME/.cargo/bin/rustc /usr/local/bin/
	sudo ln -sf $HOME/.cargo/bin/cargo-nextest /usr/local/bin/
	- name: Setup KVM and networking
	run: \|
	sudo chmod 666 /dev/kvm
	sudo mkdir -p /var/run/netns
	if [ ! -e /dev/userfaultfd ]; then
	sudo mknod /dev/userfaultfd c 10 126
	fi
	sudo chmod 666 /dev/userfaultfd
	sudo sysctl -w vm.unprivileged_userfaultfd=1
	# Disable AppArmor restriction on unprivileged user namespaces (needed for rootless networking on newer Ubuntu)
	sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null \|\| true
	# Enable IP forwarding for all interfaces (including future ones like podman0)
	# This fixes podman container networking - without this, containers can't reach the internet
	sudo sysctl -w net.ipv4.conf.all.forwarding=1
	sudo sysctl -w net.ipv4.conf.default.forwarding=1
	# Enable IPv6 forwarding for routed networking mode (veth + IPv6 routing)
	sudo sysctl -w net.ipv6.conf.all.forwarding=1
	sudo sysctl -w net.ipv6.conf.default.forwarding=1
	# Keep accepting Router Advertisements with forwarding enabled (otherwise default route expires).
	# all/default: ensures new interfaces (pasta's virtual devices) accept RAs despite forwarding=1.
	# Per-interface: ensures the host's primary interface keeps its default route.
	sudo sysctl -w net.ipv6.conf.all.accept_ra=2
	sudo sysctl -w net.ipv6.conf.default.accept_ra=2
	DEFAULT_IFACE=$(ip route show default \| awk '/dev/{print $5; exit}')
	sudo sysctl -w "net.ipv6.conf.${DEFAULT_IFACE}.accept_ra=2" 2>/dev/null \|\| true
	echo "user_allow_other" \| sudo tee /etc/fuse.conf
	# Move podman storage to btrfs (more space for large container images)
	mkdir -p ~/.config/containers
	printf '[storage]\ndriver = "overlay"\ngraphroot = "/mnt/fcvm-btrfs/containers/storage"\n' > ~/.config/containers/storage.conf
	podman system migrate \|\| true
	# Install iperf3 for network benchmarks
	sudo apt-get update && sudo apt-get install -y iperf3 \|\| true
	- name: Create test log directory
	run: sudo rm -rf /tmp/fcvm-test-logs && mkdir -p /tmp/fcvm-test-logs
	- name: Clean test data
	working-directory: fcvm
	run: make clean-test-data \|\| true
	- name: setup-fcvm
	working-directory: fcvm
	run: make setup-fcvm
	- name: Build nested kernel
	working-directory: fcvm
	run: \|
	# Regenerate config for root (setup-fcvm wrote to CI user's config dir)
	sudo ./target/release/fcvm setup --generate-config --force
	# Build nested kernel locally (don't rely on GitHub releases)
	sudo ./target/release/fcvm setup --kernel-profile nested --build-kernels
	- name: Build btrfs kernel
	working-directory: fcvm
	run: \|
	# Build btrfs kernel locally (needed for test_localhost_rootless tests)
	sudo ./target/release/fcvm setup --kernel-profile btrfs --build-kernels
	- name: Refresh KVM permissions
	run: sudo chmod 666 /dev/kvm
	- name: Allocate hugepages
	run: \|
	echo 512 \| sudo tee /proc/sys/vm/nr_hugepages
	echo "Allocated $(cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages) hugepages"
	- name: test-root
	working-directory: fcvm
	run: \|
	SNAPSHOT_DIR="${FCVM_DATA_DIR:-/mnt/fcvm-btrfs/root}/snapshots"
	for i in $(seq 1 ${{ matrix.test_runs }}); do
	echo ""
	echo "╔══════════════════════════════════════════════════════════════════╗"
	echo "║ Test Run $i of ${{ matrix.test_runs }} (${{ matrix.mode }} mode)"
	echo "╚══════════════════════════════════════════════════════════════════╝"
	echo ""
	BEFORE=$(sudo find "$SNAPSHOT_DIR" -maxdepth 1 -type d 2>/dev/null \| wc -l)
	echo "📦 Snapshot entries before run $i: $((BEFORE - 1))"
	echo ""
	make test-root
	AFTER=$(sudo find "$SNAPSHOT_DIR" -maxdepth 1 -type d 2>/dev/null \| wc -l)
	echo ""
	echo "📦 Snapshot entries after run $i: $((AFTER - 1))"
	if [ "$i" -eq 2 ] && [ "${{ matrix.mode }}" = "SnapshotEnabled" ]; then
	echo "✓ Run 2 complete - snapshot hit paths tested"
	fi
	done
	- name: bench-vm
	if: matrix.mode == 'SnapshotEnabled'
	working-directory: fcvm
	run: \|
	make clean-test-data \|\| true
	make bench-vm
	- name: Capture kernel logs
	if: always()
	run: \|
	sudo dmesg \| grep -iE 'userfault\|uffd\|kvm\|firecracker\|oom\|killed\|segfault\|page.fault' > /tmp/fcvm-test-logs/dmesg-filtered.log \|\| true
	- name: Upload test logs
	if: always()
	uses: actions/upload-artifact@v6
	with:
	name: test-logs-host-root-${{ matrix.arch }}-${{ matrix.mode }}
	path: /tmp/fcvm-test-logs/
	if-no-files-found: ignore
	retention-days: 14
	- name: Save cargo tools cache
	if: always() && steps.cache-cargo-tools.outputs.cache-hit != 'true'
	uses: actions/cache/save@v5
	with:
	path: ~/.cargo/bin
	key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}

	# Runner 2: Container (podman)
	# Runs same tests as Host but inside a container
	# Needs KVM for VM tests (container mounts /dev/kvm)
	container:
	name: Container-${{ matrix.arch }}
	needs: [skip-check]
	if: ${{ github.event.inputs.job != 'host' && needs.skip-check.outputs.skip != 'true' }}
	runs-on: [self-hosted, Linux, '${{ matrix.arch }}']
	strategy:
	fail-fast: false
	matrix:
	arch: [arm64, x64]
	permissions:
	packages: write
	env:
	CONTAINER_ARCH: ${{ matrix.arch == 'arm64' && 'aarch64' \|\| 'x86_64' }}
	steps:
	# Fix ownership of root-owned files from previous test runs (sudo cargo test)
	- name: Fix workspace permissions (pre-checkout)
	run: \|
	sudo chown -R $USER:$USER ${{ github.workspace }}/fcvm/target 2>/dev/null \|\| true
	# Fix cargo advisory-db permissions (can become read-only from previous runs)
	sudo chmod -R u+w ~/.cargo/advisory-db* 2>/dev/null \|\| true
	sudo chown -R $USER:$USER ~/.cargo/advisory-db* 2>/dev/null \|\| true
	- uses: actions/checkout@v6
	with:
	path: fcvm
	- uses: ./fcvm/.github/actions/checkout-deps
	- name: Setup KVM and rootless podman
	run: \|
	sudo chmod 666 /dev/kvm
	# Enable userfaultfd syscall for snapshot cloning
	sudo sysctl -w vm.unprivileged_userfaultfd=1
	# Disable AppArmor restriction on unprivileged user namespaces (needed for rootless networking on newer Ubuntu)
	sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null \|\| true
	# Enable IP forwarding for all interfaces (including future ones like podman0)
	# This fixes podman container networking - without this, containers can't reach the internet
	sudo sysctl -w net.ipv4.conf.all.forwarding=1
	sudo sysctl -w net.ipv4.conf.default.forwarding=1
	# Enable IPv6 forwarding for routed networking mode (veth + IPv6 routing)
	sudo sysctl -w net.ipv6.conf.all.forwarding=1
	sudo sysctl -w net.ipv6.conf.default.forwarding=1
	# Keep accepting Router Advertisements with forwarding enabled (otherwise default route expires).
	# all/default: ensures new interfaces (pasta's virtual devices) accept RAs despite forwarding=1.
	# Per-interface: ensures the host's primary interface keeps its default route.
	sudo sysctl -w net.ipv6.conf.all.accept_ra=2
	sudo sysctl -w net.ipv6.conf.default.accept_ra=2
	DEFAULT_IFACE=$(ip route show default \| awk '/dev/{print $5; exit}')
	sudo sysctl -w "net.ipv6.conf.${DEFAULT_IFACE}.accept_ra=2" 2>/dev/null \|\| true
	# Configure rootless podman to use cgroupfs (no systemd session on CI)
	mkdir -p ~/.config/containers
	printf '[engine]\ncgroup_manager = "cgroupfs"\nevents_logger = "file"\n' > ~/.config/containers/containers.conf
	# Reset podman state if corrupted from previous runs
	podman system migrate \|\| true
	# Create cargo cache directory for container
	mkdir -p ${{ github.workspace }}/cargo-cache/registry ${{ github.workspace }}/cargo-cache/target
	- name: Login to ghcr.io
	run: echo "${{ github.token }}" \| podman login ghcr.io -u ${{ github.actor }} --password-stdin
	- name: Cache container cargo
	uses: actions/cache@v5
	with:
	path: ${{ github.workspace }}/cargo-cache
	key: container-cargo-${{ hashFiles('fcvm/Cargo.lock') }}
	restore-keys: container-cargo-
	- name: Create test log directory
	run: sudo rm -rf /tmp/fcvm-test-logs && mkdir -p /tmp/fcvm-test-logs
	- name: Clean test data
	working-directory: fcvm
	run: make clean-test-data \|\| true
	- name: container-test-unit
	env:
	CARGO_CACHE_DIR: ${{ github.workspace }}/cargo-cache
	working-directory: fcvm
	run: make container-test-unit
	- name: container-setup-fcvm
	env:
	CARGO_CACHE_DIR: ${{ github.workspace }}/cargo-cache
	working-directory: fcvm
	run: make container-setup-fcvm
	- name: container-test
	env:
	CARGO_CACHE_DIR: ${{ github.workspace }}/cargo-cache
	working-directory: fcvm
	run: make container-test
	- name: Capture kernel logs
	if: always()
	run: \|
	# Filter dmesg for UFFD/memory/VM related messages only
	sudo dmesg \| grep -iE 'userfault\|uffd\|kvm\|firecracker\|oom\|killed\|segfault\|page.fault' > /tmp/fcvm-test-logs/dmesg-filtered.log \|\| true
	- name: Upload test logs
	if: always()
	uses: actions/upload-artifact@v6
	with:
	name: test-logs-container-${{ matrix.arch }}
	path: /tmp/fcvm-test-logs/
	if-no-files-found: ignore
	retention-days: 14

	# Final summary job - runs after all test jobs
	summary:
	name: Summary
	needs: [skip-check, lint, packaging, fc-mock, host, host-root, container]
	if: always()
	runs-on: ubuntu-latest
	steps:
	- name: Report skipped
	if: needs.skip-check.outputs.skip == 'true'
	run: echo "::notice::Skipped self-hosted tests — tree SHA already passed CI on a PR branch"
	- uses: actions/checkout@v6
	- name: Download all artifacts
	uses: actions/download-artifact@v7
	with:
	path: /tmp/ci-artifacts
	- name: Analyze CI run
	run: python3 scripts/analyze_ci_vms.py /tmp/ci-artifacts

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix: --ipv6-prefix for routed mode, stale state cleanup, snapshot restore #1548

Workflow file

fix: --ipv6-prefix for routed mode, stale state cleanup, snapshot restore #1548

Uh oh!

Workflow file for this run