ci.yml

name: CI

on:
  workflow_dispatch:
    inputs:
      job:
        description: 'Which job to run'
        type: choice
        default: 'both'
        options:
          - both
          - host
          - container
  pull_request:
    branches: [main]
    paths-ignore:
      - 'kernel/**'
      - 'scripts/claude-assistant/**'
      - '.github/workflows/claude*.yml'
      - '.claude/**'
      - '*.md'
      - 'docs/**'
  push:
    branches: [main]
    paths-ignore:
      - 'kernel/**'  # When kernel changes, wait for Build Kernels workflow
      - 'scripts/claude-assistant/**'
      - '.github/workflows/claude*.yml'
      - '.claude/**'
      - '*.md'
      - 'docs/**'
  workflow_run:
    workflows: ["Build Kernels"]
    types: [completed]
    branches: [main]

# Cancel in-progress runs when a new revision is pushed
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  FUSE_BACKEND_RS: ${{ github.workspace }}/fuse-backend-rs
  FUSER: ${{ github.workspace }}/fuser
  # Cache keys for cargo tools (update versions here when bumping)
  CACHE_KEY_LINT: cargo-tools-audit-0.22.0-deny-0.18.9
  CACHE_KEY_HOST: cargo-tools-nextest-0.9.115-audit-0.22.0-deny-0.18.9

jobs:
  # Skip expensive CI jobs on main pushes when the tree SHA already passed on a PR.
  # When a PR merges via fast-forward or rebase, the merge commit has the same tree
  # SHA as the PR tip. Re-running all tests on self-hosted runners is wasteful.
  skip-check:
    name: Skip Check
    runs-on: ubuntu-latest
    outputs:
      skip: ${{ steps.check.outputs.skip }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 20
      - name: Check if tree SHA already passed CI
        id: check
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [ "${{ github.event_name }}" != "push" ]; then
            echo "skip=false" >> "$GITHUB_OUTPUT"
            echo "Not a push event, running CI"
            exit 0
          fi

          TREE_SHA=$(git rev-parse HEAD^{tree})
          echo "Current tree SHA: $TREE_SHA"

          # Check recent successful CI runs on PRs for the same tree SHA
          # Look at the last 20 commits on main to find PR branch tips
          SKIP=false
          for sha in $(git log --format=%H -20 HEAD^ 2>/dev/null || true); do
            COMMIT_TREE=$(git rev-parse "$sha^{tree}" 2>/dev/null || true)
            if [ "$COMMIT_TREE" = "$TREE_SHA" ] && [ "$sha" != "$(git rev-parse HEAD)" ]; then
              # Found a commit with the same tree — check if it passed CI
              STATUS=$(gh api "repos/${{ github.repository }}/commits/$sha/status" --jq '.state' 2>/dev/null || echo "unknown")
              echo "Commit $sha has same tree SHA, CI status: $STATUS"
              if [ "$STATUS" = "success" ]; then
                SKIP=true
                echo "Tree SHA $TREE_SHA already passed CI on commit $sha"
                break
              fi
            fi
          done

          echo "skip=$SKIP" >> "$GITHUB_OUTPUT"

  # Runner 0: Lint (default GitHub runner, no KVM needed)
  # Fast checks: fmt, clippy, audit, deny
  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          path: fcvm
      - uses: ./fcvm/.github/actions/checkout-deps
      - name: Install Rust
        uses: dtolnay/rust-toolchain@1.93.0
      - name: Get dependency SHAs
        id: deps
        run: |
          echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: fcvm -> target
          shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
      - name: Install build dependencies
        run: sudo apt-get update && sudo apt-get install -y libfuse3-dev libclang-dev clang
      - name: Cache cargo tools
        uses: actions/cache@v5
        with:
          path: ~/.cargo/bin
          key: ${{ env.CACHE_KEY_LINT }}-${{ runner.os }}-${{ runner.arch }}
      - name: Install cargo tools
        run: |
          which cargo-audit || cargo install cargo-audit@0.22.0 --locked
          which cargo-deny || cargo install cargo-deny@0.18.9 --locked
      - name: Check formatting
        working-directory: fcvm
        run: cargo fmt -p fcvm -p fuse-pipe -p fc-agent --check
      - name: Clippy
        working-directory: fcvm
        run: cargo clippy --all-targets -- -D warnings
      - name: Audit
        working-directory: fcvm
        run: cargo audit
      - name: Deny
        working-directory: fcvm
        run: cargo deny check

  # Runner 0b: Packaging (default GitHub runner, no KVM needed)
  # Verifies cargo install works without source tree access
  packaging:
    name: Packaging
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          path: fcvm
      - uses: ./fcvm/.github/actions/checkout-deps
      - name: Install Rust
        uses: dtolnay/rust-toolchain@1.93.0
      - name: Get dependency SHAs
        id: deps
        run: |
          echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: fcvm -> target
          shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
      - name: Install build dependencies
        run: sudo apt-get update && sudo apt-get install -y libfuse3-dev libclang-dev clang
      - name: Build release
        working-directory: fcvm
        run: cargo build --release -p fcvm
      - name: Test packaging
        working-directory: fcvm
        run: ./scripts/test-packaging.sh ./target/release/fcvm

  # Runner 0c: fc-mock (default GitHub runner, no KVM needed)
  # Tests fc-mock (Firecracker mock) which runs containers via podman
  # instead of real Firecracker VMs. No KVM or btrfs required.
  fc-mock:
    name: fc-mock
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          path: fcvm
      - uses: ./fcvm/.github/actions/checkout-deps
      - name: Install Rust
        uses: dtolnay/rust-toolchain@1.93.0
      - name: Get dependency SHAs
        id: deps
        run: |
          echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: fcvm -> target
          shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
      - name: Install build dependencies
        run: sudo apt-get update && sudo apt-get install -y libfuse3-dev libclang-dev clang podman
      - name: Setup podman
        run: |
          # Disable AppArmor restriction on unprivileged user namespaces
          # (needed for rootless podman on Ubuntu 24.04+)
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null || true
      - name: Cache cargo tools
        uses: actions/cache@v5
        with:
          path: ~/.cargo/bin
          key: cargo-nextest-0.9.115-${{ runner.os }}-${{ runner.arch }}
      - name: Install cargo-nextest
        run: which cargo-nextest || cargo install cargo-nextest@0.9.115 --locked
      - name: Build fc-mock
        working-directory: fcvm
        run: |
          cargo build --release -p fc-mock
          sudo install -m 755 target/release/fc-mock /usr/local/bin/fc-mock
      - name: Test fc-mock
        working-directory: fcvm
        env:
          FCVM_FIRECRACKER_BIN: /usr/local/bin/fc-mock
          RUST_LOG: "fcvm=debug,health-monitor=info,fuser=warn,fuse_backend_rs=warn,passthrough=warn"
        run: |
          cargo nextest run --release -p fcvm --profile fc-mock \
            --features privileged-tests \
            -E 'package(fcvm) & (test(/fc_mock/) | test(/state_manager/) | test(/health_monitor/) | test(/no_sudo/)) & not test(=test_fc_mock_sanity) & not test(=test_fc_mock_container_launch)'

  # Runner 1a: Host (bare metal with KVM)
  # Runs: test-unit → test-fast (quick tests, no privileged)
  # Self-hosted runners with nested virtualization (ARM64: c7g.metal, x86: c5.metal)
  host:
    name: Host-${{ matrix.arch }}
    needs: [skip-check]
    if: ${{ github.event.inputs.job != 'container' && needs.skip-check.outputs.skip != 'true' }}
    runs-on: [self-hosted, Linux, '${{ matrix.arch }}']
    strategy:
      fail-fast: false
      matrix:
        arch: [arm64, x64]
    steps:
      # Fix ownership of root-owned files from previous test runs (sudo cargo test)
      - name: Fix workspace permissions (pre-checkout)
        run: |
          sudo chown -R $USER:$USER ${{ github.workspace }}/fcvm/target 2>/dev/null || true
          # Fix cargo advisory-db permissions (can become read-only from previous runs)
          sudo chmod -R u+w ~/.cargo/advisory-db* 2>/dev/null || true
          sudo chown -R $USER:$USER ~/.cargo/advisory-db* 2>/dev/null || true
      - uses: actions/checkout@v6
        with:
          path: fcvm
      - uses: ./fcvm/.github/actions/checkout-deps
      - name: Install Rust
        run: |
          curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused --location --silent --show-error --fail https://sh.rustup.rs | sh -s -- --default-toolchain none -y
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
          # rust-toolchain.toml handles version + components; just trigger install
          rustup show active-toolchain || true
      - name: Get dependency SHAs
        id: deps
        run: |
          echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: fcvm -> target
          cache-on-failure: "true"
          shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
            iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted \
            podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
            libseccomp-dev nfs-kernel-server \
            flex bison bc libelf-dev libssl-dev
      - name: Build passt from source
        working-directory: fcvm
        run: ./scripts/build-passt.sh
      - name: Install Firecracker
        run: |
          ARCH=$(uname -m)
          FC_VERSION="1.14.0"
          curl -L -o /tmp/firecracker.tgz \
            "https://github.com/firecracker-microvm/firecracker/releases/download/v${FC_VERSION}/firecracker-v${FC_VERSION}-${ARCH}.tgz"
          sudo tar -xzf /tmp/firecracker.tgz -C /usr/local/bin --strip-components=1 \
            "release-v${FC_VERSION}-${ARCH}/firecracker-v${FC_VERSION}-${ARCH}" \
            "release-v${FC_VERSION}-${ARCH}/jailer-v${FC_VERSION}-${ARCH}"
          sudo mv "/usr/local/bin/firecracker-v${FC_VERSION}-${ARCH}" /usr/local/bin/firecracker
          sudo mv "/usr/local/bin/jailer-v${FC_VERSION}-${ARCH}" /usr/local/bin/jailer
      - name: Restore cargo tools cache
        id: cache-cargo-tools
        uses: actions/cache/restore@v5
        with:
          path: ~/.cargo/bin
          key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}
      - name: Install cargo tools
        # Skip if already cached
        # Create symlinks in /usr/local/bin so sudo can find cargo tools
        run: |
          which cargo-nextest || cargo install cargo-nextest@0.9.115 --locked
          which cargo-audit || cargo install cargo-audit@0.22.0 --locked
          which cargo-deny || cargo install cargo-deny@0.18.9 --locked
          sudo ln -sf $HOME/.cargo/bin/cargo /usr/local/bin/
          sudo ln -sf $HOME/.cargo/bin/rustc /usr/local/bin/
          sudo ln -sf $HOME/.cargo/bin/cargo-nextest /usr/local/bin/
          sudo ln -sf $HOME/.cargo/bin/cargo-audit /usr/local/bin/
          sudo ln -sf $HOME/.cargo/bin/cargo-deny /usr/local/bin/
      - name: Setup KVM and networking
        run: |
          # Print kernel version and check for nested virtualization support
          echo "=== Host Kernel Info ==="
          uname -a
          cat /proc/cmdline
          echo ""
          echo "=== KVM Capabilities ==="
          ARCH=$(uname -m)
          if [ "$ARCH" = "aarch64" ]; then
            cat /sys/module/kvm/parameters/mode 2>/dev/null || echo "kvm mode param not available"
            # Check if nested virt is available (NV2 on ARM64)
            if grep -q "kvm-arm.mode=nested" /proc/cmdline; then
              echo "✓ Nested virtualization ENABLED (kvm-arm.mode=nested)"
            else
              echo "⚠ Nested virtualization NOT enabled in kernel cmdline"
              echo "  Nested virtualization tests will fail - add kvm-arm.mode=nested to GRUB_CMDLINE_LINUX"
            fi
          else
            # x86: Check for Intel VT-x or AMD-V nested virtualization
            cat /sys/module/kvm_intel/parameters/nested 2>/dev/null && echo "Intel VT-x nested support" || true
            cat /sys/module/kvm_amd/parameters/nested 2>/dev/null && echo "AMD-V nested support" || true
            if grep -E "^Y|^1" /sys/module/kvm_intel/parameters/nested 2>/dev/null || \
               grep -E "^Y|^1" /sys/module/kvm_amd/parameters/nested 2>/dev/null; then
              echo "✓ Nested virtualization ENABLED"
            else
              echo "⚠ Nested virtualization may not be enabled"
            fi
          fi
          echo ""
          sudo chmod 666 /dev/kvm
          sudo mkdir -p /var/run/netns
          if [ ! -e /dev/userfaultfd ]; then
            sudo mknod /dev/userfaultfd c 10 126
          fi
          sudo chmod 666 /dev/userfaultfd
          sudo sysctl -w vm.unprivileged_userfaultfd=1
          # Disable AppArmor restriction on unprivileged user namespaces (needed for rootless networking on newer Ubuntu)
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null || true
          # Enable IP forwarding for all interfaces (including future ones like podman0)
          # This fixes podman container networking - without this, containers can't reach the internet
          sudo sysctl -w net.ipv4.conf.all.forwarding=1
          sudo sysctl -w net.ipv4.conf.default.forwarding=1
          # Enable IPv6 forwarding for routed networking mode (veth + IPv6 routing)
          sudo sysctl -w net.ipv6.conf.all.forwarding=1
          sudo sysctl -w net.ipv6.conf.default.forwarding=1
          # Keep accepting Router Advertisements with forwarding enabled (otherwise default route expires).
          # all/default: ensures new interfaces (pasta's virtual devices) accept RAs despite forwarding=1.
          # Per-interface: ensures the host's primary interface keeps its default route.
          sudo sysctl -w net.ipv6.conf.all.accept_ra=2
          sudo sysctl -w net.ipv6.conf.default.accept_ra=2
          DEFAULT_IFACE=$(ip route show default | awk '/dev/{print $5; exit}')
          sudo sysctl -w "net.ipv6.conf.${DEFAULT_IFACE}.accept_ra=2" 2>/dev/null || true
          # Enable FUSE allow_other for tests
          echo "user_allow_other" | sudo tee /etc/fuse.conf
          # Move podman storage to btrfs (more space for large container images)
          mkdir -p ~/.config/containers
          printf '[storage]\ndriver = "overlay"\ngraphroot = "/mnt/fcvm-btrfs/containers/storage"\n' > ~/.config/containers/storage.conf
          # Reset podman state if corrupted from previous runs
          podman system migrate || true
      - name: Create test log directory
        run: sudo rm -rf /tmp/fcvm-test-logs && mkdir -p /tmp/fcvm-test-logs
      - name: Clean test data
        working-directory: fcvm
        run: make clean-test-data || true
      - name: test-unit
        working-directory: fcvm
        run: make test-unit
      - name: setup-fcvm
        working-directory: fcvm
        run: make setup-fcvm
      - name: Build btrfs kernel
        working-directory: fcvm
        run: |
          # Regenerate config for root (setup-fcvm wrote to CI user's config dir)
          sudo ./target/release/fcvm setup --generate-config --force
          # Build btrfs kernel locally (needed for test_localhost_rootless tests)
          sudo ./target/release/fcvm setup --kernel-profile btrfs --build-kernels
      - name: Refresh KVM permissions
        run: sudo chmod 666 /dev/kvm
      - name: test-packaging-e2e
        working-directory: fcvm
        run: ./scripts/test-packaging-e2e.sh ./target/release/fcvm
      - name: test-fast
        working-directory: fcvm
        run: make test-fast
      - name: test-serve-sdk
        working-directory: fcvm
        run: |
          # SDK test requires computesdk package (sibling repo)
          # On CI, check if it's available; on dev machines, always available
          COMPUTESDK_DIR="${{ github.workspace }}/computesdk"
          if [ ! -d "$COMPUTESDK_DIR/packages/computesdk" ]; then
            echo "⚠ Skipping SDK E2E test (computesdk not checked out)"
            echo "  To run locally: cd tests && npm install && cd .. && npx tsx tests/test_serve_sdk.ts"
            exit 0
          fi
          # Install Node.js if not available
          which node || { echo "⚠ Skipping SDK test (Node.js not available)"; exit 0; }
          cd tests && npm install && cd ..
          npx tsx tests/test_serve_sdk.ts
      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: test-logs-host-${{ matrix.arch }}
          path: /tmp/fcvm-test-logs/
          if-no-files-found: ignore
          retention-days: 14
      - name: Save cargo tools cache
        if: always() && steps.cache-cargo-tools.outputs.cache-hit != 'true'
        uses: actions/cache/save@v5
        with:
          path: ~/.cargo/bin
          key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}

  # Runner 1b: Host-Root (bare metal with KVM)
  # Runs test-root with matrix for snapshot testing modes and architectures.
  # IMPORTANT: Both snapshot mode entries run on the SAME self-hosted runner to test
  # snapshot persistence across runs.
  #
  # Matrix modes:
  # - SnapshotDisabled: FCVM_NO_SNAPSHOT=1, runs once
  #   Tests the code path where snapshot feature is explicitly disabled.
  #   Verifies --no-snapshot flag and FCVM_NO_SNAPSHOT env var work correctly.
  #
  # - SnapshotEnabled: No env var, runs TWICE on same runner
  #   Run 1: Snapshot miss - creates snapshots for container images
  #   Run 2: Snapshot hit - reuses snapshots from Run 1 (should be faster)
  #   This validates the full snapshot lifecycle: create -> persist -> restore
  host-root:
    name: Host-Root-${{ matrix.arch }}-${{ matrix.mode }}
    needs: [skip-check]
    if: ${{ github.event.inputs.job != 'container' && needs.skip-check.outputs.skip != 'true' }}
    runs-on: [self-hosted, Linux, '${{ matrix.arch }}']
    strategy:
      fail-fast: false
      matrix:
        arch: [arm64, x64]
        mode: [SnapshotDisabled, SnapshotEnabled]
        include:
          - mode: SnapshotDisabled
            fcvm_no_snapshot: "1"
            test_runs: "1"
          - mode: SnapshotEnabled
            fcvm_no_snapshot: ""
            test_runs: "2"
    env:
      FCVM_NO_SNAPSHOT: ${{ matrix.fcvm_no_snapshot }}
    steps:
      # Fix ownership of root-owned files from previous test runs (sudo cargo test)
      - name: Fix workspace permissions (pre-checkout)
        run: |
          sudo chown -R $USER:$USER ${{ github.workspace }}/fcvm/target 2>/dev/null || true
          # Fix cargo advisory-db permissions (can become read-only from previous runs)
          sudo chmod -R u+w ~/.cargo/advisory-db* 2>/dev/null || true
          sudo chown -R $USER:$USER ~/.cargo/advisory-db* 2>/dev/null || true
      - uses: actions/checkout@v6
        with:
          path: fcvm
      - uses: ./fcvm/.github/actions/checkout-deps
      - name: Install Rust
        run: |
          curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused --location --silent --show-error --fail https://sh.rustup.rs | sh -s -- --default-toolchain none -y
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
          # rust-toolchain.toml handles version + components; just trigger install
          rustup show active-toolchain || true
      - name: Get dependency SHAs
        id: deps
        run: |
          echo "fuser=$(git -C fuser rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "fuse-backend-rs=$(git -C fuse-backend-rs rev-parse HEAD)" >> $GITHUB_OUTPUT
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: fcvm -> target
          cache-on-failure: "true"
          shared-key: deps-${{ steps.deps.outputs.fuser }}-${{ steps.deps.outputs.fuse-backend-rs }}
      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y fuse3 libfuse3-dev libclang-dev clang musl-tools \
            iproute2 iptables passt dnsmasq qemu-utils e2fsprogs parted \
            podman skopeo busybox-static cpio zstd autoconf automake libtool cmake \
            libseccomp-dev nfs-kernel-server \
            flex bison bc libelf-dev libssl-dev
      - name: Build passt from source
        working-directory: fcvm
        run: ./scripts/build-passt.sh
      - name: Install Firecracker
        run: |
          ARCH=$(uname -m)
          FC_VERSION="1.14.0"
          curl -L -o /tmp/firecracker.tgz \
            "https://github.com/firecracker-microvm/firecracker/releases/download/v${FC_VERSION}/firecracker-v${FC_VERSION}-${ARCH}.tgz"
          sudo tar -xzf /tmp/firecracker.tgz -C /usr/local/bin --strip-components=1 \
            "release-v${FC_VERSION}-${ARCH}/firecracker-v${FC_VERSION}-${ARCH}" \
            "release-v${FC_VERSION}-${ARCH}/jailer-v${FC_VERSION}-${ARCH}"
          sudo mv "/usr/local/bin/firecracker-v${FC_VERSION}-${ARCH}" /usr/local/bin/firecracker
          sudo mv "/usr/local/bin/jailer-v${FC_VERSION}-${ARCH}" /usr/local/bin/jailer
      - name: Restore cargo tools cache
        id: cache-cargo-tools
        uses: actions/cache/restore@v5
        with:
          path: ~/.cargo/bin
          key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}
      - name: Install cargo tools
        run: |
          which cargo-nextest || cargo install cargo-nextest@0.9.115 --locked
          sudo ln -sf $HOME/.cargo/bin/cargo /usr/local/bin/
          sudo ln -sf $HOME/.cargo/bin/rustc /usr/local/bin/
          sudo ln -sf $HOME/.cargo/bin/cargo-nextest /usr/local/bin/
      - name: Setup KVM and networking
        run: |
          sudo chmod 666 /dev/kvm
          sudo mkdir -p /var/run/netns
          if [ ! -e /dev/userfaultfd ]; then
            sudo mknod /dev/userfaultfd c 10 126
          fi
          sudo chmod 666 /dev/userfaultfd
          sudo sysctl -w vm.unprivileged_userfaultfd=1
          # Disable AppArmor restriction on unprivileged user namespaces (needed for rootless networking on newer Ubuntu)
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null || true
          # Enable IP forwarding for all interfaces (including future ones like podman0)
          # This fixes podman container networking - without this, containers can't reach the internet
          sudo sysctl -w net.ipv4.conf.all.forwarding=1
          sudo sysctl -w net.ipv4.conf.default.forwarding=1
          # Enable IPv6 forwarding for routed networking mode (veth + IPv6 routing)
          sudo sysctl -w net.ipv6.conf.all.forwarding=1
          sudo sysctl -w net.ipv6.conf.default.forwarding=1
          # Keep accepting Router Advertisements with forwarding enabled (otherwise default route expires).
          # all/default: ensures new interfaces (pasta's virtual devices) accept RAs despite forwarding=1.
          # Per-interface: ensures the host's primary interface keeps its default route.
          sudo sysctl -w net.ipv6.conf.all.accept_ra=2
          sudo sysctl -w net.ipv6.conf.default.accept_ra=2
          DEFAULT_IFACE=$(ip route show default | awk '/dev/{print $5; exit}')
          sudo sysctl -w "net.ipv6.conf.${DEFAULT_IFACE}.accept_ra=2" 2>/dev/null || true
          echo "user_allow_other" | sudo tee /etc/fuse.conf
          # Move podman storage to btrfs (more space for large container images)
          mkdir -p ~/.config/containers
          printf '[storage]\ndriver = "overlay"\ngraphroot = "/mnt/fcvm-btrfs/containers/storage"\n' > ~/.config/containers/storage.conf
          podman system migrate || true
          # Install iperf3 for network benchmarks
          sudo apt-get update && sudo apt-get install -y iperf3 || true
      - name: Create test log directory
        run: sudo rm -rf /tmp/fcvm-test-logs && mkdir -p /tmp/fcvm-test-logs
      - name: Clean test data
        working-directory: fcvm
        run: make clean-test-data || true
      - name: setup-fcvm
        working-directory: fcvm
        run: make setup-fcvm
      - name: Build nested kernel
        working-directory: fcvm
        run: |
          # Regenerate config for root (setup-fcvm wrote to CI user's config dir)
          sudo ./target/release/fcvm setup --generate-config --force
          # Build nested kernel locally (don't rely on GitHub releases)
          sudo ./target/release/fcvm setup --kernel-profile nested --build-kernels
      - name: Build btrfs kernel
        working-directory: fcvm
        run: |
          # Build btrfs kernel locally (needed for test_localhost_rootless tests)
          sudo ./target/release/fcvm setup --kernel-profile btrfs --build-kernels
      - name: Refresh KVM permissions
        run: sudo chmod 666 /dev/kvm
      - name: Allocate hugepages
        run: |
          echo 512 | sudo tee /proc/sys/vm/nr_hugepages
          echo "Allocated $(cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages) hugepages"
      - name: test-root
        working-directory: fcvm
        run: |
          SNAPSHOT_DIR="${FCVM_DATA_DIR:-/mnt/fcvm-btrfs/root}/snapshots"
          for i in $(seq 1 ${{ matrix.test_runs }}); do
            echo ""
            echo "╔══════════════════════════════════════════════════════════════════╗"
            echo "║ Test Run $i of ${{ matrix.test_runs }} (${{ matrix.mode }} mode)"
            echo "╚══════════════════════════════════════════════════════════════════╝"
            echo ""
            BEFORE=$(sudo find "$SNAPSHOT_DIR" -maxdepth 1 -type d 2>/dev/null | wc -l)
            echo "📦 Snapshot entries before run $i: $((BEFORE - 1))"
            echo ""
            make test-root
            AFTER=$(sudo find "$SNAPSHOT_DIR" -maxdepth 1 -type d 2>/dev/null | wc -l)
            echo ""
            echo "📦 Snapshot entries after run $i: $((AFTER - 1))"
            if [ "$i" -eq 2 ] && [ "${{ matrix.mode }}" = "SnapshotEnabled" ]; then
              echo "✓ Run 2 complete - snapshot hit paths tested"
            fi
          done
      - name: bench-vm
        if: matrix.mode == 'SnapshotEnabled'
        working-directory: fcvm
        run: |
          make clean-test-data || true
          make bench-vm
      - name: Capture kernel logs
        if: always()
        run: |
          sudo dmesg | grep -iE 'userfault|uffd|kvm|firecracker|oom|killed|segfault|page.fault' > /tmp/fcvm-test-logs/dmesg-filtered.log || true
      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: test-logs-host-root-${{ matrix.arch }}-${{ matrix.mode }}
          path: /tmp/fcvm-test-logs/
          if-no-files-found: ignore
          retention-days: 14
      - name: Save cargo tools cache
        if: always() && steps.cache-cargo-tools.outputs.cache-hit != 'true'
        uses: actions/cache/save@v5
        with:
          path: ~/.cargo/bin
          key: ${{ env.CACHE_KEY_HOST }}-${{ runner.os }}-${{ runner.arch }}

  # Runner 2: Container (podman)
  # Runs same tests as Host but inside a container
  # Needs KVM for VM tests (container mounts /dev/kvm)
  container:
    name: Container-${{ matrix.arch }}
    needs: [skip-check]
    if: ${{ github.event.inputs.job != 'host' && needs.skip-check.outputs.skip != 'true' }}
    runs-on: [self-hosted, Linux, '${{ matrix.arch }}']
    strategy:
      fail-fast: false
      matrix:
        arch: [arm64, x64]
    permissions:
      packages: write
    env:
      CONTAINER_ARCH: ${{ matrix.arch == 'arm64' && 'aarch64' || 'x86_64' }}
    steps:
      # Fix ownership of root-owned files from previous test runs (sudo cargo test)
      - name: Fix workspace permissions (pre-checkout)
        run: |
          sudo chown -R $USER:$USER ${{ github.workspace }}/fcvm/target 2>/dev/null || true
          # Fix cargo advisory-db permissions (can become read-only from previous runs)
          sudo chmod -R u+w ~/.cargo/advisory-db* 2>/dev/null || true
          sudo chown -R $USER:$USER ~/.cargo/advisory-db* 2>/dev/null || true
      - uses: actions/checkout@v6
        with:
          path: fcvm
      - uses: ./fcvm/.github/actions/checkout-deps
      - name: Setup KVM and rootless podman
        run: |
          sudo chmod 666 /dev/kvm
          # Enable userfaultfd syscall for snapshot cloning
          sudo sysctl -w vm.unprivileged_userfaultfd=1
          # Disable AppArmor restriction on unprivileged user namespaces (needed for rootless networking on newer Ubuntu)
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null || true
          # Enable IP forwarding for all interfaces (including future ones like podman0)
          # This fixes podman container networking - without this, containers can't reach the internet
          sudo sysctl -w net.ipv4.conf.all.forwarding=1
          sudo sysctl -w net.ipv4.conf.default.forwarding=1
          # Enable IPv6 forwarding for routed networking mode (veth + IPv6 routing)
          sudo sysctl -w net.ipv6.conf.all.forwarding=1
          sudo sysctl -w net.ipv6.conf.default.forwarding=1
          # Keep accepting Router Advertisements with forwarding enabled (otherwise default route expires).
          # all/default: ensures new interfaces (pasta's virtual devices) accept RAs despite forwarding=1.
          # Per-interface: ensures the host's primary interface keeps its default route.
          sudo sysctl -w net.ipv6.conf.all.accept_ra=2
          sudo sysctl -w net.ipv6.conf.default.accept_ra=2
          DEFAULT_IFACE=$(ip route show default | awk '/dev/{print $5; exit}')
          sudo sysctl -w "net.ipv6.conf.${DEFAULT_IFACE}.accept_ra=2" 2>/dev/null || true
          # Configure rootless podman to use cgroupfs (no systemd session on CI)
          mkdir -p ~/.config/containers
          printf '[engine]\ncgroup_manager = "cgroupfs"\nevents_logger = "file"\n' > ~/.config/containers/containers.conf
          # Reset podman state if corrupted from previous runs
          podman system migrate || true
          # Create cargo cache directory for container
          mkdir -p ${{ github.workspace }}/cargo-cache/registry ${{ github.workspace }}/cargo-cache/target
      - name: Login to ghcr.io
        run: echo "${{ github.token }}" | podman login ghcr.io -u ${{ github.actor }} --password-stdin
      - name: Cache container cargo
        uses: actions/cache@v5
        with:
          path: ${{ github.workspace }}/cargo-cache
          key: container-cargo-${{ hashFiles('fcvm/Cargo.lock') }}
          restore-keys: container-cargo-
      - name: Create test log directory
        run: sudo rm -rf /tmp/fcvm-test-logs && mkdir -p /tmp/fcvm-test-logs
      - name: Clean test data
        working-directory: fcvm
        run: make clean-test-data || true
      - name: container-test-unit
        env:
          CARGO_CACHE_DIR: ${{ github.workspace }}/cargo-cache
        working-directory: fcvm
        run: make container-test-unit
      - name: container-setup-fcvm
        env:
          CARGO_CACHE_DIR: ${{ github.workspace }}/cargo-cache
        working-directory: fcvm
        run: make container-setup-fcvm
      - name: container-test
        env:
          CARGO_CACHE_DIR: ${{ github.workspace }}/cargo-cache
        working-directory: fcvm
        run: make container-test
      - name: Capture kernel logs
        if: always()
        run: |
          # Filter dmesg for UFFD/memory/VM related messages only
          sudo dmesg | grep -iE 'userfault|uffd|kvm|firecracker|oom|killed|segfault|page.fault' > /tmp/fcvm-test-logs/dmesg-filtered.log || true
      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: test-logs-container-${{ matrix.arch }}
          path: /tmp/fcvm-test-logs/
          if-no-files-found: ignore
          retention-days: 14

  # Final summary job - runs after all test jobs
  summary:
    name: Summary
    needs: [skip-check, lint, packaging, fc-mock, host, host-root, container]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Report skipped
        if: needs.skip-check.outputs.skip == 'true'
        run: echo "::notice::Skipped self-hosted tests — tree SHA already passed CI on a PR branch"
      - uses: actions/checkout@v6
      - name: Download all artifacts
        uses: actions/download-artifact@v7
        with:
          path: /tmp/ci-artifacts
      - name: Analyze CI run
        run: python3 scripts/analyze_ci_vms.py /tmp/ci-artifacts