bench.yml

name: Benchmark

on:
  push:
    branches: [main]
    paths:
      - 'src/**'
      - 'Cargo.toml'
      - '.github/workflows/bench.yml'
  pull_request:
    paths:
      - '.github/workflows/bench.yml'
  schedule:
    - cron: '0 7 * * 1'
  workflow_dispatch:

permissions:
  contents: write

concurrency:
  group: bench-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

env:
  FIXTURE_URL: https://github.com/Arkptz/mitm2openapi/releases/download/bench-fixtures-v1/big.flow
  FIXTURE_SHA256_URL: https://github.com/Arkptz/mitm2openapi/releases/download/bench-fixtures-v1/big.flow.sha256
  FIXTURE_PREFIX: http://localhost:18000

jobs:
  bench:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.head_ref || github.ref_name }}

      - uses: dtolnay/rust-toolchain@stable

      - name: Cache cargo
        uses: actions/cache@v6
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: bench-cargo-${{ hashFiles('Cargo.lock') }}

      - name: Install hyperfine
        run: |
          wget -q https://github.com/sharkdp/hyperfine/releases/download/v1.18.0/hyperfine_1.18.0_amd64.deb
          sudo dpkg -i hyperfine_1.18.0_amd64.deb
          hyperfine --version

      - name: Install Python mitmproxy2swagger
        run: |
          pip install --user "mitmproxy2swagger==0.15.0"
          echo "$HOME/.local/bin" >> $GITHUB_PATH
          python -m pip show mitmproxy2swagger | head -3

      - name: Build Rust release
        run: cargo build --release

      - name: Download benchmark fixture
        run: |
          mkdir -p /tmp/bench
          wget -q "$FIXTURE_URL" -O /tmp/bench/big.flow
          wget -q "$FIXTURE_SHA256_URL" -O /tmp/bench/big.flow.sha256
          echo "$(cat /tmp/bench/big.flow.sha256)  /tmp/bench/big.flow" | sha256sum -c -
          ls -lh /tmp/bench/big.flow

      - name: Run benchmarks
        id: bench
        run: |
          RUST=$(pwd)/target/release/mitm2openapi
          OUT=/tmp/bench/out
          mkdir -p $OUT

          run_py() {
            local flow=$1 prefix=$2
            cp "$flow" $OUT/in.flow
            mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$prefix" -f flow > /dev/null 2>&1
            sed -i 's/^- ignore:/- /' $OUT/py.yaml
            mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$prefix" -f flow > /dev/null 2>&1
          }
          run_rs() {
            local flow=$1 prefix=$2
            rm -f $OUT/tpl.yaml
            "$RUST" discover -i "$flow" -o $OUT/tpl.yaml -p "$prefix" > /dev/null 2>&1
            sed -i 's|^- ignore:|- |' $OUT/tpl.yaml
            "$RUST" generate -i "$flow" -t $OUT/tpl.yaml -o $OUT/rs.yaml -p "$prefix" > /dev/null 2>&1
          }
          export -f run_py run_rs
          export RUST OUT

          hyperfine --warmup 2 --runs 5 --shell=bash \
            --export-json $OUT/synthetic.json \
            --export-markdown $OUT/synthetic.md \
            --command-name "Python mitmproxy2swagger" "run_py '/tmp/bench/big.flow' '$FIXTURE_PREFIX'" \
            --command-name "Rust mitm2openapi"         "run_rs '/tmp/bench/big.flow' '$FIXTURE_PREFIX'"

          echo "## Peak RSS" > $OUT/rss.md
          echo "| Tool | RSS |" >> $OUT/rss.md
          echo "|------|----:|" >> $OUT/rss.md
          cp /tmp/bench/big.flow $OUT/in.flow
          mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$FIXTURE_PREFIX" -f flow > /dev/null 2>&1
          py_rss=$(/usr/bin/time -f '%M' mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$FIXTURE_PREFIX" -f flow 2>&1 > /dev/null | tail -1)
          rm -f $OUT/tpl.yaml
          rs_discover=$(/usr/bin/time -f '%M' "$RUST" discover -i /tmp/bench/big.flow -o $OUT/tpl.yaml -p "$FIXTURE_PREFIX" 2>&1 > /dev/null | tail -1)
          sed -i 's|^- ignore:|- |' $OUT/tpl.yaml
          rs_generate=$(/usr/bin/time -f '%M' "$RUST" generate -i /tmp/bench/big.flow -t $OUT/tpl.yaml -o $OUT/rs.yaml -p "$FIXTURE_PREFIX" 2>&1 > /dev/null | tail -1)
          rs_peak=$(( rs_discover > rs_generate ? rs_discover : rs_generate ))
          printf "| Python mitmproxy2swagger | %s KB |\n" "$py_rss" >> $OUT/rss.md
          printf "| Rust mitm2openapi        | %s KB |\n" "$rs_peak" >> $OUT/rss.md

          {
            echo "# Benchmark results"
            echo
            echo "_Run: $(date -u '+%Y-%m-%d %H:%M UTC'), commit \`${GITHUB_SHA:0:8}\`, runner: $(uname -sr)_"
            echo
            echo "Fixture: 89 MB, 40k requests across 8 endpoint shapes (\`bench-fixtures-v1\`)."
            echo
            echo "## Timing"
            echo
            cat $OUT/synthetic.md
            echo
            cat $OUT/rss.md
          } > $OUT/summary.md

          cat $OUT/summary.md
          cp $OUT/summary.md bench-results.md

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        with:
          name: bench-results-${{ github.sha }}
          path: |
            /tmp/bench/out/*.json
            /tmp/bench/out/*.md
            bench-results.md
          retention-days: 90

      - name: Update docs/benchmarks.md
        if: >-
          github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' ||
          (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
        run: |
          mkdir -p docs
          {
            echo "# Benchmarks"
            echo
            echo "Generated by the [benchmark workflow](.github/workflows/bench.yml)."
            echo
            cat bench-results.md
          } > docs/benchmarks.md
          git diff --quiet docs/benchmarks.md && { echo "no change"; exit 0; }
          git config user.name arkptz
          git config user.email arkptz@gmail.com
          git add docs/benchmarks.md
          git commit -m "docs(bench): refresh benchmark results"
          BRANCH="${{ github.head_ref || github.ref_name }}"
          # demo-gif.yml pushes to the same branch and can land first.
          git pull --rebase --autostash origin "$BRANCH"
          git push origin HEAD:"$BRANCH"