lambda-benchmark.yml

name: Lambda Benchmark

# Cost/runtime regression tracking on real Lambda (issues #110, #25).
#   push to main (a merge) -> run + retain to the `benchmarks` data branch and
#                             re-render the Pages charts.
#
# PRs do NOT auto-benchmark. A non-draft PR used to trigger a run, but that meant
# frequent PR pushes evicted the pending main merge from the shared concurrency
# slot (so merges got cancelled, not queued), and a PR run could also collide
# with a merge run on the single shared `process-shard-test` function. Benchmark a
# PR on demand instead with the `/benchmark` slash command (or the `benchmark`
# label) -- see lambda-benchmark-command.yml.
#
# Tier 2 (issue #25): a merge benchmark reflects the merged code only if the
# DEPLOYED worker IS that code. When the merge is "lambda-affecting" (touches
# src/zagg, deployment/aws, or pyproject) and the deploy is configured, we build
# the arm64 zips and update `process-shard-test`, then benchmark against it.
# Otherwise we benchmark the stable function (correct for orchestrator/doc-only
# changes, and the path before the AWS-side deploy is set up).
#
# The whole workflow is serialized (a single shared `test` function), so two
# runs can't deploy/benchmark over each other.

on:
  # Only run when something benchmark-relevant changed: the deployed closure
  # (src/zagg, deployment/aws, pyproject) or the matrix itself
  # (tests/data/benchmark) -- but never for docs (*.md, incl. the matrix README).
  # Doc/CI-only merges shouldn't pay for a run or add a redundant series point.
  # Use workflow_dispatch to benchmark on demand (e.g. after a workflow/runner edit).
  push:
    branches: [main]
    paths:
      - 'src/zagg/**'
      - 'deployment/aws/**'
      - 'pyproject.toml'
      - 'tests/data/benchmark/**'
      - '!**/*.md'
  workflow_dispatch:
    inputs:
      targets:
        description: Space-separated target keys (empty = all)
        required: false
        default: ""

permissions:
  contents: read

concurrency:
  group: lambda-benchmark
  cancel-in-progress: false

jobs:
  # Decide whether to deploy + benchmark the `test` function (lambda-affecting,
  # configured) or just benchmark the stable function.
  detect:
    runs-on: ubuntu-latest
    outputs:
      affecting: ${{ steps.d.outputs.affecting }}
      function_name: ${{ steps.d.outputs.function_name }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
          ref: ${{ github.sha }}
      - id: d
        env:
          DEPLOY_ROLE: ${{ vars.BENCHMARK_DEPLOY_ROLE_ARN }}
          TEST_FN: ${{ vars.BENCHMARK_TEST_FUNCTION_NAME }}
          STAGE_BUCKET: ${{ vars.BENCHMARK_TEST_STAGE_BUCKET }}
          STABLE_FN: ${{ vars.BENCHMARK_FUNCTION_NAME }}
          EVENT: ${{ github.event_name }}
          PUSH_BEFORE: ${{ github.event.before }}
          PUSH_AFTER: ${{ github.sha }}
        run: |
          set -euo pipefail
          # Tier-2 deploy is possible only when fully configured.
          deployable=true
          { [ -n "$DEPLOY_ROLE" ] && [ -n "$TEST_FN" ] && [ -n "$STAGE_BUCKET" ]; } || deployable=false

          # Does the change touch the deployed Lambda closure? (Over-detection is
          # safe -- it just redeploys; under-detection would benchmark stale code.)
          touched=false
          if [ "$EVENT" = "workflow_dispatch" ]; then
            touched=true
          else
            BASE="$PUSH_BEFORE"; HEAD="$PUSH_AFTER"
            if ! git cat-file -e "${BASE}^{commit}" 2>/dev/null; then BASE="${HEAD}~1"; fi
            # No error-masking: a git diff failure aborts the run (set -e) rather
            # than silently reading as "not affecting" -> benchmarking stale code.
            CHANGED=$(git diff --name-only "$BASE" "$HEAD")
            if echo "$CHANGED" | grep -qE '^(src/zagg/|deployment/aws/|pyproject\.toml)'; then touched=true; fi
          fi

          if [ "$deployable" = "true" ] && [ "$touched" = "true" ]; then
            echo "affecting=true" >> "$GITHUB_OUTPUT"
            echo "function_name=$TEST_FN" >> "$GITHUB_OUTPUT"
          else
            echo "affecting=false" >> "$GITHUB_OUTPUT"
            echo "function_name=$STABLE_FN" >> "$GITHUB_OUTPUT"
          fi

  # Build the arm64 zips and update process-shard-test in place (only when the
  # change is lambda-affecting). Serialized with the benchmark via the workflow
  # concurrency group above.
  deploy-test:
    needs: detect
    if: needs.detect.outputs.affecting == 'true'
    runs-on: ubuntu-24.04-arm
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@v5
        with:
          ref: ${{ github.sha }}
      - name: Build arm64 layer
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace/deployment/aws \
            quay.io/pypa/manylinux_2_28_aarch64 \
            bash -c "yum install -y zip && chmod +x build_layer.sh && ./build_layer.sh arm64"
      - name: Set up Python 3.12
        uses: actions/setup-python@v6
        with:
          python-version: '3.12'
      - name: Build arm64 function
        run: |
          chmod +x deployment/aws/build_function.sh
          deployment/aws/build_function.sh
      - name: Assume deploy role (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ vars.BENCHMARK_DEPLOY_ROLE_ARN }}
          aws-region: ${{ vars.BENCHMARK_AWS_REGION }}
      - name: Stage layer + deploy test function
        env:
          STAGE_BUCKET: ${{ vars.BENCHMARK_TEST_STAGE_BUCKET }}
          FUNCTION_NAME: ${{ vars.BENCHMARK_TEST_FUNCTION_NAME }}
          REGION: ${{ vars.BENCHMARK_AWS_REGION }}
          SHA: ${{ github.sha }}
        run: |
          set -euo pipefail
          KEY="lambda-test/${SHA}/lambda_layer_arm64.zip"
          aws s3 cp deployment/layers/lambda_layer_arm64.zip "s3://${STAGE_BUCKET}/${KEY}" --region "$REGION"
          .github/scripts/deploy_lambda.sh \
            --function "$FUNCTION_NAME" \
            --layer-bucket "$STAGE_BUCKET" \
            --layer-key "$KEY" \
            --function-zip "$(ls deployment/builds/lambda_function_arm64_*.zip)" \
            --region "$REGION"

  merge-publish:
    needs: [detect, deploy-test]
    # !cancelled() lets this run when deploy-test was skipped (non-affecting);
    # a deploy *failure* skips the benchmark (don't measure a broken deploy).
    if: >-
      !cancelled() &&
      vars.BENCHMARK_ROLE_ARN != '' &&
      needs.detect.result == 'success' &&
      (needs.deploy-test.result == 'success' || needs.deploy-test.result == 'skipped')
    runs-on: ubuntu-latest
    # contents: write to push the series + charts to the benchmarks branch.
    permissions:
      contents: write
      id-token: write
    steps:
      - name: Checkout merge commit
        uses: actions/checkout@v5

      # The runner resolves urs.earthdata.nasa.gov to an AAAA record but has no
      # usable IPv6 route, so earthaccess.login() (inside run_benchmark.py) dies
      # immediately with `[Errno 101] Network is unreachable` (issue #137).
      # Raising the getaddrinfo precedence of IPv4-mapped addresses makes glibc
      # return+try the A record first, so the login handshake takes the routable
      # IPv4 path. Runner-only config -- the shipped package and user auth are
      # untouched.
      - name: Prefer IPv4 for Earthdata login (issue #137)
        run: echo 'precedence ::ffff:0:0/96 100' | sudo tee -a /etc/gai.conf

      - name: Run benchmark
        uses: ./.github/actions/run-benchmark
        with:
          targets: ${{ github.event.inputs.targets }}
          role-arn: ${{ vars.BENCHMARK_ROLE_ARN }}
          aws-region: ${{ vars.BENCHMARK_AWS_REGION }}
          function-name: ${{ needs.detect.outputs.function_name }}
          store-prefix: ${{ vars.BENCHMARK_STORE_PREFIX }}
          edl-token: ${{ secrets.EARTHDATA_TOKEN }}
          event: merge
          commit: ${{ github.sha }}
          ref-label: ${{ github.ref_name }}

      - name: Check out benchmarks data branch
        uses: actions/checkout@v5
        with:
          ref: benchmarks
          path: _series
          fetch-depth: 0

      - name: Append series + render charts
        run: |
          set -euo pipefail
          uv sync --extra benchmark
          uv run python .github/scripts/update_series.py \
            --series _series/series.parquet --records metrics.json
          uv run python .github/scripts/plot_series.py \
            --series _series/series.parquet --out _series/site

      - name: Commit + push to benchmarks branch
        working-directory: _series
        run: |
          set -euo pipefail
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add series.parquet site
          if git diff --cached --quiet; then
            echo "no benchmark changes to commit"
          else
            git commit -m "benchmark: ${{ github.sha }}"
            git push origin benchmarks
          fi