e2e.yml

name: E2E Tests

on:
  workflow_dispatch:
    inputs:
      tests:
        description: 'E2E test filter (pytest -k expression, leave empty to run all e2e tests)'
        required: false
        default: ''

permissions:
  contents: read

env:
  PYTHON_VERSION: '3.11'

jobs:
  # Note: there's intentionally no unit-tests job here. ci.yml already runs the
  # full quality matrix on every PR and push:main; duplicating it under
  # workflow_dispatch would just be a third copy of the same install + pytest
  # setup. If you need unit results outside of CI, run ci.yml manually or
  # rerun a previous run.
  e2e:
    name: E2E
    runs-on: ubuntu-latest
    timeout-minutes: 45

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Install dependencies
        run: make dev

      - name: Run E2E tests
        env:
          RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
          FLASH_SDK_GIT_REF: ${{ github.sha }}
        run: |
          uv run pytest e2e/ \
            ${{ inputs.tests != '' && format('-k "{0}"', inputs.tests) || '' }} \
            -v \
            --timeout=0 \
            --no-cov \
            -p no:xdist \
            --override-ini="addopts=" \
            --junitxml=e2e-results.xml \
            -s

      - name: Check at least one test ran
        if: always()
        run: |
          python - <<'EOF'
          import xml.etree.ElementTree as ET, sys
          try:
              tree = ET.parse("e2e-results.xml")
              root = tree.getroot()
              if root.tag == "testsuites":
                  tests = sum(int(s.attrib.get("tests", 0)) for s in root.findall("testsuite"))
              else:
                  tests = int(root.attrib.get("tests", 0))
              print(f"Tests run: {tests}")
              if tests == 0:
                  print("ERROR: 0 tests ran -- check test filter or test collection")
                  sys.exit(1)
          except FileNotFoundError:
              print("ERROR: e2e-results.xml not found -- pytest did not run")
              sys.exit(1)
          EOF

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: e2e-results
          path: e2e-results.xml

      - name: Write summary
        if: always()
        run: |
          python - <<'EOF'
          import xml.etree.ElementTree as ET, os, sys

          summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
          out = open(summary_file, "a") if summary_file else sys.stdout

          try:
              root = ET.parse("e2e-results.xml").getroot()
              suites = root.findall("testsuite") if root.tag == "testsuites" else [root]
              total    = sum(int(s.attrib.get("tests",    0)) for s in suites)
              failures = sum(int(s.attrib.get("failures", 0)) + int(s.attrib.get("errors", 0)) for s in suites)
              duration = sum(float(s.attrib.get("time",   0)) for s in suites)
              failed_names = [
                  tc.get("classname", "") + "::" + tc.get("name", "")
                  for s in suites
                  for tc in s.findall("testcase")
                  if tc.find("failure") is not None or tc.find("error") is not None
              ]
          except FileNotFoundError:
              total, failures, duration, failed_names = None, None, None, []

          if total is None:
              status = ":x: Did not run"
          elif total == 0:
              status = ":warning: No tests ran"
          elif failures == 0:
              status = ":white_check_mark: Passed"
          else:
              status = ":x: Failed"

          passed = (total - failures) if total is not None else None

          def _num(v):
              return str(v) if v is not None else "-"
          def _dur(v):
              return f"{v:.1f}s" if v is not None else "-"

          print("# E2E Results\n", file=out)
          print("| Status | Passed | Failed | Total | Duration |", file=out)
          print("|---|---|---|---|---|", file=out)
          print(f"| {status} | {_num(passed)} | {_num(failures)} | {_num(total)} | {_dur(duration)} |",
                file=out)
          print("", file=out)

          if failed_names:
              print("## Failed Tests\n", file=out)
              print("| Test |", file=out)
              print("|---|", file=out)
              for name in failed_names:
                  print(f"| `{name}` |", file=out)
          EOF