[WIP] add SWE-bench Lite accuracy eval / 添加 SWE-bench Lite 准确率评估

.github/workflows/benchmark-tmpl.yml

@@ -104,6 +104,16 @@ on:
         required: false
         type: string
         default: '3600'
+      eval-limit:
+        description: "Limit eval to first N instances (smoke tests); empty = full set"
+        required: false
+        type: string
+        default: ""
+      swebench-gen-mode:
+        description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
+        required: false
+        type: string
+        default: ""
 env:
   RANDOM_RANGE_RATIO: 0.8
   HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
@@ -136,10 +146,22 @@ env:
   KV_OFFLOAD_BACKEND: ${{ inputs.kv-offload-backend }}
   TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }}
   DURATION: ${{ inputs.duration }}
+  EVAL_LIMIT: ${{ inputs.eval-limit }}
+  SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }}
   AIPERF_FAILED_REQUEST_THRESHOLD: '0.10'
   RESULT_DIR: /workspace/results
   PYTHONDONTWRITEBYTECODE: '1'
   PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache
+  # SWE-bench scoring runs on Modal (GPU runners have no Docker); creds are
+  # bootstrapped into ~/.modal.toml by benchmark_lib.sh:_ensure_modal_credentials.
+  SWEBENCH_USE_MODAL: 'true'
+  MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+  MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+  # Known-bad b300 nodes, excluded until infra repairs them (salloc honors
+  # this env natively). b300-005: NCCL/CUDA init death (jobs 30508/30548).
+  # b300-006: wedged nvidia driver, container-cli rpc timeout (run
+  # 28958779201). Remove entries as nodes are fixed.
+  SALLOC_EXCLUDE: 'b300-005,b300-006'
 
 permissions:
   contents: read
@@ -296,6 +318,10 @@ jobs:
             meta_env.json
             results*.json
             sample*.jsonl
+            agent_preds.json
+            predictions.jsonl
+            swebench_report_*.json
+            *.traj*
           if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}
 
       - name: Verify eval scores
@@ -309,6 +335,7 @@ jobs:
           # Remove any eval results JSONs that were moved into workspace
           rm -f results*.json || true
           rm -f sample*.jsonl || true
+          rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
 
       - name: Resource cleanup (post-run)
         if: always()

.github/workflows/e2e-tests.yml

@@ -21,6 +21,16 @@
                 required: false
                 type: string
                 default: ""
+            eval-limit:
+                description: "Eval instance count: empty = 50-slice CI default (swebench); N = first-N; full = whole split"
+                required: false
+                type: string
+                default: ""
+            swebench-gen-mode:
+                description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
+                required: false
+                type: string
+                default: ""
     workflow_call:
         inputs:
             generate-cli-command:
@@ -40,6 +50,16 @@
                 required: false
                 type: string
                 default: ""
+            eval-limit:
+                description: "Eval instance count: empty = 50-slice CI default (swebench); N = first-N; full = whole split"
+                required: false
+                type: string
+                default: ""
+            swebench-gen-mode:
+                description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
+                required: false
+                type: string
+                default: ""
 
 jobs:
     get-jobs:
@@ -50,6 +70,7 @@
             eval-config: ${{ steps.get-jobs.outputs.eval-config }}
             multi-node-eval-config: ${{ steps.get-jobs.outputs.multi-node-eval-config }}
             agentic-config: ${{ steps.get-jobs.outputs.agentic-config }}
+            agentic-eval-config: ${{ steps.get-jobs.outputs.agentic-eval-config }}
             multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }}
         steps:
             - name: Checkout code (ref)
@@ -69,13 +90,15 @@
                   pip install pydantic
                   CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \
                     ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }})
-                  AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))")
+                  AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))")
+                  AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))")
                   MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))")
                   SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))")
                   MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))")
                   EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))")
                   MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))")
                   echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT
+                  echo "agentic-eval-config=$AGENTIC_EVAL" >> $GITHUB_OUTPUT
                   echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT
                   echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT
                   echo "multi-node-config=$MULTI" >> $GITHUB_OUTPUT
@@ -216,7 +239,46 @@
             scenario-type: agentic-coding
             ref: ${{ inputs.ref }}
 
+    test-sweep-agentic-evals:
+        needs: get-jobs
+        if: ${{ needs.get-jobs.outputs.agentic-eval-config != '[]' }}
+        uses: ./.github/workflows/benchmark-tmpl.yml
+        name: agentic eval /
+        strategy:
+            fail-fast: false
+            matrix:
+                config: ${{ fromJson(needs.get-jobs.outputs.agentic-eval-config) }}
+        secrets: inherit
+        with:
+            exp-name: ${{ matrix.config.exp-name }}
+            runner: ${{ matrix.config.runner }}
+            image: ${{ matrix.config.image }}
+            model: ${{ matrix.config.model }}
+            model-prefix: ${{ matrix.config.model-prefix }}
+            framework: ${{ matrix.config.framework }}
+            precision: ${{ matrix.config.precision }}
+            tp: ${{ matrix.config.tp }}
+            ep: ${{ matrix.config.ep }}
+            dp-attn: ${{ matrix.config.dp-attn }}
+            conc: ${{ matrix.config.conc }}
+            kv-offloading: ${{ matrix.config.kv-offloading }}
+            kv-offload-backend: ${{ matrix.config.kv-offload-backend }}
+            total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }}
+            duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }}
+            isl: '0'
+            osl: '0'
+            max-model-len: '0'
+            spec-decoding: 'none'
+            disagg: 'false'
+            # scenario-type agentic-coding => run_eval auto-selects swebench.
+            run-eval: true
+            eval-only: true
+            eval-limit: ${{ inputs.eval-limit }}
+            swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
+            scenario-type: agentic-coding
+            ref: ${{ inputs.ref }}
+
     test-sweep-multi-node-agentic:
         needs: get-jobs
         if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '[]' }}
         uses: ./.github/workflows/benchmark-multinode-tmpl.yml
@@ -331,6 +393,7 @@
             disagg: ${{ matrix.config.disagg }}
             run-eval: true
             eval-only: true
+            eval-limit: ${{ inputs.eval-limit }}
             ref: ${{ inputs.ref }}
 
     collect-results:
@@ -342,12 +405,12 @@
             result-prefix: "bmk"
 
     collect-evals:
-        needs: [test-sweep-evals, test-sweep-multi-node-evals]
-        if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped') }}
+        needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals]
+        if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped') }}
         uses: ./.github/workflows/collect-evals.yml
         secrets: inherit
 
     calc-success-rate:
         needs: [collect-results, collect-evals]
         if: ${{ always() }}
         runs-on: ubuntu-latest