e2e Test - DSV4 fp4 B300 vLLM DSpark smoke (retry 2, sched budget + resume fix) #3984
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Tests | |
| run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || github.event.inputs.generate-cli-command }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: true | |
| type: string | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| workflow_call: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: true | |
| type: string | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| jobs: | |
| get-jobs: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| single-node-config: ${{ steps.get-jobs.outputs.single-node-config }} | |
| multi-node-config: ${{ steps.get-jobs.outputs.multi-node-config }} | |
| eval-config: ${{ steps.get-jobs.outputs.eval-config }} | |
| multi-node-eval-config: ${{ steps.get-jobs.outputs.multi-node-eval-config }} | |
| agentic-config: ${{ steps.get-jobs.outputs.agentic-config }} | |
| multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }} | |
| steps: | |
| - name: Checkout code (ref) | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ inputs.ref }} | |
| - name: Checkout code (default) | |
| if: ${{ !inputs.ref || inputs.ref == '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.sha }} | |
| - id: get-jobs | |
| run: | | |
| pip install pydantic | |
| CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \ | |
| ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}) | |
| AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))") | |
| MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))") | |
| SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") | |
| MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") | |
| EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))") | |
| MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))") | |
| echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT | |
| echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT | |
| echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT | |
| echo "multi-node-config=$MULTI" >> $GITHUB_OUTPUT | |
| echo "eval-config=$EVALS" >> $GITHUB_OUTPUT | |
| echo "multi-node-eval-config=$MULTI_EVAL" >> $GITHUB_OUTPUT | |
| test-sweep-multi-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: false | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node eval / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-eval-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: true | |
| eval-only: true | |
| eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} | |
| ref: ${{ inputs.ref }} | |
| test-sweep-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: agentic / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| runner: ${{ matrix.config.runner }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| tp: ${{ matrix.config.tp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config.kv-offload-backend }} | |
| total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} | |
| duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| spec-decoding: 'none' | |
| disagg: 'false' | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node agentic / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| runner: ${{ matrix.config.runner }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| conc: ${{ matrix.config.conc[0] }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config.kv-offload-backend }} | |
| duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-single-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.single-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: single-node / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.single-node-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| tp: ${{ matrix.config.tp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: false | |
| ref: ${{ inputs.ref }} | |
| test-sweep-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: eval / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.eval-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| tp: ${{ matrix.config.tp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: true | |
| eval-only: true | |
| ref: ${{ inputs.ref }} | |
| collect-results: | |
| needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] | |
| if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-results.yml | |
| secrets: inherit | |
| with: | |
| result-prefix: "bmk" | |
| collect-evals: | |
| needs: [test-sweep-evals, test-sweep-multi-node-evals] | |
| if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-evals.yml | |
| secrets: inherit | |
| calc-success-rate: | |
| needs: [collect-results, collect-evals] | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RESULTS_DIR: "results/" | |
| STATS_FILENAME: "run_stats" | |
| GITHUB_TOKEN: ${{ secrets.REPO_PAT }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| token: ${{ secrets.REPO_PAT }} | |
| fetch-depth: 0 | |
| - name: Download results artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ${{ env.RESULTS_DIR }} | |
| pattern: results_* | |
| - name: Install python dependencies | |
| run: pip install PyGithub | |
| - name: Calculate success rate | |
| run: python3 utils/calc_success_rate.py $STATS_FILENAME | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: "run-stats" | |
| path: ${{ env.STATS_FILENAME }}.json | |
| trigger-agentic-ingest: | |
| needs: | |
| [ | |
| test-sweep-agentic, | |
| test-sweep-multi-node-agentic, | |
| calc-success-rate, | |
| ] | |
| if: >- | |
| always() && | |
| github.event_name == 'workflow_dispatch' && | |
| needs.calc-success-rate.result == 'success' && | |
| ( | |
| needs.test-sweep-agentic.result == 'success' || | |
| needs.test-sweep-multi-node-agentic.result == 'success' | |
| ) && | |
| ( | |
| needs.test-sweep-agentic.result == 'success' || | |
| needs.test-sweep-agentic.result == 'skipped' | |
| ) && | |
| ( | |
| needs.test-sweep-multi-node-agentic.result == 'success' || | |
| needs.test-sweep-multi-node-agentic.result == 'skipped' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Trigger agentic database ingest | |
| run: | | |
| curl -sSf -X POST \ | |
| -H "Authorization: Bearer ${{ secrets.INFX_FRONTEND_PAT }}" \ | |
| -H "Accept: application/vnd.github+v3+json" \ | |
| https://api.github.com/repos/SemiAnalysisAI/InferenceX-app/actions/workflows/ingest-agentic-results.yml/dispatches \ | |
| -d '{ | |
| "ref": "feat/agentx", | |
| "inputs": { | |
| "run-id": "${{ github.run_id }}", | |
| "run-attempt": "${{ github.run_attempt }}", | |
| "database-target": "production" | |
| } | |
| }' |