diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index b2219144f1..985065df99 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -6,6 +6,19 @@ on: runner: required: true type: string + priority: + description: "Higher-is-sooner CI priority score" + required: true + type: string + queue-token: + description: "One-shot queue token generated for this job" + required: true + type: string + skip-queue-pr: + description: "Requested skip_queue PR number" + required: false + type: string + default: '' image: required: true type: string @@ -231,10 +244,30 @@ permissions: jobs: benchmark: - runs-on: ${{ inputs.runner }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + ) || + format( + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) + ) || + format('[{0}]', toJSON(inputs.runner)) + ) }} timeout-minutes: 480 name: >- - ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} + p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} ${{ inputs.prefill-num-worker }}P (TP${{ inputs.prefill-tp }}${{ inputs.prefill-pp != '' && inputs.prefill-pp != '1' && format('/PP{0}', inputs.prefill-pp) || '' }}${{ inputs.prefill-dcp-size != '' && inputs.prefill-dcp-size != '1' && format('/DCP{0}', inputs.prefill-dcp-size) || '' }}${{ inputs.prefill-pcp-size != '' && inputs.prefill-pcp-size != '1' && format('/PCP{0}', inputs.prefill-pcp-size) || '' }}${{ inputs.prefill-ep != '' && inputs.prefill-ep != '1' && format('/EP{0}', inputs.prefill-ep) || '' }}${{ inputs.prefill-dp-attn == 'true' && '/DPA' || '' }}) x ${{ inputs.decode-num-worker }}D (TP${{ inputs.decode-tp }}${{ inputs.decode-pp != '' && inputs.decode-pp != '1' && format('/PP{0}', inputs.decode-pp) || '' }}${{ inputs.decode-dcp-size != '' && inputs.decode-dcp-size != '1' && format('/DCP{0}', inputs.decode-dcp-size) || '' }}${{ inputs.decode-pcp-size != '' && inputs.decode-pcp-size != '1' && format('/PCP{0}', inputs.decode-pcp-size) || '' }}${{ inputs.decode-ep != '' && inputs.decode-ep != '1' && format('/EP{0}', inputs.decode-ep) || '' }}${{ inputs.decode-dp-attn == 'true' && '/DPA' || '' }}) ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 241dafc4a4..f3cef5413f 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -5,6 +5,19 @@ on: runner: required: true type: string + priority: + description: "Higher-is-sooner CI priority score" + required: true + type: string + queue-token: + description: "One-shot queue token generated for this job" + required: true + type: string + skip-queue-pr: + description: "Requested skip_queue PR number" + required: false + type: string + default: '' image: required: true type: string @@ -161,10 +174,30 @@ permissions: jobs: benchmark: - runs-on: ${{ inputs.runner }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) + ) || + format( + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) + ) || + format('[{0}]', toJSON(inputs.runner)) + ) }} timeout-minutes: 500 name: >- - ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} + p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} TP${{ inputs.tp }}${{ inputs.pp != '' && inputs.pp != '1' && format('/PP{0}', inputs.pp) || '' }}${{ inputs.dcp-size != '' && inputs.dcp-size != '1' && format('/DCP{0}', inputs.dcp-size) || '' }}${{ inputs.pcp-size != '' && inputs.pcp-size != '1' && format('/PCP{0}', inputs.pcp-size) || '' }}${{ inputs.ep != '' && inputs.ep != '1' && format('/EP{0}', inputs.ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }} ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }} diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index f3688998c0..0d685e0506 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -34,9 +34,15 @@ jobs: outputs: matrix: ${{ steps.gen.outputs.matrix }} n: ${{ steps.gen.outputs.n }} + priority: ${{ steps.score.outputs.priority }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0 with: { clean: true, persist-credentials: false } + - id: score + run: | + scored=$(printf '%s' '[{"runner":"collectivex","framework":"collectivex"}]' | + python3 utils/ci_priority.py) + echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" - id: gen working-directory: experimental/CollectiveX env: @@ -44,6 +50,8 @@ jobs: INPUT_ONLY_SKU: ${{ inputs.only_sku }} INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }} INPUT_EP_SIZES: ${{ inputs.ep_sizes }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -euo pipefail args=(--backend "$INPUT_BACKEND") @@ -52,12 +60,18 @@ jobs: [ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES") python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null python3 - "$GITHUB_OUTPUT" <<'PY' + import hashlib import json + import os import pathlib import sys matrix = json.loads(pathlib.Path("matrix_full.json").read_text()) cells = matrix["include"] + for index, cell in enumerate(cells): + canonical = json.dumps(cell, sort_keys=True, separators=(",", ":")) + material = f"{os.environ['RUN_ID']}:{os.environ['RUN_ATTEMPT']}:{index}:{canonical}".encode() + cell["queue-token"] = hashlib.sha256(material).hexdigest()[:32] slim = {"include": cells} with open(sys.argv[1], "a", encoding="utf-8") as output: output.write(f"matrix={json.dumps(slim, separators=(',', ':'))}\n") @@ -83,7 +97,22 @@ jobs: # first jobs under this cap spread over pools instead of queuing on one. max-parallel: 10 matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} - runs-on: ${{ matrix.sku }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted",{0},{1},{2}]', + toJSON(matrix.sku), + toJSON(format( + 'ci-job-{0}-{1}', + needs.setup.outputs.priority, + matrix.queue-token + )), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) || + format('[{0}]', toJSON(matrix.sku)) + ) }} + name: p${{ needs.setup.outputs.priority }} | ${{ matrix.sku }} ${{ matrix.backend }} shard ${{ matrix.id }} timeout-minutes: 350 env: COLLX_BENCH: ${{ matrix.backend }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 51c68e1d85..9e941df5e9 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -63,18 +63,40 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.sha }} + - name: Checkout priority scheduler tooling + if: ${{ inputs.ref && inputs.ref != '' }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.workflow_sha }} + path: .ci-priority + persist-credentials: false + - id: get-jobs + env: + PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} run: | pip install pydantic CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \ ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}) - AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))") - MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))") - SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") - MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") - EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))") - MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))") + PRIORITY_ROOT="${GITHUB_WORKSPACE}" + if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then + PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" + fi + score_matrix() { + local family="$1" + python3 "${PRIORITY_ROOT}/utils/ci_priority.py" \ + --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ + --event-name "${{ github.event_name }}" \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}:${family}" \ + --labels-json "$PR_LABELS" + } + AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))" | score_matrix agentic) + MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))" | score_matrix multi-agentic) + SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) + MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) + EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) + MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval) echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT @@ -97,6 +119,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -146,6 +170,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -195,6 +221,8 @@ jobs: with: exp-name: ${{ matrix.config.exp-name }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -239,6 +267,8 @@ jobs: osl: '0' max-model-len: '0' runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -292,6 +322,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -327,6 +359,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} diff --git a/.github/workflows/priority-scheduler-canary.yml b/.github/workflows/priority-scheduler-canary.yml new file mode 100644 index 0000000000..5a62b8b980 --- /dev/null +++ b/.github/workflows/priority-scheduler-canary.yml @@ -0,0 +1,121 @@ +name: Priority Scheduler Canary +run-name: Priority Scheduler Canary - ${{ inputs.runner }} at p${{ inputs.score }} + +on: + workflow_dispatch: + inputs: + runner: + description: "Runner label to test" + required: true + type: choice + default: b200 + options: + - b200 + - b300 + - h100 + - h200 + - mi355x + score: + description: "Numeric priority score" + required: true + type: string + default: "1.000" + +permissions: + contents: read + actions: write + +jobs: + prepare: + name: Prepare one-shot scheduling label + runs-on: ubuntu-latest + outputs: + job-label-base: ${{ steps.labels.outputs.job-label-base }} + steps: + - id: labels + name: Generate unique canary label + shell: python + env: + RUNNER_LABEL: ${{ inputs.runner }} + PRIORITY_SCORE: ${{ inputs.score }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + import hashlib + import json + import os + import re + + score = os.environ["PRIORITY_SCORE"] + if re.fullmatch(r"[0-9]+(?:\.[0-9]+)?", score) is None: + raise SystemExit(f"Invalid numeric priority: {score}") + + material = f"{os.environ['RUN_ID']}:{os.environ['RUN_ATTEMPT']}:canary" + token = hashlib.sha256(material.encode()).hexdigest()[:32] + job_label_base = f"ci-job-{score}-{token}" + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"job-label-base={job_label_base}\n") + + canary: + name: Verify priority dispatch + needs: prepare + runs-on: >- + ${{ fromJSON( + github.run_attempt == 1 && + format( + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(needs.prepare.outputs.job-label-base), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) || + '["ubuntu-latest"]' + ) }} + timeout-minutes: 10 + steps: + - name: Reject partial rerun + if: github.run_attempt != 1 + run: | + echo "Priority canaries are one-shot; dispatch a fresh workflow run." >&2 + exit 1 + - name: Confirm assignment + if: github.run_attempt == 1 + env: + JOB_LABEL: ${{ needs.prepare.outputs.job-label-base }} + ATTEMPT_LABEL: ${{ format('ci-attempt-{0}', github.run_attempt) }} + RUNNER_LABEL: ${{ inputs.runner }} + run: | + echo "Controller dispatched $JOB_LABEL $ATTEMPT_LABEL to $RUNNER_LABEL runner $RUNNER_NAME" + { + echo "### Priority scheduler canary passed" + echo "- Job label: \`$JOB_LABEL\`" + echo "- Attempt label: \`$ATTEMPT_LABEL\`" + echo "- Runner class: \`$RUNNER_LABEL\`" + echo "- Runner: \`$RUNNER_NAME\`" + } >> "$GITHUB_STEP_SUMMARY" + + watchdog: + name: Enforce canary queue deadline + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 12 + steps: + - name: Cancel if the canary is still queued after ten minutes + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for _ in {1..40}; do + status=$(gh api \ + "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=latest" \ + --jq '.jobs[] | select(.name == "Verify priority dispatch") | .status') + if [ "$status" = "in_progress" ] || [ "$status" = "completed" ]; then + exit 0 + fi + sleep 15 + done + + echo "Priority canary remained queued for ten minutes; cancelling the run." >&2 + gh api --method POST \ + "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/cancel" + exit 1 diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 10eb631d03..20d5895046 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -49,6 +49,14 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.ref || github.sha }} + - name: Checkout priority scheduler tooling + if: ${{ inputs.ref && inputs.ref != '' }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.workflow_sha }} + path: .ci-priority + persist-credentials: false + - id: gen name: Generate matrix via script @@ -56,6 +64,16 @@ jobs: pip install pydantic CLI_ARGS="test-config --config-files ${{ inputs.config-file }} --config-keys ${{ inputs.config-key }} --conc ${{ inputs.conc }}" CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py $CLI_ARGS) + PRIORITY_ROOT="${GITHUB_WORKSPACE}" + if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then + PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" + fi + CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ + "${PRIORITY_ROOT}/utils/ci_priority.py" \ + --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ + --event-name "${{ github.event_name }}" \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}" \ + --labels-json '${{ inputs.moe-debug && '["ci-patchwork"]' || '[]' }}') echo "raw=$CONFIG_JSON" >> $GITHUB_OUTPUT - id: filter @@ -99,9 +117,23 @@ jobs: matrix: config: ${{ fromJson(needs.get-jobs.outputs.filtered-matrix) }} name: >- - ${{ matrix.config.model-prefix }} ${{ matrix.config.precision }} ${{ matrix.config.runner }} ${{ matrix.config.framework }} + p${{ matrix.config.priority }} | ${{ matrix.config.model-prefix }} ${{ matrix.config.precision }} ${{ matrix.config.runner }} ${{ matrix.config.framework }} TP${{ matrix.config.tp }}${{ matrix.config.pp != '' && matrix.config.pp != '1' && format('/PP{0}', matrix.config.pp) || '' }}${{ matrix.config.dcp-size != '' && matrix.config.dcp-size != '1' && format('/DCP{0}', matrix.config.dcp-size) || '' }}${{ matrix.config.pcp-size != '' && matrix.config.pcp-size != '1' && format('/PCP{0}', matrix.config.pcp-size) || '' }} c${{ matrix.config.conc }} - runs-on: ${{ matrix.config.runner }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted",{0},{1},{2}]', + toJSON(matrix.config.runner), + toJSON(format( + 'ci-job-{0}-{1}', + matrix.config.priority, + matrix.config['queue-token'] + )), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) || + format('[{0}]', toJSON(matrix.config.runner)) + ) }} env: EXP_NAME: ${{ matrix.config.exp-name }} MODEL: ${{ matrix.config.model }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b983982af8..e8933634ce 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -13,6 +13,11 @@ concurrency: github.event.label.name != 'full-sweep-fail-fast-no-canary' && github.event.label.name != 'all-evals' && github.event.label.name != 'evals-only' && + github.event.label.name != 'skip_queue' && + github.event.label.name != 'ci-patchwork' && + github.event.label.name != 'engine-patch' && + github.event.label.name != 'ci-patchwork-waived' && + github.event.label.name != 'ci-checklist-complete' && github.run_id || 'active' }} @@ -49,7 +54,12 @@ jobs: github.event.label.name == 'full-sweep-fail-fast' || github.event.label.name == 'full-sweep-fail-fast-no-canary' || github.event.label.name == 'all-evals' || - github.event.label.name == 'evals-only' + github.event.label.name == 'evals-only' || + github.event.label.name == 'skip_queue' || + github.event.label.name == 'ci-patchwork' || + github.event.label.name == 'engine-patch' || + github.event.label.name == 'ci-patchwork-waived' || + github.event.label.name == 'ci-checklist-complete' ) outputs: skip-pr-sweep: ${{ steps.sweep_policy.outputs.skip-pr-sweep }} @@ -152,9 +162,87 @@ jobs: --ref "${{ github.ref }}" \ --workflow-id "run-sweep.yml" + classify-priority: + needs: check-changelog + runs-on: ubuntu-latest + if: >- + always() && + github.event_name == 'pull_request' && + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + !github.event.pull_request.draft && + ( + needs.check-changelog.result == 'success' || + needs.check-changelog.result == 'skipped' + ) + permissions: + contents: read + pull-requests: read + outputs: + criteria: ${{ steps.result.outputs.criteria }} + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Classify priority criteria with Fable + id: fable + continue-on-error: true + uses: anthropics/claude-code-action@12531344451323133b0493233c759991ac61da12 # v1.0.174 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + track_progress: false + settings: | + {"fastMode": true} + claude_args: | + --model 'claude-fable-5' + --max-turns 8 + --allowedTools "Read,Glob,Grep,Bash(git diff:*)" + --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' + prompt: | + Inspect this PR's diff against its base branch. + + Return every matching priority criterion: + + - multi-node: separate prefill and decode workers + - agentic: agentic-coding or AgentX workload + - eval-only: evaluation without throughput measurement + - fp4: FP4 precision + - mtp, eagle, eagle3: speculative decoding method + - sglang, vllm, dynamo-vllm: runtime framework + - model criterion: matching configured model family + - checklist-complete: PR checklist is satisfied + - patchwork: modified upstream engine or runtime source + + Patchwork includes patch files, git apply, source mutation, + monkey-patching, local engine builds, forks, or images + containing unmerged engine changes. + + Return a concise reason grounded in the diff. + + - name: Normalize classification + id: result + if: always() + env: + OUTPUT: ${{ steps.fable.outputs.structured_output || '{}' }} + run: | + if jq -e 'type == "object" and (.criteria | type == "array")' \ + >/dev/null <<<"$OUTPUT"; then + criteria=$(jq -c '.criteria' <<<"$OUTPUT") + else + criteria='["patchwork"]' + fi + echo "criteria=$criteria" >> "$GITHUB_OUTPUT" + echo "Fable priority criteria: $criteria" + setup: - needs: [check-changelog, reuse-sweep-gate] + needs: [check-changelog, reuse-sweep-gate, classify-priority] runs-on: ubuntu-latest + permissions: + actions: read + contents: read + issues: read + pull-requests: read if: >- always() && ( @@ -168,6 +256,10 @@ jobs: needs.reuse-sweep-gate.outputs.skip-pr-sweep != 'true' ) ) && + ( + needs.classify-priority.result == 'success' || + needs.classify-priority.result == 'skipped' + ) && ( ( github.event_name == 'pull_request' && @@ -188,7 +280,12 @@ jobs: github.event.label.name == 'full-sweep-fail-fast' || github.event.label.name == 'full-sweep-fail-fast-no-canary' || github.event.label.name == 'all-evals' || - github.event.label.name == 'evals-only' + github.event.label.name == 'evals-only' || + github.event.label.name == 'skip_queue' || + github.event.label.name == 'ci-patchwork' || + github.event.label.name == 'engine-patch' || + github.event.label.name == 'ci-patchwork-waived' || + github.event.label.name == 'ci-checklist-complete' ) ) || ( @@ -212,6 +309,8 @@ jobs: - id: setup env: GH_TOKEN: ${{ github.token }} + PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} + FABLE_CRITERIA: ${{ needs.classify-priority.outputs.criteria }} TRIM_CONC: >- ${{ github.event_name == 'pull_request' && @@ -255,6 +354,13 @@ jobs: fi CONFIG_JSON=$("${CMD[@]}") + CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ + "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ + --event-name "${{ github.event_name }}" \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}" \ + --labels-json "$PR_LABELS" \ + --pr-number "${{ github.event.pull_request.number || 0 }}" \ + --criteria-json "$FABLE_CRITERIA") echo "search-space-config=$CONFIG_JSON" >> "$GITHUB_OUTPUT" python3 "${GITHUB_WORKSPACE}/utils/find_reusable_sweep_run.py" \ @@ -325,6 +431,9 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -365,6 +474,9 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -441,6 +553,9 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -499,6 +614,9 @@ jobs: with: exp-name: ${{ matrix.config.exp-name }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -549,6 +667,9 @@ jobs: osl: '0' max-model-len: '0' runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -609,6 +730,9 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -652,6 +776,9 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index 10c6ddd9ad..8a785f4b96 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -103,10 +103,54 @@ env: PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache jobs: + setup: + runs-on: ubuntu-latest + outputs: + priority: ${{ steps.score.outputs.priority }} + queue-token: ${{ steps.score.outputs.queue-token }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: { clean: true, persist-credentials: false } + - id: score + env: + RUNNER: ${{ inputs.runner }} + MODEL_PREFIX: ${{ inputs.model-prefix }} + run: | + entry=$(jq -cn \ + --arg runner "$RUNNER" \ + --arg model_prefix "$MODEL_PREFIX" \ + '[{ + runner: $runner, + framework: "vllm", + "model-prefix": $model_prefix, + precision: "fp4", + "spec-decoding": "mtp" + }]') + scored=$(printf '%s' "$entry" | + python3 utils/ci_priority.py \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}") + echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" + echo "queue-token=$(jq -r '.[0][\"queue-token\"]' <<<"$scored")" >> "$GITHUB_OUTPUT" + collect-al: - runs-on: ${{ inputs.runner }} + needs: setup + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format( + 'ci-job-{0}-{1}', + needs.setup.outputs.priority, + needs.setup.outputs.queue-token + )), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) || + format('[{0}]', toJSON(inputs.runner)) + ) }} timeout-minutes: 600 - name: "SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" + name: "p${{ needs.setup.outputs.priority }} | SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" steps: - name: Resource cleanup (pre-run) run: &resource-cleanup | diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index fa8af73919..30812246f2 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -7,12 +7,20 @@ on: - main paths: - ".claude/commands/recover-failed-ingest.md" + - ".github/workflows/benchmark-tmpl.yml" - ".github/workflows/benchmark-multinode-tmpl.yml" + - ".github/workflows/collectivex-sweep.yml" + - ".github/workflows/priority-scheduler-canary.yml" + - ".github/workflows/profile.yml" - ".github/workflows/e2e-tests.yml" - ".github/workflows/run-sweep.yml" + - ".github/workflows/speedbench-al.yml" - ".github/workflows/test-changelog-gate.yml" - "benchmarks/benchmark_lib.sh" + - "configs/ci-priority.yaml" - "benchmarks/multi_node/amd_utils/job.slurm" + - "utils/ci_priority.py" + - "utils/test_ci_priority.py" - "utils/find_reusable_sweep_run.py" - "utils/test_find_reusable_sweep_run.py" - "utils/process_changelog.py" @@ -71,6 +79,7 @@ jobs: utils/changelog_gate_tests/test_recover_failed_ingest.py \ utils/test_validate_reusable_sweep_artifacts.py \ utils/changelog_gate_tests/test_run_sweep_gating.py \ + utils/test_ci_priority.py \ utils/test_process_changelog.py \ utils/test_collect_eval_results.py \ utils/evals/test_batched_eval.py \ diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml new file mode 100644 index 0000000000..f86402c23c --- /dev/null +++ b/configs/ci-priority.yaml @@ -0,0 +1,45 @@ +version: 1 + +# Higher scores run first. Scores are uncapped and remain human-readable in +# `ci-job--` workflow labels; `ci-attempt-` prevents +# stale runner labels from matching partial reruns. +base-score: 1.0 + +adjustments: + event: + push: 2.0 + multi-node: 1.25 + agentic: 1.0 + eval-only: 0.0 + precision: + fp4: 0.75 + spec-decoding: + mtp: 0.75 + eagle: 0.75 + eagle3: 0.75 + framework-prefix: + sglang: 0.5 + vllm: 0.5 + dynamo-sglang: 0.5 + dynamo-vllm: 0.5 + model-prefix: + glm5: 0.75 + glm5.1: 0.75 + kimik2.5: 0.75 + dsv4: 0.75 + minimaxm3: 0.75 + qwen3.5: 0.75 + dsr1: 0.75 + +labels: + # inferencex-ci-tracker verifies the active actor with INFX_CI_SCHEDULER_PAT. + skip-queue: + name: skip_queue + checklist-complete: + names: [ci-checklist-complete] + adjustment: 0.25 + patchwork: + names: [ci-patchwork, engine-patch] + score: 0.0 + waived-by: [ci-patchwork-waived] + diff --git a/utils/changelog_gate_tests/test_run_sweep_gating.py b/utils/changelog_gate_tests/test_run_sweep_gating.py index a36f1eb726..7c47f8e0f1 100644 --- a/utils/changelog_gate_tests/test_run_sweep_gating.py +++ b/utils/changelog_gate_tests/test_run_sweep_gating.py @@ -2,10 +2,10 @@ The simulation jobs in `.github/workflows/test-changelog-gate.yml` hand-copy two of the gating `if` conditions and exercise two scenarios. This test parses -the real `check-changelog` -> `reuse-sweep-gate` -> `setup` conditions out of -`run-sweep.yml` and evaluates them with a minimal GitHub Actions expression -engine, so it cannot drift from production and it covers every distinct -skip/run decision. +the real `check-changelog` -> `reuse-sweep-gate` / `classify-priority` -> +`setup` conditions out of `run-sweep.yml` and evaluates them with a minimal +GitHub Actions expression engine, so it cannot drift from production and covers +every distinct skip/run decision. """ from __future__ import annotations @@ -25,6 +25,7 @@ ) CHECK_IF = _WF["jobs"]["check-changelog"]["if"] GATE_IF = _WF["jobs"]["reuse-sweep-gate"]["if"] +CLASSIFY_IF = _WF["jobs"]["classify-priority"]["if"] SETUP_IF = _WF["jobs"]["setup"]["if"] PR_TYPES = set(_WF["on"]["pull_request"]["types"]) @@ -38,8 +39,14 @@ "full-sweep-fail-fast", "full-sweep-fail-fast-no-canary", } -MODIFIER_LABELS = {"all-evals", "evals-only"} -RELEVANT_LABELS = SWEEP_LABELS | MODIFIER_LABELS +MODIFIER_LABELS = {"all-evals", "evals-only", "skip_queue"} +POLICY_LABELS = { + "ci-patchwork", + "engine-patch", + "ci-patchwork-waived", + "ci-checklist-complete", +} +RELEVANT_LABELS = SWEEP_LABELS | MODIFIER_LABELS | POLICY_LABELS REUSE_ELIGIBLE_LABELS = SWEEP_LABELS - {"sweep-enabled"} REUSE_INCOMPATIBLE_LABELS = {"evals-only"} @@ -187,7 +194,7 @@ def _eval(expr: str, ctx: dict) -> bool: # -------------------------------------------------------------------------- -# DAG evaluation: check-changelog -> reuse-sweep-gate -> setup +# DAG evaluation: check-changelog -> reuse-sweep-gate / classify-priority -> setup # -------------------------------------------------------------------------- def _ctx(sc: dict) -> dict: return { @@ -196,6 +203,7 @@ def _ctx(sc: dict) -> dict: "github.event.pull_request.draft": sc.get("draft", False), "github.event.pull_request.labels.*.name": sc.get("labels", []), "github.event.label.name": sc.get("label_name"), + "vars.PRIORITY_SCHEDULER_ENABLED": sc.get("scheduler_enabled", "true"), "github.event.head_commit.message": sc.get("msg", ""), } @@ -223,6 +231,9 @@ def run_dag(sc: dict) -> tuple[str, str, str]: ctx["needs.reuse-sweep-gate.result"] = gate_result ctx["needs.reuse-sweep-gate.outputs.skip-pr-sweep"] = skip + classify_result = "success" if _eval(CLASSIFY_IF, ctx) else "skipped" + ctx["needs.classify-priority.result"] = classify_result + setup = "RUN" if _eval(SETUP_IF, ctx) else "SKIP" return check_result, gate_result, setup @@ -285,6 +296,22 @@ def run_dag(sc: dict) -> tuple[str, str, str]: {**_PR, "action": "labeled", "label_name": "evals-only", "labels": ["full-sweep-enabled", "evals-only"]}, ("success", "skipped", "RUN")), + ("PR-labeled-skip-queue-restarts-full-sweep", + {**_PR, "action": "labeled", "label_name": "skip_queue", + "labels": ["full-sweep-enabled", "skip_queue"]}, + ("success", "skipped", "RUN")), + ("PR-unlabeled-skip-queue-restarts-numeric-sweep", + {**_PR, "action": "unlabeled", "label_name": "skip_queue", + "labels": ["full-sweep-enabled"]}, + ("success", "skipped", "RUN")), + ("PR-labeled-patchwork-restarts-full-sweep", + {**_PR, "action": "labeled", "label_name": "ci-patchwork", + "labels": ["full-sweep-enabled", "ci-patchwork"]}, + ("success", "skipped", "RUN")), + ("PR-unlabeled-patchwork-restarts-full-sweep", + {**_PR, "action": "unlabeled", "label_name": "ci-patchwork", + "labels": ["full-sweep-enabled"]}, + ("success", "skipped", "RUN")), ("PR-labeled-with-unrelated-label", {**_PR, "action": "labeled", "label_name": "documentation", "labels": ["full-sweep-enabled"]}, ("skipped", "skipped", "SKIP")), @@ -346,6 +373,21 @@ def test_trigger_types_enable_gated_events() -> None: assert {"opened", "reopened"}.isdisjoint(PR_TYPES) +def test_priority_classifier_only_runs_when_scheduler_is_enabled() -> None: + scenario = { + **_PR, + "action": "synchronize", + "labels": ["full-sweep-enabled"], + } + disabled = _ctx({**scenario, "scheduler_enabled": "false"}) + enabled = _ctx({**scenario, "scheduler_enabled": "true"}) + disabled["needs.check-changelog.result"] = "success" + enabled["needs.check-changelog.result"] = "success" + + assert not _eval(CLASSIFY_IF, disabled) + assert _eval(CLASSIFY_IF, enabled) + + # -------------------------------------------------------------------------- # Independent reference spec of the INTENDED gating, plus an exhaustive # cross-product cross-check: every combination of the input axes is fed to @@ -424,6 +466,8 @@ def _all_scenarios() -> list[dict]: ["full-sweep-enabled", "evals-only"], ["sweep-enabled", "all-evals", "evals-only"], ["full-sweep-enabled", "all-evals", "evals-only"], + ["skip_queue"], + ["full-sweep-enabled", "skip_queue"], ] pr_axes = itertools.product( ["ready_for_review", "synchronize", "labeled", "unlabeled"], # action @@ -434,6 +478,11 @@ def _all_scenarios() -> list[dict]: "sweep-enabled", "all-evals", "evals-only", + "skip_queue", + "ci-patchwork", + "engine-patch", + "ci-patchwork-waived", + "ci-checklist-complete", "documentation", None, ], # label.name @@ -462,9 +511,9 @@ def test_exhaustive_cross_product() -> None: ] assert not mismatches, mismatches[:10] # Sanity: confirm the sweep actually covered the whole input space - # (4 actions x 2 draft x 18 label-configs x 6 label-names x 2 reuse x - # 2 changelog outcomes x 2 messages = 6912 PR cases, plus 2 push cases). - assert len(scenarios) == 6914 + # (4 actions x 2 draft x 20 label-configs x 11 label-names x 2 reuse x + # 2 changelog outcomes x 2 messages = 14080 PR cases, plus 2 push cases). + assert len(scenarios) == 14082 def test_named_cases_match_reference_spec() -> None: diff --git a/utils/changelog_gate_tests/test_validate_perf_changelog.py b/utils/changelog_gate_tests/test_validate_perf_changelog.py index d6b6a3565f..5a4b67e348 100644 --- a/utils/changelog_gate_tests/test_validate_perf_changelog.py +++ b/utils/changelog_gate_tests/test_validate_perf_changelog.py @@ -320,9 +320,11 @@ def test_run_sweep_checks_changelog_before_reuse_and_setup() -> None: == "${{ steps.sweep_policy.outputs.skip-pr-sweep }}" ) assert jobs["reuse-sweep-gate"]["needs"] == "check-changelog" + assert jobs["classify-priority"]["needs"] == "check-changelog" assert jobs["setup"]["needs"] == [ "check-changelog", "reuse-sweep-gate", + "classify-priority", ] assert ( "needs.check-changelog.result == 'success'" diff --git a/utils/ci_priority.py b/utils/ci_priority.py new file mode 100755 index 0000000000..89b51d3ed5 --- /dev/null +++ b/utils/ci_priority.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Assign deterministic CI priority scores to generated benchmark jobs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from dataclasses import dataclass +from decimal import Decimal, ROUND_HALF_UP +from pathlib import Path +from typing import Any + +import yaml + +SCORE_QUANTUM = Decimal("0.001") + + +@dataclass(frozen=True) +class PriorityContext: + event_name: str = "" + labels: frozenset[str] = frozenset() + queue_namespace: str = "local" + pr_number: int | None = None + criteria: frozenset[str] | None = None + + +def _decimal(value: Any) -> Decimal: + return Decimal(str(value)) + + +def load_policy(path: str | Path) -> dict[str, Any]: + with Path(path).open() as policy_file: + policy = yaml.safe_load(policy_file) + if policy.get("version") != 1: + raise ValueError(f"Unsupported CI priority policy version: {policy.get('version')}") + return policy + + +def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal: + for prefix, adjustment in adjustments.items(): + if value == prefix or value.startswith(f"{prefix}-"): + return _decimal(adjustment) + return Decimal(0) + + +def supported_criteria(policy: dict[str, Any]) -> frozenset[str]: + """Return every classifier fact understood by the configured policy.""" + adjustments = policy["adjustments"] + return frozenset( + { + "multi-node", + "agentic", + "eval-only", + "checklist-complete", + "patchwork", + } + | set(adjustments.get("precision", {})) + | set(adjustments.get("spec-decoding", {})) + | set(adjustments.get("framework-prefix", {})) + | set(adjustments.get("model-prefix", {})) + ) + + +def _entry_from_criteria( + criteria: frozenset[str], + policy: dict[str, Any], + entry: dict[str, Any], +) -> dict[str, Any]: + unknown = criteria - supported_criteria(policy) + if unknown: + raise ValueError(f"Unknown CI priority criteria: {sorted(unknown)}") + precision = str(entry.get("precision", "")) + spec_decoding = str(entry.get("spec-decoding", "")) + framework = str(entry.get("framework", "")) + model_prefix = str(entry.get("model-prefix", "")) + framework_criteria = tuple(policy["adjustments"].get("framework-prefix", {})) + model_criteria = tuple(policy["adjustments"].get("model-prefix", {})) + return { + "prefill": ( + {} if "multi-node" in criteria and entry.get("prefill") is not None else None + ), + "scenario-type": ( + "agentic-coding" + if "agentic" in criteria and entry.get("scenario-type") == "agentic-coding" + else "" + ), + "eval-only": "eval-only" in criteria and entry.get("eval-only") is True, + "precision": precision if precision in criteria else "", + "spec-decoding": spec_decoding if spec_decoding in criteria else "", + "framework": ( + framework + if any( + criterion in criteria + and ( + framework == criterion + or framework.startswith(f"{criterion}-") + ) + for criterion in framework_criteria + ) + else "" + ), + "model-prefix": ( + model_prefix + if any( + criterion in criteria + and ( + model_prefix == criterion + or model_prefix.startswith(f"{criterion}-") + ) + for criterion in model_criteria + ) + else "" + ), + } + + +def calculate_priority( + entry: dict[str, Any], + policy: dict[str, Any], + context: PriorityContext = PriorityContext(), +) -> Decimal: + """Return a higher-is-sooner priority score for one benchmark matrix entry.""" + patchwork = policy["labels"]["patchwork"] + patch_labels = set(patchwork["names"]) + waiver_labels = set(patchwork.get("waived-by", [])) + criteria = context.criteria + if ( + ( + (criteria is not None and "patchwork" in criteria) + or context.labels & patch_labels + ) + and not context.labels & waiver_labels + ): + return _decimal(patchwork["score"]).quantize(SCORE_QUANTUM, ROUND_HALF_UP) + + if criteria is not None: + entry = _entry_from_criteria(criteria, policy, entry) + adjustments = policy["adjustments"] + score = _decimal(policy["base-score"]) + score += _decimal(adjustments.get("event", {}).get(context.event_name, 0)) + + if entry.get("prefill") is not None: + score += _decimal(adjustments.get("multi-node", 0)) + if entry.get("scenario-type") == "agentic-coding": + score += _decimal(adjustments.get("agentic", 0)) + if entry.get("eval-only") is True: + score += _decimal(adjustments.get("eval-only", 0)) + + score += _decimal(adjustments.get("precision", {}).get(str(entry.get("precision", "")), 0)) + score += _decimal( + adjustments.get("spec-decoding", {}).get(str(entry.get("spec-decoding", "")), 0) + ) + score += _first_prefix_adjustment( + str(entry.get("framework", "")), adjustments.get("framework-prefix", {}) + ) + score += _decimal( + adjustments.get("model-prefix", {}).get(str(entry.get("model-prefix", "")), 0) + ) + + checklist = policy["labels"].get("checklist-complete", {}) + if ( + (criteria is not None and "checklist-complete" in criteria) + or context.labels & set(checklist.get("names", [])) + ): + score += _decimal(checklist.get("adjustment", 0)) + + return score.quantize(SCORE_QUANTUM, ROUND_HALF_UP) + + +def format_priority(score: Decimal) -> str: + return f"{score:.3f}" + + +def queue_token(value: dict[str, Any], namespace: str, path: tuple[str, ...]) -> str: + canonical = json.dumps(value, sort_keys=True, separators=(",", ":")) + material = f"{namespace}:{'/'.join(path)}:{canonical}".encode() + return hashlib.sha256(material).hexdigest()[:32] + + +def annotate_jobs( + value: Any, + policy: dict[str, Any], + context: PriorityContext = PriorityContext(), + *, + _path: tuple[str, ...] = (), +) -> Any: + """Copy a generated matrix payload and add scheduling metadata to every job.""" + if isinstance(value, list): + return [ + annotate_jobs(item, policy, context, _path=(*_path, str(index))) + for index, item in enumerate(value) + ] + if not isinstance(value, dict): + return value + + annotated = { + key: annotate_jobs(item, policy, context, _path=(*_path, key)) + for key, item in value.items() + } + if "runner" in value and "framework" in value: + annotated["priority"] = format_priority(calculate_priority(value, policy, context)) + annotated["queue-token"] = queue_token(value, context.queue_namespace, _path) + if ( + context.pr_number is not None + and policy["labels"]["skip-queue"]["name"] in context.labels + ): + annotated["skip-queue-pr"] = context.pr_number + return annotated + + +def _labels_from_json(raw_labels: str) -> frozenset[str]: + if not raw_labels: + return frozenset() + value = json.loads(raw_labels) + if value is None: + return frozenset() + if not isinstance(value, list) or not all(isinstance(label, str) for label in value): + raise ValueError("--labels-json must be a JSON array of strings") + return frozenset(value) + + +def _criteria_from_json(raw_criteria: str) -> frozenset[str] | None: + if not raw_criteria: + return None + value = json.loads(raw_criteria) + if value is None: + return None + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError("--criteria-json must be a JSON array of strings") + return frozenset(value) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--policy", default="configs/ci-priority.yaml") + parser.add_argument("--event-name", default="") + parser.add_argument("--labels-json", default="[]") + parser.add_argument("--queue-namespace", default="local") + parser.add_argument("--pr-number", type=int) + parser.add_argument("--criteria-json", default="") + parser.add_argument( + "--input", + type=Path, + help="Read matrix JSON from this file instead of stdin", + ) + args = parser.parse_args() + + policy = load_policy(args.policy) + context = PriorityContext( + event_name=args.event_name, + labels=_labels_from_json(args.labels_json), + queue_namespace=args.queue_namespace, + pr_number=args.pr_number, + criteria=_criteria_from_json(args.criteria_json), + ) + source = args.input.read_text() if args.input else sys.stdin.read() + payload = json.loads(source) + json.dump(annotate_jobs(payload, policy, context), sys.stdout, separators=(",", ":")) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py new file mode 100644 index 0000000000..47ec74efbb --- /dev/null +++ b/utils/test_ci_priority.py @@ -0,0 +1,231 @@ +import json +import shlex +from copy import deepcopy +from decimal import Decimal +from pathlib import Path + +import pytest +import yaml + +from ci_priority import ( + PriorityContext, + annotate_jobs, + calculate_priority, + load_policy, + queue_token, + supported_criteria, +) + + +POLICY = load_policy(Path(__file__).parents[1] / "configs" / "ci-priority.yaml") + + +def test_combined_high_value_signals_outrank_baseline_job(): + baseline = { + "runner": "h100", + "framework": "trt", + "model-prefix": "other", + "precision": "fp8", + "spec-decoding": "none", + } + high_value = { + **baseline, + "runner": "b200-multinode", + "framework": "sglang", + "model-prefix": "dsv4", + "precision": "fp4", + "spec-decoding": "mtp", + "scenario-type": "agentic-coding", + "prefill": {"hardware": "b200"}, + "decode": {"hardware": "b200"}, + } + + assert calculate_priority(high_value, POLICY) == Decimal("6.000") + assert calculate_priority(baseline, POLICY) == Decimal("1.000") + + +def test_main_branch_jobs_receive_an_automatic_boost(): + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext(event_name="push"), + ) == Decimal("3.000") + + +def test_patchwork_score_uses_half_up_rounding(): + policy = deepcopy(POLICY) + policy["labels"]["patchwork"]["score"] = 0.7225 + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + policy, + PriorityContext(labels=frozenset({"ci-patchwork"})), + ) == Decimal("0.723") + + +def test_skip_queue_request_keeps_numeric_priority(): + entry = {"runner": "h100", "framework": "sglang", "precision": "fp4"} + + annotated = annotate_jobs( + [entry], + POLICY, + PriorityContext( + labels=frozenset({"skip_queue"}), + pr_number=2124, + ), + ) + + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"skip_queue"})), + ) == Decimal("2.250") + assert annotated[0]["priority"] == "2.250" + assert annotated[0]["skip-queue-pr"] == 2124 + + +def test_patchwork_label_forces_bottom_priority_without_waiver(): + entry = {"runner": "b200", "framework": "sglang", "precision": "fp4"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-patchwork"})), + ) == Decimal("0.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-patchwork", "ci-patchwork-waived"})), + ) > Decimal("0.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext(criteria=frozenset({"patchwork"})), + ) == Decimal("0.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext( + labels=frozenset({"ci-patchwork-waived"}), + criteria=frozenset({"patchwork"}), + ), + ) > Decimal("0.000") + + +def test_fable_criteria_drive_all_configured_adjustments(): + criteria = frozenset({"multi-node", "agentic", "fp4", "mtp", "vllm", "dsr1"}) + equivalent_entry = { + "prefill": {}, + "scenario-type": "agentic-coding", + "precision": "fp4", + "spec-decoding": "mtp", + "framework": "vllm", + "model-prefix": "dsr1", + } + entry = dict(equivalent_entry) + + assert calculate_priority( + entry, + POLICY, + PriorityContext(criteria=criteria), + ) == calculate_priority(equivalent_entry, POLICY) + unrelated_entry = {"runner": "h100", "framework": "trt"} + assert calculate_priority( + unrelated_entry, + POLICY, + PriorityContext(criteria=criteria), + ) == Decimal("1.000") + + +def test_checklist_label_applies_alongside_classifier_criteria(): + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext( + labels=frozenset({"ci-checklist-complete"}), + criteria=frozenset(), + ), + ) == Decimal("1.250") + + +def test_fable_criteria_reject_unknown_values_and_allow_mixed_jobs(): + entry = {"runner": "h100", "framework": "vllm"} + + with pytest.raises(ValueError, match="Unknown CI priority criteria"): + calculate_priority( + entry, + POLICY, + PriorityContext(criteria=frozenset({"unknown"})), + ) + assert calculate_priority( + entry, + POLICY, + PriorityContext(criteria=frozenset({"vllm", "sglang"})), + ) == Decimal("1.500") + + +def test_priority_labels_do_not_override_automatic_score(): + entry = {"runner": "h100", "framework": "trt"} + labels = frozenset( + {"ci-priority:p0", "ci-priority:p4.5", "ci-priority:p1000000"} + ) + + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=labels), + ) == Decimal("1.000") + + +def test_annotation_only_touches_runnable_matrix_entries(): + payload = { + "single_node": { + "1k1k": [ + { + "runner": "b200", + "framework": "sglang", + "model-prefix": "qwen3.5", + "precision": "fp4", + "spec-decoding": "mtp", + } + ] + }, + "changelog_metadata": {"runner": "not-a-job"}, + } + + annotated = annotate_jobs(payload, POLICY) + + assert annotated["single_node"]["1k1k"][0]["priority"] == "3.750" + assert len(annotated["single_node"]["1k1k"][0]["queue-token"]) == 32 + assert "priority" not in annotated["changelog_metadata"] + assert "priority" not in payload["single_node"]["1k1k"][0] + assert "queue-token" not in payload["single_node"]["1k1k"][0] + + +def test_classifier_schema_matches_the_policy_vocabulary(): + workflow = yaml.safe_load( + ( + Path(__file__).parents[1] / ".github" / "workflows" / "run-sweep.yml" + ).read_text() + ) + classifier = workflow["jobs"]["classify-priority"]["steps"][1] + arguments = shlex.split(classifier["with"]["claude_args"]) + schema = json.loads(arguments[arguments.index("--json-schema") + 1]) + schema_criteria = schema["properties"]["criteria"]["items"]["enum"] + + assert set(schema_criteria) == set(supported_criteria(POLICY)) + + +def test_queue_tokens_change_between_run_attempts(): + entry = {"runner": "b200", "framework": "sglang"} + + assert queue_token(entry, "123:1", ("0",)) != queue_token( + entry, + "123:2", + ("0",), + )