From 0783c63c7968970c7884f075558a43b581f48f29 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Wed, 6 May 2026 17:20:44 +0800 Subject: [PATCH] support multi devices --- .pipelines/Modelkit E2E Test.yml | 8 + .pipelines/templates/e2e-eval-jobs.yml | 259 ++++++++++++++++--------- 2 files changed, 176 insertions(+), 91 deletions(-) diff --git a/.pipelines/Modelkit E2E Test.yml b/.pipelines/Modelkit E2E Test.yml index e8a5c24e4..ea39dada0 100644 --- a/.pipelines/Modelkit E2E Test.yml +++ b/.pipelines/Modelkit E2E Test.yml @@ -17,6 +17,13 @@ parameters: displayName: 'Skip already-evaluated models (--continue)' type: boolean default: true + - name: ovDevices + displayName: 'NPU-OV: devices to evaluate' + type: object + default: + - cpu + - gpu + - npu stages: - stage: NPU_QNN @@ -39,6 +46,7 @@ stages: agentSuffix: ov evalDate: ${{ parameters.evalDate }} continueRun: ${{ parameters.continueRun }} + devices: ${{ parameters.ovDevices }} - stage: NPU_AMD displayName: 'E2E Eval — NPU-AMD' diff --git a/.pipelines/templates/e2e-eval-jobs.yml b/.pipelines/templates/e2e-eval-jobs.yml index 1c90999df..ebc486728 100644 --- a/.pipelines/templates/e2e-eval-jobs.yml +++ b/.pipelines/templates/e2e-eval-jobs.yml @@ -15,6 +15,10 @@ parameters: - name: modelTimeout type: number default: 3600 + - name: devices + type: object + default: + - npu jobs: - job: Prepare_${{ parameters.agentSuffix }} @@ -95,54 +99,90 @@ jobs: if (-not $evalDate -or $evalDate -eq 'auto') { $evalDate = Get-Date -Format 'yyyy-MM-dd' } $dir = "${{ parameters.evalOutputBase }}/$evalDate/${{ parameters.agentSuffix }}" Write-Host "##vso[task.setvariable variable=EVAL_DIR;isOutput=true]$dir" - Write-Host "Eval output directory: $dir" + Write-Host "Eval output directory (base): $dir" name: set_output_dir displayName: 'Set eval output directory' + - ${{ each device in parameters.devices }}: + - powershell: | + $args = @( + "run", "python", "scripts/e2e_eval/run_eval.py", + "--list-json", "temp/model_list_${{ device }}.json", + "--device", "${{ device }}" + ) + if ('${{ parameters.continueRun }}' -eq 'True') { + $args += @("--continue", "--output-dir", "$(set_output_dir.EVAL_DIR)/${{ device }}") + } + & uv @args + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Generate model list (${{ device }})' + - powershell: | - $args = @( - "run", "python", "scripts/e2e_eval/run_eval.py", - "--list-json", "temp/model_list.json", - "--device", "npu" - ) - if ('${{ parameters.continueRun }}' -eq 'True') { - $args += @("--continue", "--output-dir", "$(set_output_dir.EVAL_DIR)") + $devices = '${{ join(',', parameters.devices) }}' -split ',' + # rowsByKey: matrix-key -> @{ entry = base-fields; pendingDevices = ordered list } + $rowsByKey = [ordered]@{} + $usedKeys = @{} + + foreach ($device in $devices) { + $listPath = "$(Build.SourcesDirectory)/temp/model_list_$device.json" + if (-not (Test-Path $listPath)) { + Write-Warning "Model list missing for device '$device' at $listPath" + continue + } + $models = Get-Content $listPath | ConvertFrom-Json + if ($null -eq $models) { continue } + # Force collection even if a single object was returned + $models = @($models) + Write-Host "Device '$device': $($models.Count) pending models" + + foreach ($m in $models) { + $slug = (($m.hf_id + '_' + $m.task) -replace '[^A-Za-z0-9]', '_') + if ($usedKeys.ContainsKey($slug)) { + $key = $usedKeys[$slug] + } else { + $key = $slug + $suffix = 2 + while ($rowsByKey.Contains($key)) { + $key = "${slug}_${suffix}" + $suffix++ + } + $usedKeys[$slug] = $key + } + + if (-not $rowsByKey.Contains($key)) { + $rowsByKey[$key] = @{ + entry = @{ + hf_id = [string]$m.hf_id + hf_task = [string]$m.task + priority = [string]$m.priority + model_type = [string]$m.model_type + model_group = [string]$m.group + } + pendingDevices = New-Object System.Collections.Generic.List[string] + } + } + $rowsByKey[$key].pendingDevices.Add($device) + } } - & uv @args - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Generate model list' - - powershell: | - $models = Get-Content "$(Build.SourcesDirectory)/temp/model_list.json" | ConvertFrom-Json - $total = $models.Count + $total = $rowsByKey.Count if ($total -eq 0) { - Write-Host "All models already evaluated — nothing to run" + Write-Host "All models already evaluated for all selected devices — nothing to run" Write-Host "##vso[task.setvariable variable=modelMatrix;isOutput=true]{}" Write-Host "##vso[task.setvariable variable=skipEval;isOutput=true]true" return } $matrix = @{} - for ($i = 0; $i -lt $total; $i++) { - $m = $models[$i] - $slug = (($m.hf_id + '_' + $m.task) -replace '[^A-Za-z0-9]', '_') - $key = $slug - $suffix = 2 - while ($matrix.ContainsKey($key)) { - $key = "${slug}_${suffix}" - $suffix++ - } - $matrix[$key] = @{ - hf_id = [string]$m.hf_id - hf_task = [string]$m.task - priority = [string]$m.priority - model_type = [string]$m.model_type - model_group = [string]$m.group - } + foreach ($key in $rowsByKey.Keys) { + $row = $rowsByKey[$key] + $entry = $row.entry.Clone() + $entry['devices_csv'] = ($row.pendingDevices -join ',') + $matrix[$key] = $entry } $json = $matrix | ConvertTo-Json -Compress -Depth 5 - Write-Host "Prepared matrix for $total models" + Write-Host "Prepared matrix for $total models across $($devices.Count) device(s)" Write-Host "##vso[task.setvariable variable=modelMatrix;isOutput=true]$json" name: set_matrix displayName: 'Create matrix variables' @@ -187,72 +227,109 @@ jobs: Write-Host "Build.SourcesDirectory: $(Build.SourcesDirectory)" Write-Host "Model: $(hf_id) / $(hf_task)" Write-Host "Priority: $(priority)" - Write-Host "Output: $(EVAL_DIR)" - - $uvArgs = @( - "run", "--no-sync", "python", "scripts/e2e_eval/run_eval.py", - "--hf-model", "$(hf_id)", - "--output-dir", "$(EVAL_DIR)", - "--device", "npu", - "--continue", - "--verbose", - "--timeout", "${{ parameters.modelTimeout }}", - "--no-report", - "--clean-cache" - ) - if ("$(hf_task)") { - $uvArgs += @("--task", "$(hf_task)") + Write-Host "Output base: $(EVAL_DIR)" + Write-Host "Pending devices: $(devices_csv)" + + $devices = '$(devices_csv)' -split ',' + $devices = $devices | Where-Object { $_ -and $_.Trim() -ne '' } + if (-not $devices -or $devices.Count -eq 0) { + Write-Warning "No pending devices for $(hf_id) / $(hf_task) — skipping" + exit 0 } - & uv @uvArgs - $evalExit = $LASTEXITCODE - if ($evalExit -ne 0) { - Write-Warning "Model eval exited with code $evalExit for $(hf_id) / $(hf_task) (model failure — non-blocking)" + for ($i = 0; $i -lt $devices.Count; $i++) { + $device = $devices[$i].Trim() + $isLast = ($i -eq ($devices.Count - 1)) + Write-Host "============================================================" + Write-Host "[$($i + 1)/$($devices.Count)] device='$device' (last=$isLast) for $(hf_id)" + Write-Host "============================================================" + + $deviceOutput = "$(EVAL_DIR)/$device" + + $uvArgs = @( + "run", "--no-sync", "python", "scripts/e2e_eval/run_eval.py", + "--hf-model", "$(hf_id)", + "--output-dir", $deviceOutput, + "--device", $device, + "--continue", + "--verbose", + "--timeout", "${{ parameters.modelTimeout }}", + "--no-report" + ) + if ($isLast) { + # Only clean cache after the LAST device so HF download and + # any reusable build artifacts can be shared across devices + # for this model. + $uvArgs += "--clean-cache" + } + if ("$(hf_task)") { + $uvArgs += @("--task", "$(hf_task)") + } + + & uv @uvArgs + $evalExit = $LASTEXITCODE + if ($evalExit -ne 0) { + Write-Warning "Model eval exited with code $evalExit for $(hf_id) / $(hf_task) on device '$device' (model failure — non-blocking)" + } } exit 0 workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run eval for current model' - - - job: Report_${{ parameters.agentSuffix }} - displayName: 'Generate Eval Report (${{ parameters.agentSuffix }})' - dependsOn: - - Prepare_${{ parameters.agentSuffix }} - - EvalModel_${{ parameters.agentSuffix }} - condition: always() - pool: - name: modelkit-selfhost-pool - demands: - - Agent.Name -equals ${{ parameters.agentName }} - variables: - EVAL_DIR: $[ dependencies.Prepare_${{ parameters.agentSuffix }}.outputs['set_output_dir.EVAL_DIR'] ] + displayName: 'Run eval for current model (all pending devices)' - steps: - - checkout: self - clean: false - fetchDepth: 1 - path: s + - ${{ each device in parameters.devices }}: + - job: Report_${{ parameters.agentSuffix }}_${{ device }} + displayName: 'Generate Eval Report (${{ parameters.agentSuffix }} / ${{ device }})' + dependsOn: + - Prepare_${{ parameters.agentSuffix }} + - EvalModel_${{ parameters.agentSuffix }} + condition: always() + pool: + name: modelkit-selfhost-pool + demands: + - Agent.Name -equals ${{ parameters.agentName }} + variables: + EVAL_DIR_BASE: $[ dependencies.Prepare_${{ parameters.agentSuffix }}.outputs['set_output_dir.EVAL_DIR'] ] - - checkout: ModelKitArtifacts - clean: false - fetchDepth: 1 - lfs: true - path: artifacts + steps: + - checkout: self + clean: false + fetchDepth: 1 + path: s - - powershell: | - $uvBin = "$env:USERPROFILE\.local\bin" - $venvDir = "$(Build.SourcesDirectory)\.venv\Scripts" - Write-Host "##vso[task.prependpath]$uvBin" - Write-Host "##vso[task.prependpath]$venvDir" - displayName: 'Activate Python environment' + - checkout: ModelKitArtifacts + clean: false + fetchDepth: 1 + lfs: true + path: artifacts - - script: > - uv run --no-sync python scripts/e2e_eval/generate_report.py - --input-dir $(EVAL_DIR) - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Generate evaluation report' + - powershell: | + $uvBin = "$env:USERPROFILE\.local\bin" + $venvDir = "$(Build.SourcesDirectory)\.venv\Scripts" + Write-Host "##vso[task.prependpath]$uvBin" + Write-Host "##vso[task.prependpath]$venvDir" + displayName: 'Activate Python environment' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(EVAL_DIR) - artifactName: EvalReport_${{ parameters.agentSuffix }} - displayName: 'Publish eval results as artifact' + - powershell: | + $deviceDir = "$(EVAL_DIR_BASE)/${{ device }}" + if (-not (Test-Path $deviceDir)) { + Write-Host "No results found at $deviceDir — skipping report" + Write-Host "##vso[task.setvariable variable=HAS_RESULTS]false" + return + } + Write-Host "##vso[task.setvariable variable=HAS_RESULTS]true" + Write-Host "##vso[task.setvariable variable=DEVICE_DIR]$deviceDir" + displayName: 'Resolve device output directory (${{ device }})' + + - script: > + uv run --no-sync python scripts/e2e_eval/generate_report.py + --input-dir $(DEVICE_DIR) + workingDirectory: $(Build.SourcesDirectory) + condition: and(succeeded(), eq(variables['HAS_RESULTS'], 'true')) + displayName: 'Generate evaluation report (${{ device }})' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(DEVICE_DIR) + artifactName: EvalReport_${{ parameters.agentSuffix }}_${{ device }} + condition: and(succeeded(), eq(variables['HAS_RESULTS'], 'true')) + displayName: 'Publish eval results as artifact (${{ device }})'