From 9aea72e19fd295d5498544fe5629f34ef4ec877b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 16 Jul 2026 11:37:54 -0500 Subject: [PATCH 1/3] feat: ingest reusable sweeps from source runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Centralize CI artifact preparation in TypeScript. Select the newest valid artifact family per logical benchmark point, use publication-run changelog metadata, discover attempts from GitHub, preserve source provenance, and fail closed on preparation errors. 将 CI 工件准备集中到 TypeScript。按每个逻辑基准点选择最新有效的工件组,使用发布运行的变更日志元数据,从 GitHub 自动获取尝试次数,保留源运行来源,并在准备错误时安全失败。 --- .github/workflows/ingest-agentic-results.yml | 91 +++----- .github/workflows/ingest-results.yml | 117 +++------- package.json | 1 + packages/db/package.json | 1 + packages/db/src/ingest-ci-run.ts | 2 +- .../src/lib/ci-artifact-preparation.test.ts | 180 +++++++++++++++ .../db/src/lib/ci-artifact-preparation.ts | 159 +++++++++++++ packages/db/src/lib/github-artifacts.test.ts | 26 ++- packages/db/src/lib/github-artifacts.ts | 31 ++- packages/db/src/prepare-ci-artifacts.ts | 210 ++++++++++++++++++ 10 files changed, 664 insertions(+), 154 deletions(-) create mode 100644 packages/db/src/lib/ci-artifact-preparation.test.ts create mode 100644 packages/db/src/lib/ci-artifact-preparation.ts create mode 100644 packages/db/src/prepare-ci-artifacts.ts diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index ab6c21e7..5e21aa97 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -8,7 +8,8 @@ name: Ingest Agentic Benchmark Results # -H "Accept: application/vnd.github+v3+json" \ # https://api.github.com/repos/SemiAnalysisAI/InferenceX-app/dispatches \ # -d '{"event_type": "ingest-agentic-results", -# "client_payload": {"run-id": "", "run-attempt": "", +# "client_payload": {"source-run-id": "", +# "merge-run-id": "", # "database-target": "production"}}' # # The ingest script (packages/db/src/ingest-ci-run.ts) auto-detects agentic @@ -86,7 +87,7 @@ on: permissions: {} concurrency: - group: ${{ github.workflow }}-${{ github.event.client_payload.run-id || inputs.run-id }} + group: ${{ github.workflow }}-${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }} cancel-in-progress: false jobs: @@ -106,7 +107,11 @@ jobs: - name: Wait for source run to finish if: >- github.event_name == 'repository_dispatch' && - github.event.action == 'ingest-agentic-results' + github.event.action == 'ingest-agentic-results' && + ( + github.event.client_payload.source-run-id == '' || + github.event.client_payload.source-run-id == github.event.client_payload.merge-run-id + ) run: sleep 300 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -183,68 +188,40 @@ jobs: echo "Selected ingest target: $REQUESTED_DATABASE_TARGET" echo "Cache invalidate URL: $cache_invalidate_url" - - name: Run migrations - run: pnpm admin:db:migrate --yes - - - name: Download artifacts from InferenceX run + - name: Prepare artifacts from InferenceX + id: artifacts env: GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} - RUN_ID: ${{ github.event.client_payload.run-id || inputs.run-id }} + SOURCE_RUN_ID: ${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }} + MERGE_RUN_ID: ${{ github.event.client_payload.merge-run-id || github.event.client_payload.run-id || inputs.run-id }} ARTIFACTS_PATH: ${{ github.workspace }}/artifacts - run: | - mkdir -p "$ARTIFACTS_PATH" - - # Download all artifacts for the run, deduplicated by name (keep latest). - gh api "repos/SemiAnalysisAI/InferenceX/actions/runs/${RUN_ID}/artifacts" --paginate \ - | jq -r ' - [.artifacts[]] - | group_by(.name) | map(sort_by(.created_at) | last)[] - | "\(.name)\t\(.archive_download_url)"' \ - | while IFS=$'\t' read -r name url; do - echo "Downloading artifact: ${name}" - ok=false - for attempt in 1 2 3; do - if gh api "${url}" > artifact.zip; then - ok=true - break - fi - echo " Attempt ${attempt}/3 failed, retrying in ${attempt}s..." - sleep "$attempt" - done - if [ "$ok" = false ]; then - echo "::warning::Failed to download artifact after 3 attempts: ${name} — skipping" - rm -f artifact.zip - echo "$name" >> "$ARTIFACTS_PATH/.failures" - continue - fi - mkdir -p "${ARTIFACTS_PATH}/${name}" - if ! unzip -o artifact.zip -d "${ARTIFACTS_PATH}/${name}"; then - echo "::warning::Failed to extract artifact: ${name} — skipping" - rm -rf "${ARTIFACTS_PATH:?}/${name}" - echo "$name" >> "$ARTIFACTS_PATH/.failures" - fi - rm -f artifact.zip - done + INGEST_REPO: SemiAnalysisAI/InferenceX + run: pnpm admin:db:prepare:ci - if [ -f "$ARTIFACTS_PATH/.failures" ]; then - count=$(wc -l < "$ARTIFACTS_PATH/.failures") - rm "$ARTIFACTS_PATH/.failures" - echo "::warning::${count} artifact(s) failed to download; ingesting what's available" - fi + - name: Checkout reusable-artifact validator + if: steps.artifacts.outputs.reused == 'true' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: SemiAnalysisAI/InferenceX + ref: main + token: ${{ secrets.INFX_MAIN_PAT }} + path: inferencex-validator + persist-credentials: false - echo "Downloaded artifacts:" - ls "$ARTIFACTS_PATH/" + - name: Validate reusable artifacts + if: steps.artifacts.outputs.reused == 'true' + run: >- + python3 inferencex-validator/utils/validate_reusable_sweep_artifacts.py + --artifacts-dir "${{ github.workspace }}/artifacts" - if [ -z "$(ls -A "$ARTIFACTS_PATH")" ]; then - echo "::error::No artifacts could be downloaded from run ${RUN_ID}" - exit 1 - fi + - name: Run migrations + run: pnpm admin:db:migrate --yes - name: Ingest results to DB env: GITHUB_TOKEN: ${{ secrets.INFX_MAIN_PAT }} - INGEST_RUN_ID: ${{ github.event.client_payload.run-id || inputs.run-id }} - INGEST_RUN_ATTEMPT: ${{ github.event.client_payload.run-attempt || inputs.run-attempt }} + INGEST_RUN_ID: ${{ steps.artifacts.outputs.merge-run-id }} + INGEST_RUN_ATTEMPT: ${{ steps.artifacts.outputs.merge-run-attempt }} INGEST_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts INGEST_REPO: SemiAnalysisAI/InferenceX UNMAPPED_ENTITIES_OUTPUT: ${{ github.workspace }}/unmapped-entities.json @@ -299,7 +276,7 @@ jobs: webhook-type: incoming-webhook payload: | { - "text": ":warning: *Unrecognized entities during agentic ingest*\nRun ID: ${{ github.event.client_payload.run-id || inputs.run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + "text": ":warning: *Unrecognized entities during agentic ingest*\nRun ID: ${{ steps.artifacts.outputs.source-run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" } - name: Notify Slack on failure @@ -310,5 +287,5 @@ jobs: webhook-type: incoming-webhook payload: | { - "text": ":rotating_light: *Agentic ingest workflow failed*\nRun ID: ${{ github.event.client_payload.run-id || inputs.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + "text": ":rotating_light: *Agentic ingest workflow failed*\nRun ID: ${{ steps.artifacts.outputs.source-run-id || github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" } diff --git a/.github/workflows/ingest-results.yml b/.github/workflows/ingest-results.yml index 069db129..27c0b2ea 100644 --- a/.github/workflows/ingest-results.yml +++ b/.github/workflows/ingest-results.yml @@ -7,18 +7,21 @@ on: permissions: {} concurrency: - group: ${{ github.workflow }}-${{ github.event.client_payload.run-id }} + group: ${{ github.workflow }}-${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id }} cancel-in-progress: false jobs: ingest: name: Ingest benchmark results - timeout-minutes: 15 + timeout-minutes: 30 runs-on: ubuntu-latest permissions: contents: read steps: - name: Wait for source run to finish + if: >- + github.event.client_payload.source-run-id == '' || + github.event.client_payload.source-run-id == github.event.client_payload.merge-run-id run: sleep 300 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -34,99 +37,43 @@ jobs: env: CYPRESS_INSTALL_BINARY: '0' - - name: Run migrations - env: - DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} - run: pnpm admin:db:migrate --yes - - - name: Download artifacts from InferenceX run + - name: Prepare artifacts from InferenceX + id: artifacts env: GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} - RUN_ID: ${{ github.event.client_payload.run-id }} + SOURCE_RUN_ID: ${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id }} + MERGE_RUN_ID: ${{ github.event.client_payload.merge-run-id || github.event.client_payload.run-id }} ARTIFACTS_PATH: ${{ github.workspace }}/artifacts - run: | - mkdir -p "$ARTIFACTS_PATH" - - # Download all artifacts for the run, deduplicated by name (keep latest). - gh api "repos/SemiAnalysisAI/InferenceX/actions/runs/${RUN_ID}/artifacts" --paginate \ - | jq -r ' - [.artifacts[]] - | group_by(.name) | map(sort_by(.created_at) | last)[] - | "\(.name)\t\(.archive_download_url)"' \ - | while IFS=$'\t' read -r name url; do - echo "Downloading artifact: ${name}" - ok=false - for attempt in 1 2 3; do - if gh api "${url}" > artifact.zip; then - ok=true - break - fi - echo " Attempt ${attempt}/3 failed, retrying in ${attempt}s..." - sleep "$attempt" - done - if [ "$ok" = false ]; then - echo "::warning::Failed to download artifact after 3 attempts: ${name} — skipping" - rm -f artifact.zip - echo "$name" >> "$ARTIFACTS_PATH/.failures" - continue - fi - mkdir -p "${ARTIFACTS_PATH}/${name}" - if ! unzip -o artifact.zip -d "${ARTIFACTS_PATH}/${name}"; then - echo "::warning::Failed to extract artifact: ${name} — skipping" - rm -rf "${ARTIFACTS_PATH:?}/${name}" - echo "$name" >> "$ARTIFACTS_PATH/.failures" - fi - rm -f artifact.zip - done - - if [ -f "$ARTIFACTS_PATH/.failures" ]; then - count=$(wc -l < "$ARTIFACTS_PATH/.failures") - rm "$ARTIFACTS_PATH/.failures" - echo "::warning::${count} artifact(s) failed to download; ingesting what's available" - fi + INGEST_REPO: SemiAnalysisAI/InferenceX + run: pnpm admin:db:prepare:ci - echo "Downloaded artifacts:" - ls "$ARTIFACTS_PATH/" + - name: Checkout reusable-artifact validator + if: steps.artifacts.outputs.reused == 'true' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: SemiAnalysisAI/InferenceX + ref: main + token: ${{ secrets.INFX_MAIN_PAT }} + path: inferencex-validator + persist-credentials: false - if [ -z "$(ls -A "$ARTIFACTS_PATH")" ]; then - echo "::error::No artifacts could be downloaded from run ${RUN_ID}" - exit 1 - fi + - name: Validate reusable artifacts + if: steps.artifacts.outputs.reused == 'true' + run: >- + python3 inferencex-validator/utils/validate_reusable_sweep_artifacts.py + --artifacts-dir "${{ github.workspace }}/artifacts" - - name: Flatten reused ingest artifact bundle + - name: Run migrations env: - ARTIFACTS_PATH: ${{ github.workspace }}/artifacts - run: | - bundle="$ARTIFACTS_PATH/reused-ingest-artifacts" - if [ ! -d "$bundle" ]; then - echo "No reused ingest artifact bundle found." - exit 0 - fi - - echo "Flattening reused ingest artifact bundle:" - for child in "$bundle"/*; do - [ -e "$child" ] || continue - name=$(basename "$child") - dest="$ARTIFACTS_PATH/$name" - if [ -e "$dest" ]; then - echo "::warning::Skipping reused artifact '$name'; the run has a fresher copy" - rm -rf "$child" - continue - fi - mv "$child" "$dest" - echo " $name" - done - rmdir "$bundle" - - echo "Artifacts after flattening:" - ls "$ARTIFACTS_PATH/" + DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} + run: pnpm admin:db:migrate --yes - name: Ingest results to DB env: DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} GITHUB_TOKEN: ${{ secrets.INFX_MAIN_PAT }} - INGEST_RUN_ID: ${{ github.event.client_payload.run-id }} - INGEST_RUN_ATTEMPT: ${{ github.event.client_payload.run-attempt }} + INGEST_RUN_ID: ${{ steps.artifacts.outputs.merge-run-id }} + INGEST_RUN_ATTEMPT: ${{ steps.artifacts.outputs.merge-run-attempt }} INGEST_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts INGEST_REPO: SemiAnalysisAI/InferenceX UNMAPPED_ENTITIES_OUTPUT: ${{ github.workspace }}/unmapped-entities.json @@ -180,7 +127,7 @@ jobs: webhook-type: incoming-webhook payload: | { - "text": ":warning: *Unrecognized entities during ingest*\nRun ID: ${{ github.event.client_payload.run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + "text": ":warning: *Unrecognized entities during ingest*\nRun ID: ${{ steps.artifacts.outputs.source-run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" } - name: Notify Slack on failure @@ -191,5 +138,5 @@ jobs: webhook-type: incoming-webhook payload: | { - "text": ":rotating_light: *Ingest results workflow failed*\nRun ID: ${{ github.event.client_payload.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + "text": ":rotating_light: *Ingest results workflow failed*\nRun ID: ${{ steps.artifacts.outputs.source-run-id || github.event.client_payload.source-run-id || github.event.client_payload.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" } diff --git a/package.json b/package.json index 25540849..ee0366b4 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "admin:cache:warmup": "pnpm --filter *app cache:warmup", "admin:db:ingest:run": "pnpm --filter *db db:ingest:run", "admin:db:ingest:ci": "pnpm --filter *db db:ingest:ci", + "admin:db:prepare:ci": "pnpm --filter *db db:prepare:ci", "admin:db:ingest:gcs": "pnpm --filter *db db:ingest:gcs", "admin:db:ingest:supplemental": "pnpm --filter *db db:ingest:supplemental", "admin:db:migrate": "pnpm --filter *db db:migrate", diff --git a/packages/db/package.json b/packages/db/package.json index 134d9709..b1e7ff51 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -12,6 +12,7 @@ "./queries/*": "./src/queries/*.ts" }, "scripts": { + "db:prepare:ci": "tsx src/prepare-ci-artifacts.ts", "db:ingest:ci": "tsx src/ingest-ci-run.ts", "db:ingest:run": "dotenv -e ../../.env -- tsx src/ingest-ci-run.ts --download", "db:ingest:gcs": "dotenv -e ../../.env -- tsx src/ingest-gcs-backup.ts", diff --git a/packages/db/src/ingest-ci-run.ts b/packages/db/src/ingest-ci-run.ts index 94e1efdb..b6cd4d3a 100644 --- a/packages/db/src/ingest-ci-run.ts +++ b/packages/db/src/ingest-ci-run.ts @@ -128,7 +128,7 @@ if (isDownloadMode) { console.log(`\n--- Downloading artifacts to ${artifactsDir} ---`); // Retried configs produce artifacts on multiple runners — keep only the - // most recent per logical name (see RUNNER_SUFFIX_RE in github-artifacts) + // most recent per logical name (see logicalArtifactName in github-artifacts) // so a failed attempt's empty metrics can't overwrite the good one via // ON CONFLICT DO UPDATE. const byLogical = dedupeArtifactsByLogicalName(listRunArtifacts(REPO, runIdStr)); diff --git a/packages/db/src/lib/ci-artifact-preparation.test.ts b/packages/db/src/lib/ci-artifact-preparation.test.ts new file mode 100644 index 00000000..f23ce8fe --- /dev/null +++ b/packages/db/src/lib/ci-artifact-preparation.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it } from 'vitest'; + +import type { ArtifactMeta } from './github-artifacts.js'; +import { buildArtifactPlan, isIngestableAgenticBenchmarkData } from './ci-artifact-preparation.js'; + +const artifact = (id: number, name: string, created_at: string, expired = false): ArtifactMeta => ({ + id, + name, + created_at, + expired, + archive_download_url: `https://api.github.com/artifacts/${id}/zip`, +}); + +describe('buildArtifactPlan', () => { + it('keeps the newest physical upload for each logical benchmark artifact', () => { + const plan = buildArtifactPlan( + '29413860950', + '29508236865', + [ + artifact( + 8343226821, + 'bmk_agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_01', + '2026-07-15T12:43:07Z', + ), + artifact( + 8347624083, + 'agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_02', + '2026-07-15T15:10:05Z', + ), + artifact( + 8350635857, + 'bmk_agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_02', + '2026-07-15T16:51:38Z', + ), + artifact( + 8350639457, + 'agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_02', + '2026-07-15T16:51:46Z', + ), + artifact( + 8345740968, + 'bmk_agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_04', + '2026-07-15T14:10:11Z', + ), + artifact( + 8347672280, + 'agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_01', + '2026-07-15T15:11:37Z', + ), + artifact( + 8350714141, + 'bmk_agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_05', + '2026-07-15T16:54:24Z', + ), + artifact( + 8350718292, + 'agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_05', + '2026-07-15T16:54:33Z', + ), + artifact(8350722709, 'changelog-metadata', '2026-07-15T16:54:42Z'), + ], + [artifact(9000000001, 'changelog-metadata', '2026-07-16T01:00:00Z')], + ); + + expect(plan.reused).toBe(true); + expect(plan.artifacts.map((item) => item.id)).toEqual([ + 8350639457, 8350718292, 8350635857, 8350714141, 9000000001, + ]); + expect(plan.artifacts.map((item) => item.name)).not.toContain( + 'bmk_agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_04', + ); + }); + + it('uses the publication run changelog and ignores source bundles during reuse', () => { + const plan = buildArtifactPlan( + '100', + '200', + [ + artifact(1, 'bmk_model_h200-cw_0', '2026-01-01T00:00:00Z'), + artifact(2, 'changelog-metadata', '2026-01-01T00:01:00Z'), + artifact(3, 'reused-ingest-artifacts', '2026-01-01T00:02:00Z'), + ], + [ + artifact(4, 'changelog-metadata', '2026-01-02T00:00:00Z'), + artifact(5, 'changelog-metadata', '2026-01-03T00:00:00Z'), + ], + ); + + expect(plan.artifacts.map((item) => item.id)).toEqual([1, 5]); + }); + + it('keeps legacy bundled and non-changelog artifacts for a normal run', () => { + const plan = buildArtifactPlan('100', '100', [ + artifact(1, 'reused-ingest-artifacts', '2026-01-01T00:00:00Z'), + artifact(2, 'changelog-metadata', '2026-01-01T00:01:00Z'), + artifact(3, 'changelog-metadata', '2026-01-02T00:01:00Z'), + ]); + + expect(plan.reused).toBe(false); + expect(plan.artifacts.map((item) => item.id)).toEqual([3, 1]); + }); + + it('ignores expired artifacts and requires a merge-run changelog for reuse', () => { + expect(() => + buildArtifactPlan( + '100', + '200', + [artifact(1, 'bmk_model_h200-cw_0', '2026-01-01T00:00:00Z')], + [artifact(2, 'changelog-metadata', '2026-01-02T00:00:00Z', true)], + ), + ).toThrow('No changelog-metadata artifact found on merge run 200'); + }); + + it('rejects a reused run with no source artifacts before adding the merge changelog', () => { + expect(() => + buildArtifactPlan( + '100', + '200', + [artifact(1, 'changelog-metadata', '2026-01-01T00:00:00Z')], + [artifact(2, 'changelog-metadata', '2026-01-02T00:00:00Z')], + ), + ).toThrow('No unexpired ingestable artifacts found on source run 100'); + }); + + it('anchors raw agentic artifacts to the newest valid benchmark generation', () => { + const plan = buildArtifactPlan( + '100', + '100', + [ + artifact(1, 'bmk_agentic_model_conc64_b300-nv_05', '2026-01-01T00:00:00Z'), + artifact(2, 'agentic_model_conc64_b300-nv_05', '2026-01-01T00:00:01Z'), + artifact(3, 'server_logs_model_conc64_b300-nv_05', '2026-01-01T00:00:02Z'), + artifact(4, 'agentic_model_conc64_b300-nv_13', '2026-01-02T00:00:00Z'), + artifact(5, 'server_logs_model_conc64_b300-nv_13', '2026-01-02T00:00:01Z'), + ], + undefined, + { validAgenticBenchmarkIds: new Set([1]) }, + ); + + expect(plan.artifacts.map((item) => item.id)).toEqual([2, 1, 3]); + }); + + it('chooses the companion upload closest to an exact-name benchmark rerun', () => { + const plan = buildArtifactPlan( + '100', + '100', + [ + artifact(1, 'agentic_model_conc16_b200-nv_02', '2026-01-01T00:00:00Z'), + artifact(2, 'bmk_agentic_model_conc16_b200-nv_02', '2026-01-02T00:00:00Z'), + artifact(3, 'agentic_model_conc16_b200-nv_02', '2026-01-02T00:00:01Z'), + ], + undefined, + { validAgenticBenchmarkIds: new Set([2]) }, + ); + + expect(plan.artifacts.map((item) => item.id)).toEqual([3, 2]); + }); +}); + +describe('isIngestableAgenticBenchmarkData', () => { + it('accepts completed agentic rows with successful requests and throughput', () => { + expect( + isIngestableAgenticBenchmarkData({ + scenario_type: 'agentic-coding', + num_requests_successful: 42, + request_metrics: { throughput: { mean: 12.5 } }, + }), + ).toBe(true); + }); + + it('rejects always-uploaded rows from failed benchmark jobs', () => { + expect( + isIngestableAgenticBenchmarkData({ + scenario_type: 'agentic-coding', + num_requests_successful: 0, + request_metrics: { throughput: {} }, + }), + ).toBe(false); + }); +}); diff --git a/packages/db/src/lib/ci-artifact-preparation.ts b/packages/db/src/lib/ci-artifact-preparation.ts new file mode 100644 index 00000000..2bb8b49a --- /dev/null +++ b/packages/db/src/lib/ci-artifact-preparation.ts @@ -0,0 +1,159 @@ +import { dedupeArtifactsByLogicalName, type ArtifactMeta } from './github-artifacts.js'; + +export const CHANGELOG_ARTIFACT_NAME = 'changelog-metadata'; +const REUSED_BUNDLE_ARTIFACT_NAME = 'reused-ingest-artifacts'; + +export interface ArtifactPlan { + artifacts: ArtifactMeta[]; + reused: boolean; +} + +export interface ArtifactPlanOptions { + validAgenticBenchmarkIds?: ReadonlySet; +} + +export function isIngestableAgenticBenchmarkData(data: unknown): boolean { + const rows = Array.isArray(data) ? data : [data]; + return ( + rows.length > 0 && + rows.every((row) => { + if (!row || typeof row !== 'object' || Array.isArray(row)) return false; + const value = row as Record; + if (value.scenario_type !== 'agentic-coding') return false; + if (typeof value.num_requests_successful !== 'number' || value.num_requests_successful <= 0) { + return false; + } + const requestMetrics = value.request_metrics; + if (!requestMetrics || typeof requestMetrics !== 'object' || Array.isArray(requestMetrics)) { + return false; + } + const throughput = (requestMetrics as Record).throughput; + return ( + throughput !== null && + typeof throughput === 'object' && + !Array.isArray(throughput) && + Object.keys(throughput).length > 0 + ); + }) + ); +} + +function newestArtifact(artifacts: readonly ArtifactMeta[]): ArtifactMeta | undefined { + return artifacts.reduce((newest, artifact) => { + if (!newest) return artifact; + if (artifact.created_at > newest.created_at) return artifact; + if (artifact.created_at < newest.created_at) return newest; + return (artifact.id ?? 0) > (newest.id ?? 0) ? artifact : newest; + }, undefined); +} + +function closestCompanion( + artifacts: readonly ArtifactMeta[], + name: string, + anchor: ArtifactMeta, +): ArtifactMeta | undefined { + const anchorTime = Date.parse(anchor.created_at); + return artifacts + .filter((artifact) => artifact.name === name) + .toSorted((left, right) => { + const leftDelta = Date.parse(left.created_at) - anchorTime; + const rightDelta = Date.parse(right.created_at) - anchorTime; + const distance = Math.abs(leftDelta) - Math.abs(rightDelta); + if (distance !== 0) return distance; + const leftIsAfter = leftDelta >= 0; + const rightIsAfter = rightDelta >= 0; + if (leftIsAfter !== rightIsAfter) return leftIsAfter ? -1 : 1; + return (right.id ?? 0) - (left.id ?? 0); + })[0]; +} + +function isPointCompanion(name: string): boolean { + return ( + name.startsWith('agentic_') || + name.startsWith('server_logs_') || + name.startsWith('multinode_server_logs_') || + name.startsWith('gpu_metrics_') + ); +} + +function companionNames(anchorName: string): string[] { + if (anchorName.startsWith('bmk_agentic_')) { + const suffix = anchorName.slice('bmk_agentic_'.length); + return [`agentic_${suffix}`, `server_logs_${suffix}`, `gpu_metrics_${suffix}`]; + } + const suffix = anchorName.slice('bmk_'.length); + return [`server_logs_${suffix}`, `multinode_server_logs_${suffix}`, `gpu_metrics_${suffix}`]; +} + +/** + * Select the artifact set consumed by one official ingest. + * + * GitHub keeps artifacts from every rerun attempt under the same run ID. + * Runner retries can also change the physical runner/index suffix while still + * representing the same benchmark point. Collapse those physical names to one + * logical name and keep the newest upload. For reused sweeps, benchmark data + * comes from the source PR run while changelog metadata comes from the + * publication run on main. + */ +export function buildArtifactPlan( + sourceRunId: string, + mergeRunId: string, + sourceArtifacts: readonly ArtifactMeta[], + mergeArtifacts: readonly ArtifactMeta[] = sourceArtifacts, + options: ArtifactPlanOptions = {}, +): ArtifactPlan { + const reused = sourceRunId !== mergeRunId; + const usableSource = sourceArtifacts.filter( + (artifact) => + artifact.expired !== true && + (!reused || + (artifact.name !== CHANGELOG_ARTIFACT_NAME && + artifact.name !== REUSED_BUNDLE_ARTIFACT_NAME)), + ); + const agenticBenchmarks = usableSource.filter( + (artifact) => + artifact.name.startsWith('bmk_agentic_') && + (options.validAgenticBenchmarkIds === undefined || + (artifact.id !== undefined && options.validAgenticBenchmarkIds.has(artifact.id))), + ); + const fixedBenchmarks = usableSource.filter( + (artifact) => artifact.name.startsWith('bmk_') && !artifact.name.startsWith('bmk_agentic_'), + ); + const benchmarkAnchors = [ + ...dedupeArtifactsByLogicalName(agenticBenchmarks).values(), + ...dedupeArtifactsByLogicalName(fixedBenchmarks).values(), + ]; + const standaloneArtifacts = usableSource.filter( + (artifact) => !artifact.name.startsWith('bmk_') && !isPointCompanion(artifact.name), + ); + const selected = [...dedupeArtifactsByLogicalName(standaloneArtifacts).values()]; + + for (const anchor of benchmarkAnchors) { + selected.push(anchor); + for (const companionName of companionNames(anchor.name)) { + const companion = closestCompanion(usableSource, companionName, anchor); + if (companion) selected.push(companion); + } + } + + if (selected.length === 0) { + throw new Error(`No unexpired ingestable artifacts found on source run ${sourceRunId}`); + } + + if (reused) { + const changelog = newestArtifact( + mergeArtifacts.filter( + (artifact) => artifact.expired !== true && artifact.name === CHANGELOG_ARTIFACT_NAME, + ), + ); + if (!changelog) { + throw new Error(`No ${CHANGELOG_ARTIFACT_NAME} artifact found on merge run ${mergeRunId}`); + } + selected.push(changelog); + } + + return { + artifacts: selected.toSorted((left, right) => left.name.localeCompare(right.name)), + reused, + }; +} diff --git a/packages/db/src/lib/github-artifacts.test.ts b/packages/db/src/lib/github-artifacts.test.ts index 571643a5..2d049365 100644 --- a/packages/db/src/lib/github-artifacts.test.ts +++ b/packages/db/src/lib/github-artifacts.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { RUNNER_SUFFIX_RE, dedupeArtifactsByLogicalName } from './github-artifacts.js'; +import { + RUNNER_SUFFIX_RE, + dedupeArtifactsByLogicalName, + logicalArtifactName, +} from './github-artifacts.js'; const art = (name: string, created_at: string) => ({ name, @@ -24,6 +28,22 @@ describe('RUNNER_SUFFIX_RE', () => { }); }); +describe('logicalArtifactName', () => { + it('collapses runner pools for the same hardware', () => { + expect(logicalArtifactName('bmk_dsr1_conc4_h200-cw_00')).toBe('bmk_dsr1_conc4_h200'); + expect(logicalArtifactName('bmk_dsr1_conc4_h200-dgxc-slurm_1')).toBe('bmk_dsr1_conc4_h200'); + }); + + it('keeps different hardware generations separate', () => { + expect(logicalArtifactName('bmk_agentic_dsv4_conc16_b200-dgxc_2')).toBe( + 'bmk_agentic_dsv4_conc16_b200', + ); + expect(logicalArtifactName('bmk_agentic_dsv4_conc16_b300-nv_2')).toBe( + 'bmk_agentic_dsv4_conc16_b300', + ); + }); +}); + describe('dedupeArtifactsByLogicalName', () => { it('keeps only the most recent artifact per logical name', () => { const deduped = dedupeArtifactsByLogicalName([ @@ -31,8 +51,8 @@ describe('dedupeArtifactsByLogicalName', () => { art('bmk_dsr1_conc4_h200-dgxc-slurm_1', '2026-06-02T00:00:00Z'), art('bmk_dsr1_conc8_h200-cw_00', '2026-06-01T00:00:00Z'), ]); - expect([...deduped.keys()].toSorted()).toEqual(['bmk_dsr1_conc4', 'bmk_dsr1_conc8']); - expect(deduped.get('bmk_dsr1_conc4')?.name).toBe('bmk_dsr1_conc4_h200-dgxc-slurm_1'); + expect([...deduped.keys()].toSorted()).toEqual(['bmk_dsr1_conc4_h200', 'bmk_dsr1_conc8_h200']); + expect(deduped.get('bmk_dsr1_conc4_h200')?.name).toBe('bmk_dsr1_conc4_h200-dgxc-slurm_1'); }); it('passes through names without a runner suffix unchanged', () => { diff --git a/packages/db/src/lib/github-artifacts.ts b/packages/db/src/lib/github-artifacts.ts index c96ae830..6b52f89a 100644 --- a/packages/db/src/lib/github-artifacts.ts +++ b/packages/db/src/lib/github-artifacts.ts @@ -8,19 +8,21 @@ import { execSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; +import { hwToGpuKey } from '../etl/normalizers.js'; + export interface ArtifactMeta { + id?: number; name: string; archive_download_url: string; created_at: string; + expired?: boolean; } /** - * Strips the trailing `__` token from an - * artifact name so retries on different runners collapse to one logical - * artifact. Without this, two artifacts produced for the same logical - * config (e.g. `…_h200-cw_00` and `…_h200-dgxc-slurm_1`) both land in the - * DB and the failed one's empty metrics can overwrite the good one via - * ON CONFLICT DO UPDATE. + * Matches the trailing `__` token in an artifact + * name. The logical name normalizes the runner pool to its hardware class, so + * retries on `h200-cw` and `h200-dgxc-slurm` collapse while genuinely distinct + * B200 and B300 points remain separate. * * The runner pool name itself has no underscores (`h200-cw`, * `h200-dgxc-slurm`, `b200-nb`), so `[a-zA-Z0-9.-]*` keeps the strip @@ -30,6 +32,13 @@ export interface ArtifactMeta { */ export const RUNNER_SUFFIX_RE = /_[a-zA-Z][a-zA-Z0-9.-]*_\d+$/u; +export function logicalArtifactName(name: string): string { + const match = name.match(/^(?.*)_(?[a-zA-Z][a-zA-Z0-9.-]*)_\d+$/u); + if (!match?.groups) return name; + const hardware = hwToGpuKey(match.groups.runner) ?? match.groups.runner.toLowerCase(); + return `${match.groups.prefix}_${hardware}`; +} + /** List a workflow run's artifacts via `gh api` (paginated). Malformed lines are skipped. */ export function listRunArtifacts(repo: string, runId: string): ArtifactMeta[] { const json = execSync( @@ -57,9 +66,15 @@ export function dedupeArtifactsByLogicalName( ): Map { const byLogical = new Map(); for (const a of artifacts) { - const key = a.name.replace(RUNNER_SUFFIX_RE, ''); + const key = logicalArtifactName(a.name); const existing = byLogical.get(key); - if (!existing || a.created_at > existing.created_at) byLogical.set(key, a); + if ( + !existing || + a.created_at > existing.created_at || + (a.created_at === existing.created_at && (a.id ?? 0) > (existing.id ?? 0)) + ) { + byLogical.set(key, a); + } } return byLogical; } diff --git a/packages/db/src/prepare-ci-artifacts.ts b/packages/db/src/prepare-ci-artifacts.ts new file mode 100644 index 00000000..df96af2b --- /dev/null +++ b/packages/db/src/prepare-ci-artifacts.ts @@ -0,0 +1,210 @@ +#!/usr/bin/env tsx + +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + buildArtifactPlan, + isIngestableAgenticBenchmarkData, +} from './lib/ci-artifact-preparation.js'; +import { downloadArtifact, listRunArtifacts, type ArtifactMeta } from './lib/github-artifacts.js'; + +const DEFAULT_REPO = 'SemiAnalysisAI/InferenceX'; + +interface GithubRunMetadata { + head_sha?: string; + html_url?: string; + pull_requests?: { number?: number }[]; + run_attempt?: number; +} + +function requiredRunId(value: string | undefined, label: string): string { + if (!value || !/^\d+$/u.test(value)) { + throw new Error(`${label} must be a numeric GitHub Actions run ID`); + } + return value; +} + +function fetchRunMetadata(repo: string, runId: string): GithubRunMetadata { + return JSON.parse( + execFileSync('gh', ['api', `repos/${repo}/actions/runs/${runId}`], { + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }), + ) as GithubRunMetadata; +} + +function downloadWithRetries(artifact: ArtifactMeta, artifactsPath: string, attempt = 1): void { + const zipPath = path.join(artifactsPath, 'artifact.zip'); + const artifactPath = path.join(artifactsPath, artifact.name); + try { + fs.rmSync(zipPath, { force: true }); + fs.rmSync(artifactPath, { recursive: true, force: true }); + downloadArtifact(artifact, artifactsPath); + } catch (error) { + fs.rmSync(zipPath, { force: true }); + fs.rmSync(artifactPath, { recursive: true, force: true }); + if (attempt >= 3) { + throw new Error(`Failed to download artifact ${artifact.name} after 3 attempts`, { + cause: error, + }); + } + console.warn(` attempt ${attempt}/3 failed; retrying in ${attempt}s`); + execFileSync('sleep', [String(attempt)]); + downloadWithRetries(artifact, artifactsPath, attempt + 1); + } +} + +function validAgenticBenchmarkIds(artifacts: readonly ArtifactMeta[]): Set { + const candidates = artifacts.filter( + (artifact) => + artifact.expired !== true && + artifact.id !== undefined && + artifact.name.startsWith('bmk_agentic_'), + ); + const validIds = new Set(); + if (candidates.length === 0) return validIds; + + const inspectionRoot = fs.mkdtempSync( + path.join(process.env.RUNNER_TEMP ?? os.tmpdir(), 'bmk-check-'), + ); + try { + for (const artifact of candidates) { + const candidateRoot = path.join(inspectionRoot, String(artifact.id)); + fs.mkdirSync(candidateRoot, { recursive: true }); + downloadWithRetries(artifact, candidateRoot); + const artifactDir = path.join(candidateRoot, artifact.name); + const jsonFiles = fs + .readdirSync(artifactDir) + .filter((name) => name.endsWith('.json')) + .map((name) => path.join(artifactDir, name)); + if ( + jsonFiles.length > 0 && + jsonFiles.every((file) => + isIngestableAgenticBenchmarkData(JSON.parse(fs.readFileSync(file, 'utf8'))), + ) + ) { + validIds.add(artifact.id!); + } + } + } finally { + fs.rmSync(inspectionRoot, { recursive: true, force: true }); + } + + console.log( + `Validated ${validIds.size} of ${candidates.length} agentic benchmark artifact upload(s)`, + ); + return validIds; +} + +function writeOutputs(values: Record): void { + const outputPath = process.env.GITHUB_OUTPUT; + if (!outputPath) return; + fs.appendFileSync( + outputPath, + `${Object.entries(values) + .map(([key, value]) => `${key}=${String(value)}`) + .join('\n')}\n`, + ); +} + +function writeReuseMetadata( + artifactsPath: string, + sourceRunId: string, + mergeRunId: string, + source: GithubRunMetadata, + merge: GithubRunMetadata, +): void { + const metadataDir = path.join(artifactsPath, 'reused-ingest-metadata'); + fs.mkdirSync(metadataDir, { recursive: true }); + fs.writeFileSync( + path.join(metadataDir, 'reuse_source_run.json'), + `${JSON.stringify( + { + source_run_id: sourceRunId, + source_run_attempt: source.run_attempt ?? 1, + source_run_url: + source.html_url ?? + `https://github.com/SemiAnalysisAI/InferenceX/actions/runs/${sourceRunId}`, + source_pr_number: source.pull_requests?.[0]?.number ?? null, + source_head_sha: source.head_sha ?? null, + ingest_run_id: mergeRunId, + ingest_run_attempt: merge.run_attempt ?? 1, + ingest_run_url: + merge.html_url ?? + `https://github.com/SemiAnalysisAI/InferenceX/actions/runs/${mergeRunId}`, + }, + null, + 2, + )}\n`, + ); +} + +function main(): void { + const args = process.argv.slice(2).filter((arg) => arg !== '--'); + const dryRun = args.includes('--dry-run'); + const positional = args.filter((arg) => arg !== '--dry-run'); + const sourceRunId = requiredRunId(process.env.SOURCE_RUN_ID ?? positional[0], 'SOURCE_RUN_ID'); + const mergeRunId = requiredRunId( + process.env.MERGE_RUN_ID ?? positional[1] ?? sourceRunId, + 'MERGE_RUN_ID', + ); + const repo = process.env.INGEST_REPO ?? DEFAULT_REPO; + const artifactsPath = process.env.ARTIFACTS_PATH ?? path.resolve('artifacts'); + + const sourceMetadata = fetchRunMetadata(repo, sourceRunId); + const mergeMetadata = + mergeRunId === sourceRunId ? sourceMetadata : fetchRunMetadata(repo, mergeRunId); + const sourceArtifacts = listRunArtifacts(repo, sourceRunId); + const validAgenticIds = validAgenticBenchmarkIds(sourceArtifacts); + const mergeArtifacts = + mergeRunId === sourceRunId ? sourceArtifacts : listRunArtifacts(repo, mergeRunId); + const plan = buildArtifactPlan(sourceRunId, mergeRunId, sourceArtifacts, mergeArtifacts, { + validAgenticBenchmarkIds: validAgenticIds, + }); + + console.log(`Source run: ${sourceRunId} (attempt ${sourceMetadata.run_attempt ?? 1})`); + console.log(`Merge run: ${mergeRunId} (attempt ${mergeMetadata.run_attempt ?? 1})`); + console.log( + `Selected ${plan.artifacts.length} of ${sourceArtifacts.length} source-run artifact upload(s)${ + plan.reused ? ' plus the merge-run changelog' : '' + }:`, + ); + for (const artifact of plan.artifacts) { + console.log(` ${artifact.created_at} ${artifact.id ?? '-'} ${artifact.name}`); + } + + writeOutputs({ + 'source-run-id': sourceRunId, + 'source-run-attempt': sourceMetadata.run_attempt ?? 1, + 'merge-run-id': mergeRunId, + 'merge-run-attempt': mergeMetadata.run_attempt ?? 1, + reused: plan.reused, + }); + + if (dryRun) { + console.log('Dry run complete; only small benchmark artifacts were inspected temporarily.'); + return; + } + + fs.mkdirSync(artifactsPath, { recursive: true }); + if (fs.readdirSync(artifactsPath).length > 0) { + throw new Error(`ARTIFACTS_PATH must be empty before preparation: ${artifactsPath}`); + } + for (const artifact of plan.artifacts) { + console.log(`Downloading artifact: ${artifact.name}`); + downloadWithRetries(artifact, artifactsPath); + } + if (plan.reused) { + writeReuseMetadata(artifactsPath, sourceRunId, mergeRunId, sourceMetadata, mergeMetadata); + } +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +} From 608e5a2b20a293214a372232c574fca165a4ec5e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 16 Jul 2026 11:40:42 -0500 Subject: [PATCH 2/3] fix: avoid workflow template injection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass the workspace artifact path through the step environment before using it in validator shell commands, satisfying the GitHub Actions security audit without changing behavior. 先通过步骤环境变量传递工作区工件路径,再在验证器的 shell 命令中使用,以在不改变行为的情况下通过 GitHub Actions 安全审计。 --- .github/workflows/ingest-agentic-results.yml | 4 +++- .github/workflows/ingest-results.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index 5e21aa97..8b658bbc 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -210,9 +210,11 @@ jobs: - name: Validate reusable artifacts if: steps.artifacts.outputs.reused == 'true' + env: + ARTIFACTS_PATH: ${{ github.workspace }}/artifacts run: >- python3 inferencex-validator/utils/validate_reusable_sweep_artifacts.py - --artifacts-dir "${{ github.workspace }}/artifacts" + --artifacts-dir "$ARTIFACTS_PATH" - name: Run migrations run: pnpm admin:db:migrate --yes diff --git a/.github/workflows/ingest-results.yml b/.github/workflows/ingest-results.yml index 27c0b2ea..95a70309 100644 --- a/.github/workflows/ingest-results.yml +++ b/.github/workflows/ingest-results.yml @@ -59,9 +59,11 @@ jobs: - name: Validate reusable artifacts if: steps.artifacts.outputs.reused == 'true' + env: + ARTIFACTS_PATH: ${{ github.workspace }}/artifacts run: >- python3 inferencex-validator/utils/validate_reusable_sweep_artifacts.py - --artifacts-dir "${{ github.workspace }}/artifacts" + --artifacts-dir "$ARTIFACTS_PATH" - name: Run migrations env: From 9adcb240431faeff61614a808de6d9baa792ee6d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 16 Jul 2026 12:13:49 -0500 Subject: [PATCH 3/3] refactor: use normal artifact ingestion semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove benchmark-content inspection, logical-point inference, companion pairing, and reusable-set validation. Keep only exact-name deduplication and merge-run changelog substitution before the existing ingestion path.\n\n中文:移除基准测试内容检查、逻辑点推断、配套产物匹配和复用产物集验证。仅保留按精确名称去重及合并运行变更日志替换,然后交给现有摄取流程处理。 --- .github/workflows/ingest-agentic-results.yml | 18 -- .github/workflows/ingest-results.yml | 18 -- packages/db/src/ingest-ci-run.ts | 2 +- .../src/lib/ci-artifact-preparation.test.ts | 162 +++--------------- .../db/src/lib/ci-artifact-preparation.ts | 145 ++-------------- packages/db/src/lib/github-artifacts.test.ts | 26 +-- packages/db/src/lib/github-artifacts.ts | 29 +--- packages/db/src/prepare-ci-artifacts.ts | 55 +----- 8 files changed, 53 insertions(+), 402 deletions(-) diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index 8b658bbc..b1d37009 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -198,24 +198,6 @@ jobs: INGEST_REPO: SemiAnalysisAI/InferenceX run: pnpm admin:db:prepare:ci - - name: Checkout reusable-artifact validator - if: steps.artifacts.outputs.reused == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: SemiAnalysisAI/InferenceX - ref: main - token: ${{ secrets.INFX_MAIN_PAT }} - path: inferencex-validator - persist-credentials: false - - - name: Validate reusable artifacts - if: steps.artifacts.outputs.reused == 'true' - env: - ARTIFACTS_PATH: ${{ github.workspace }}/artifacts - run: >- - python3 inferencex-validator/utils/validate_reusable_sweep_artifacts.py - --artifacts-dir "$ARTIFACTS_PATH" - - name: Run migrations run: pnpm admin:db:migrate --yes diff --git a/.github/workflows/ingest-results.yml b/.github/workflows/ingest-results.yml index 95a70309..4c3ba4f3 100644 --- a/.github/workflows/ingest-results.yml +++ b/.github/workflows/ingest-results.yml @@ -47,24 +47,6 @@ jobs: INGEST_REPO: SemiAnalysisAI/InferenceX run: pnpm admin:db:prepare:ci - - name: Checkout reusable-artifact validator - if: steps.artifacts.outputs.reused == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: SemiAnalysisAI/InferenceX - ref: main - token: ${{ secrets.INFX_MAIN_PAT }} - path: inferencex-validator - persist-credentials: false - - - name: Validate reusable artifacts - if: steps.artifacts.outputs.reused == 'true' - env: - ARTIFACTS_PATH: ${{ github.workspace }}/artifacts - run: >- - python3 inferencex-validator/utils/validate_reusable_sweep_artifacts.py - --artifacts-dir "$ARTIFACTS_PATH" - - name: Run migrations env: DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} diff --git a/packages/db/src/ingest-ci-run.ts b/packages/db/src/ingest-ci-run.ts index b6cd4d3a..94e1efdb 100644 --- a/packages/db/src/ingest-ci-run.ts +++ b/packages/db/src/ingest-ci-run.ts @@ -128,7 +128,7 @@ if (isDownloadMode) { console.log(`\n--- Downloading artifacts to ${artifactsDir} ---`); // Retried configs produce artifacts on multiple runners — keep only the - // most recent per logical name (see logicalArtifactName in github-artifacts) + // most recent per logical name (see RUNNER_SUFFIX_RE in github-artifacts) // so a failed attempt's empty metrics can't overwrite the good one via // ON CONFLICT DO UPDATE. const byLogical = dedupeArtifactsByLogicalName(listRunArtifacts(REPO, runIdStr)); diff --git a/packages/db/src/lib/ci-artifact-preparation.test.ts b/packages/db/src/lib/ci-artifact-preparation.test.ts index f23ce8fe..9c436f5d 100644 --- a/packages/db/src/lib/ci-artifact-preparation.test.ts +++ b/packages/db/src/lib/ci-artifact-preparation.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; +import { buildArtifactPlan } from './ci-artifact-preparation.js'; import type { ArtifactMeta } from './github-artifacts.js'; -import { buildArtifactPlan, isIngestableAgenticBenchmarkData } from './ci-artifact-preparation.js'; const artifact = (id: number, name: string, created_at: string, expired = false): ArtifactMeta => ({ id, @@ -12,169 +12,47 @@ const artifact = (id: number, name: string, created_at: string, expired = false) }); describe('buildArtifactPlan', () => { - it('keeps the newest physical upload for each logical benchmark artifact', () => { - const plan = buildArtifactPlan( - '29413860950', - '29508236865', - [ - artifact( - 8343226821, - 'bmk_agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_01', - '2026-07-15T12:43:07Z', - ), - artifact( - 8347624083, - 'agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_02', - '2026-07-15T15:10:05Z', - ), - artifact( - 8350635857, - 'bmk_agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_02', - '2026-07-15T16:51:38Z', - ), - artifact( - 8350639457, - 'agentic_dsv4_tp8_conc16_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc16_mi355x-amds_02', - '2026-07-15T16:51:46Z', - ), - artifact( - 8345740968, - 'bmk_agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_04', - '2026-07-15T14:10:11Z', - ), - artifact( - 8347672280, - 'agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_01', - '2026-07-15T15:11:37Z', - ), - artifact( - 8350714141, - 'bmk_agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_05', - '2026-07-15T16:54:24Z', - ), - artifact( - 8350718292, - 'agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_05', - '2026-07-15T16:54:33Z', - ), - artifact(8350722709, 'changelog-metadata', '2026-07-15T16:54:42Z'), - ], - [artifact(9000000001, 'changelog-metadata', '2026-07-16T01:00:00Z')], - ); - - expect(plan.reused).toBe(true); - expect(plan.artifacts.map((item) => item.id)).toEqual([ - 8350639457, 8350718292, 8350635857, 8350714141, 9000000001, + it('matches normal ingestion by keeping the newest upload for each exact name', () => { + const plan = buildArtifactPlan('100', '100', [ + artifact(1, 'bmk_model_runner_01', '2026-01-01T00:00:00Z'), + artifact(2, 'bmk_model_runner_01', '2026-01-02T00:00:00Z'), + artifact(3, 'bmk_model_runner_02', '2026-01-01T00:00:00Z'), ]); - expect(plan.artifacts.map((item) => item.name)).not.toContain( - 'bmk_agentic_dsv4_tp8_conc32_kvnone_fp4_sglang_tp8-pp1-dcp1-pcp1-ep1-dpatrue_disagg-false_spec-none_conc32_mi355x-amds_04', - ); + + expect(plan.artifacts.map((item) => item.id)).toEqual([2, 3]); + expect(plan.reused).toBe(false); }); - it('uses the publication run changelog and ignores source bundles during reuse', () => { + it('uses the merge-run changelog for reused sweeps', () => { const plan = buildArtifactPlan( '100', '200', [ - artifact(1, 'bmk_model_h200-cw_0', '2026-01-01T00:00:00Z'), + artifact(1, 'bmk_model', '2026-01-01T00:00:00Z'), artifact(2, 'changelog-metadata', '2026-01-01T00:01:00Z'), - artifact(3, 'reused-ingest-artifacts', '2026-01-01T00:02:00Z'), ], [ - artifact(4, 'changelog-metadata', '2026-01-02T00:00:00Z'), - artifact(5, 'changelog-metadata', '2026-01-03T00:00:00Z'), + artifact(3, 'changelog-metadata', '2026-01-02T00:00:00Z'), + artifact(4, 'changelog-metadata', '2026-01-03T00:00:00Z'), ], ); - expect(plan.artifacts.map((item) => item.id)).toEqual([1, 5]); - }); - - it('keeps legacy bundled and non-changelog artifacts for a normal run', () => { - const plan = buildArtifactPlan('100', '100', [ - artifact(1, 'reused-ingest-artifacts', '2026-01-01T00:00:00Z'), - artifact(2, 'changelog-metadata', '2026-01-01T00:01:00Z'), - artifact(3, 'changelog-metadata', '2026-01-02T00:01:00Z'), - ]); - - expect(plan.reused).toBe(false); - expect(plan.artifacts.map((item) => item.id)).toEqual([3, 1]); + expect(plan.artifacts.map((item) => item.id)).toEqual([1, 4]); + expect(plan.reused).toBe(true); }); - it('ignores expired artifacts and requires a merge-run changelog for reuse', () => { + it('rejects reuse without source artifacts or a merge changelog', () => { expect(() => buildArtifactPlan( '100', '200', - [artifact(1, 'bmk_model_h200-cw_0', '2026-01-01T00:00:00Z')], + [artifact(1, 'changelog-metadata', '2026-01-01T00:00:00Z')], [artifact(2, 'changelog-metadata', '2026-01-02T00:00:00Z', true)], ), - ).toThrow('No changelog-metadata artifact found on merge run 200'); - }); + ).toThrow('No unexpired artifacts found on source run 100'); - it('rejects a reused run with no source artifacts before adding the merge changelog', () => { expect(() => - buildArtifactPlan( - '100', - '200', - [artifact(1, 'changelog-metadata', '2026-01-01T00:00:00Z')], - [artifact(2, 'changelog-metadata', '2026-01-02T00:00:00Z')], - ), - ).toThrow('No unexpired ingestable artifacts found on source run 100'); - }); - - it('anchors raw agentic artifacts to the newest valid benchmark generation', () => { - const plan = buildArtifactPlan( - '100', - '100', - [ - artifact(1, 'bmk_agentic_model_conc64_b300-nv_05', '2026-01-01T00:00:00Z'), - artifact(2, 'agentic_model_conc64_b300-nv_05', '2026-01-01T00:00:01Z'), - artifact(3, 'server_logs_model_conc64_b300-nv_05', '2026-01-01T00:00:02Z'), - artifact(4, 'agentic_model_conc64_b300-nv_13', '2026-01-02T00:00:00Z'), - artifact(5, 'server_logs_model_conc64_b300-nv_13', '2026-01-02T00:00:01Z'), - ], - undefined, - { validAgenticBenchmarkIds: new Set([1]) }, - ); - - expect(plan.artifacts.map((item) => item.id)).toEqual([2, 1, 3]); - }); - - it('chooses the companion upload closest to an exact-name benchmark rerun', () => { - const plan = buildArtifactPlan( - '100', - '100', - [ - artifact(1, 'agentic_model_conc16_b200-nv_02', '2026-01-01T00:00:00Z'), - artifact(2, 'bmk_agentic_model_conc16_b200-nv_02', '2026-01-02T00:00:00Z'), - artifact(3, 'agentic_model_conc16_b200-nv_02', '2026-01-02T00:00:01Z'), - ], - undefined, - { validAgenticBenchmarkIds: new Set([2]) }, - ); - - expect(plan.artifacts.map((item) => item.id)).toEqual([3, 2]); - }); -}); - -describe('isIngestableAgenticBenchmarkData', () => { - it('accepts completed agentic rows with successful requests and throughput', () => { - expect( - isIngestableAgenticBenchmarkData({ - scenario_type: 'agentic-coding', - num_requests_successful: 42, - request_metrics: { throughput: { mean: 12.5 } }, - }), - ).toBe(true); - }); - - it('rejects always-uploaded rows from failed benchmark jobs', () => { - expect( - isIngestableAgenticBenchmarkData({ - scenario_type: 'agentic-coding', - num_requests_successful: 0, - request_metrics: { throughput: {} }, - }), - ).toBe(false); + buildArtifactPlan('100', '200', [artifact(1, 'bmk_model', '2026-01-01T00:00:00Z')], []), + ).toThrow('No changelog-metadata artifact found on merge run 200'); }); }); diff --git a/packages/db/src/lib/ci-artifact-preparation.ts b/packages/db/src/lib/ci-artifact-preparation.ts index 2bb8b49a..497d8989 100644 --- a/packages/db/src/lib/ci-artifact-preparation.ts +++ b/packages/db/src/lib/ci-artifact-preparation.ts @@ -1,159 +1,50 @@ -import { dedupeArtifactsByLogicalName, type ArtifactMeta } from './github-artifacts.js'; +import type { ArtifactMeta } from './github-artifacts.js'; export const CHANGELOG_ARTIFACT_NAME = 'changelog-metadata'; -const REUSED_BUNDLE_ARTIFACT_NAME = 'reused-ingest-artifacts'; export interface ArtifactPlan { artifacts: ArtifactMeta[]; reused: boolean; } -export interface ArtifactPlanOptions { - validAgenticBenchmarkIds?: ReadonlySet; -} - -export function isIngestableAgenticBenchmarkData(data: unknown): boolean { - const rows = Array.isArray(data) ? data : [data]; - return ( - rows.length > 0 && - rows.every((row) => { - if (!row || typeof row !== 'object' || Array.isArray(row)) return false; - const value = row as Record; - if (value.scenario_type !== 'agentic-coding') return false; - if (typeof value.num_requests_successful !== 'number' || value.num_requests_successful <= 0) { - return false; - } - const requestMetrics = value.request_metrics; - if (!requestMetrics || typeof requestMetrics !== 'object' || Array.isArray(requestMetrics)) { - return false; - } - const throughput = (requestMetrics as Record).throughput; - return ( - throughput !== null && - typeof throughput === 'object' && - !Array.isArray(throughput) && - Object.keys(throughput).length > 0 - ); - }) - ); -} - -function newestArtifact(artifacts: readonly ArtifactMeta[]): ArtifactMeta | undefined { - return artifacts.reduce((newest, artifact) => { - if (!newest) return artifact; - if (artifact.created_at > newest.created_at) return artifact; - if (artifact.created_at < newest.created_at) return newest; - return (artifact.id ?? 0) > (newest.id ?? 0) ? artifact : newest; - }, undefined); -} - -function closestCompanion( - artifacts: readonly ArtifactMeta[], - name: string, - anchor: ArtifactMeta, -): ArtifactMeta | undefined { - const anchorTime = Date.parse(anchor.created_at); - return artifacts - .filter((artifact) => artifact.name === name) - .toSorted((left, right) => { - const leftDelta = Date.parse(left.created_at) - anchorTime; - const rightDelta = Date.parse(right.created_at) - anchorTime; - const distance = Math.abs(leftDelta) - Math.abs(rightDelta); - if (distance !== 0) return distance; - const leftIsAfter = leftDelta >= 0; - const rightIsAfter = rightDelta >= 0; - if (leftIsAfter !== rightIsAfter) return leftIsAfter ? -1 : 1; - return (right.id ?? 0) - (left.id ?? 0); - })[0]; -} - -function isPointCompanion(name: string): boolean { - return ( - name.startsWith('agentic_') || - name.startsWith('server_logs_') || - name.startsWith('multinode_server_logs_') || - name.startsWith('gpu_metrics_') - ); -} - -function companionNames(anchorName: string): string[] { - if (anchorName.startsWith('bmk_agentic_')) { - const suffix = anchorName.slice('bmk_agentic_'.length); - return [`agentic_${suffix}`, `server_logs_${suffix}`, `gpu_metrics_${suffix}`]; - } - const suffix = anchorName.slice('bmk_'.length); - return [`server_logs_${suffix}`, `multinode_server_logs_${suffix}`, `gpu_metrics_${suffix}`]; -} - /** - * Select the artifact set consumed by one official ingest. - * - * GitHub keeps artifacts from every rerun attempt under the same run ID. - * Runner retries can also change the physical runner/index suffix while still - * representing the same benchmark point. Collapse those physical names to one - * logical name and keep the newest upload. For reused sweeps, benchmark data - * comes from the source PR run while changelog metadata comes from the - * publication run on main. + * Match normal ingestion: keep the newest upload for each exact artifact name. + * Reused sweeps only differ by taking changelog metadata from the merge run. */ export function buildArtifactPlan( sourceRunId: string, mergeRunId: string, sourceArtifacts: readonly ArtifactMeta[], mergeArtifacts: readonly ArtifactMeta[] = sourceArtifacts, - options: ArtifactPlanOptions = {}, ): ArtifactPlan { const reused = sourceRunId !== mergeRunId; - const usableSource = sourceArtifacts.filter( - (artifact) => - artifact.expired !== true && - (!reused || - (artifact.name !== CHANGELOG_ARTIFACT_NAME && - artifact.name !== REUSED_BUNDLE_ARTIFACT_NAME)), - ); - const agenticBenchmarks = usableSource.filter( - (artifact) => - artifact.name.startsWith('bmk_agentic_') && - (options.validAgenticBenchmarkIds === undefined || - (artifact.id !== undefined && options.validAgenticBenchmarkIds.has(artifact.id))), - ); - const fixedBenchmarks = usableSource.filter( - (artifact) => artifact.name.startsWith('bmk_') && !artifact.name.startsWith('bmk_agentic_'), - ); - const benchmarkAnchors = [ - ...dedupeArtifactsByLogicalName(agenticBenchmarks).values(), - ...dedupeArtifactsByLogicalName(fixedBenchmarks).values(), - ]; - const standaloneArtifacts = usableSource.filter( - (artifact) => !artifact.name.startsWith('bmk_') && !isPointCompanion(artifact.name), - ); - const selected = [...dedupeArtifactsByLogicalName(standaloneArtifacts).values()]; - - for (const anchor of benchmarkAnchors) { - selected.push(anchor); - for (const companionName of companionNames(anchor.name)) { - const companion = closestCompanion(usableSource, companionName, anchor); - if (companion) selected.push(companion); + const selected = new Map(); + for (const artifact of sourceArtifacts) { + if (artifact.expired || (reused && artifact.name === CHANGELOG_ARTIFACT_NAME)) continue; + const current = selected.get(artifact.name); + if (!current || artifact.created_at > current.created_at) { + selected.set(artifact.name, artifact); } } - if (selected.length === 0) { - throw new Error(`No unexpired ingestable artifacts found on source run ${sourceRunId}`); + if (selected.size === 0) { + throw new Error(`No unexpired artifacts found on source run ${sourceRunId}`); } if (reused) { - const changelog = newestArtifact( - mergeArtifacts.filter( - (artifact) => artifact.expired !== true && artifact.name === CHANGELOG_ARTIFACT_NAME, - ), - ); + const changelog = mergeArtifacts + .filter((artifact) => !artifact.expired && artifact.name === CHANGELOG_ARTIFACT_NAME) + .toSorted((left, right) => right.created_at.localeCompare(left.created_at))[0]; if (!changelog) { throw new Error(`No ${CHANGELOG_ARTIFACT_NAME} artifact found on merge run ${mergeRunId}`); } - selected.push(changelog); + selected.set(CHANGELOG_ARTIFACT_NAME, changelog); } return { - artifacts: selected.toSorted((left, right) => left.name.localeCompare(right.name)), + artifacts: [...selected.values()].toSorted((left, right) => + left.name.localeCompare(right.name), + ), reused, }; } diff --git a/packages/db/src/lib/github-artifacts.test.ts b/packages/db/src/lib/github-artifacts.test.ts index 2d049365..571643a5 100644 --- a/packages/db/src/lib/github-artifacts.test.ts +++ b/packages/db/src/lib/github-artifacts.test.ts @@ -1,10 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { - RUNNER_SUFFIX_RE, - dedupeArtifactsByLogicalName, - logicalArtifactName, -} from './github-artifacts.js'; +import { RUNNER_SUFFIX_RE, dedupeArtifactsByLogicalName } from './github-artifacts.js'; const art = (name: string, created_at: string) => ({ name, @@ -28,22 +24,6 @@ describe('RUNNER_SUFFIX_RE', () => { }); }); -describe('logicalArtifactName', () => { - it('collapses runner pools for the same hardware', () => { - expect(logicalArtifactName('bmk_dsr1_conc4_h200-cw_00')).toBe('bmk_dsr1_conc4_h200'); - expect(logicalArtifactName('bmk_dsr1_conc4_h200-dgxc-slurm_1')).toBe('bmk_dsr1_conc4_h200'); - }); - - it('keeps different hardware generations separate', () => { - expect(logicalArtifactName('bmk_agentic_dsv4_conc16_b200-dgxc_2')).toBe( - 'bmk_agentic_dsv4_conc16_b200', - ); - expect(logicalArtifactName('bmk_agentic_dsv4_conc16_b300-nv_2')).toBe( - 'bmk_agentic_dsv4_conc16_b300', - ); - }); -}); - describe('dedupeArtifactsByLogicalName', () => { it('keeps only the most recent artifact per logical name', () => { const deduped = dedupeArtifactsByLogicalName([ @@ -51,8 +31,8 @@ describe('dedupeArtifactsByLogicalName', () => { art('bmk_dsr1_conc4_h200-dgxc-slurm_1', '2026-06-02T00:00:00Z'), art('bmk_dsr1_conc8_h200-cw_00', '2026-06-01T00:00:00Z'), ]); - expect([...deduped.keys()].toSorted()).toEqual(['bmk_dsr1_conc4_h200', 'bmk_dsr1_conc8_h200']); - expect(deduped.get('bmk_dsr1_conc4_h200')?.name).toBe('bmk_dsr1_conc4_h200-dgxc-slurm_1'); + expect([...deduped.keys()].toSorted()).toEqual(['bmk_dsr1_conc4', 'bmk_dsr1_conc8']); + expect(deduped.get('bmk_dsr1_conc4')?.name).toBe('bmk_dsr1_conc4_h200-dgxc-slurm_1'); }); it('passes through names without a runner suffix unchanged', () => { diff --git a/packages/db/src/lib/github-artifacts.ts b/packages/db/src/lib/github-artifacts.ts index 6b52f89a..39f143bd 100644 --- a/packages/db/src/lib/github-artifacts.ts +++ b/packages/db/src/lib/github-artifacts.ts @@ -8,8 +8,6 @@ import { execSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; -import { hwToGpuKey } from '../etl/normalizers.js'; - export interface ArtifactMeta { id?: number; name: string; @@ -19,10 +17,12 @@ export interface ArtifactMeta { } /** - * Matches the trailing `__` token in an artifact - * name. The logical name normalizes the runner pool to its hardware class, so - * retries on `h200-cw` and `h200-dgxc-slurm` collapse while genuinely distinct - * B200 and B300 points remain separate. + * Strips the trailing `__` token from an + * artifact name so retries on different runners collapse to one logical + * artifact. Without this, two artifacts produced for the same logical + * config (e.g. `…_h200-cw_00` and `…_h200-dgxc-slurm_1`) both land in the + * DB and the failed one's empty metrics can overwrite the good one via + * ON CONFLICT DO UPDATE. * * The runner pool name itself has no underscores (`h200-cw`, * `h200-dgxc-slurm`, `b200-nb`), so `[a-zA-Z0-9.-]*` keeps the strip @@ -32,13 +32,6 @@ export interface ArtifactMeta { */ export const RUNNER_SUFFIX_RE = /_[a-zA-Z][a-zA-Z0-9.-]*_\d+$/u; -export function logicalArtifactName(name: string): string { - const match = name.match(/^(?.*)_(?[a-zA-Z][a-zA-Z0-9.-]*)_\d+$/u); - if (!match?.groups) return name; - const hardware = hwToGpuKey(match.groups.runner) ?? match.groups.runner.toLowerCase(); - return `${match.groups.prefix}_${hardware}`; -} - /** List a workflow run's artifacts via `gh api` (paginated). Malformed lines are skipped. */ export function listRunArtifacts(repo: string, runId: string): ArtifactMeta[] { const json = execSync( @@ -66,15 +59,9 @@ export function dedupeArtifactsByLogicalName( ): Map { const byLogical = new Map(); for (const a of artifacts) { - const key = logicalArtifactName(a.name); + const key = a.name.replace(RUNNER_SUFFIX_RE, ''); const existing = byLogical.get(key); - if ( - !existing || - a.created_at > existing.created_at || - (a.created_at === existing.created_at && (a.id ?? 0) > (existing.id ?? 0)) - ) { - byLogical.set(key, a); - } + if (!existing || a.created_at > existing.created_at) byLogical.set(key, a); } return byLogical; } diff --git a/packages/db/src/prepare-ci-artifacts.ts b/packages/db/src/prepare-ci-artifacts.ts index df96af2b..fbf834d7 100644 --- a/packages/db/src/prepare-ci-artifacts.ts +++ b/packages/db/src/prepare-ci-artifacts.ts @@ -2,13 +2,9 @@ import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; -import { - buildArtifactPlan, - isIngestableAgenticBenchmarkData, -} from './lib/ci-artifact-preparation.js'; +import { buildArtifactPlan } from './lib/ci-artifact-preparation.js'; import { downloadArtifact, listRunArtifacts, type ArtifactMeta } from './lib/github-artifacts.js'; const DEFAULT_REPO = 'SemiAnalysisAI/InferenceX'; @@ -57,48 +53,6 @@ function downloadWithRetries(artifact: ArtifactMeta, artifactsPath: string, atte } } -function validAgenticBenchmarkIds(artifacts: readonly ArtifactMeta[]): Set { - const candidates = artifacts.filter( - (artifact) => - artifact.expired !== true && - artifact.id !== undefined && - artifact.name.startsWith('bmk_agentic_'), - ); - const validIds = new Set(); - if (candidates.length === 0) return validIds; - - const inspectionRoot = fs.mkdtempSync( - path.join(process.env.RUNNER_TEMP ?? os.tmpdir(), 'bmk-check-'), - ); - try { - for (const artifact of candidates) { - const candidateRoot = path.join(inspectionRoot, String(artifact.id)); - fs.mkdirSync(candidateRoot, { recursive: true }); - downloadWithRetries(artifact, candidateRoot); - const artifactDir = path.join(candidateRoot, artifact.name); - const jsonFiles = fs - .readdirSync(artifactDir) - .filter((name) => name.endsWith('.json')) - .map((name) => path.join(artifactDir, name)); - if ( - jsonFiles.length > 0 && - jsonFiles.every((file) => - isIngestableAgenticBenchmarkData(JSON.parse(fs.readFileSync(file, 'utf8'))), - ) - ) { - validIds.add(artifact.id!); - } - } - } finally { - fs.rmSync(inspectionRoot, { recursive: true, force: true }); - } - - console.log( - `Validated ${validIds.size} of ${candidates.length} agentic benchmark artifact upload(s)`, - ); - return validIds; -} - function writeOutputs(values: Record): void { const outputPath = process.env.GITHUB_OUTPUT; if (!outputPath) return; @@ -158,12 +112,9 @@ function main(): void { const mergeMetadata = mergeRunId === sourceRunId ? sourceMetadata : fetchRunMetadata(repo, mergeRunId); const sourceArtifacts = listRunArtifacts(repo, sourceRunId); - const validAgenticIds = validAgenticBenchmarkIds(sourceArtifacts); const mergeArtifacts = mergeRunId === sourceRunId ? sourceArtifacts : listRunArtifacts(repo, mergeRunId); - const plan = buildArtifactPlan(sourceRunId, mergeRunId, sourceArtifacts, mergeArtifacts, { - validAgenticBenchmarkIds: validAgenticIds, - }); + const plan = buildArtifactPlan(sourceRunId, mergeRunId, sourceArtifacts, mergeArtifacts); console.log(`Source run: ${sourceRunId} (attempt ${sourceMetadata.run_attempt ?? 1})`); console.log(`Merge run: ${mergeRunId} (attempt ${mergeMetadata.run_attempt ?? 1})`); @@ -185,7 +136,7 @@ function main(): void { }); if (dryRun) { - console.log('Dry run complete; only small benchmark artifacts were inspected temporarily.'); + console.log('Dry run complete; no artifacts were downloaded.'); return; }