Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 20 additions & 59 deletions .github/workflows/ingest-agentic-results.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ name: Ingest Agentic Benchmark Results
# -H "Accept: application/vnd.github+v3+json" \
# https://api.github.com/repos/SemiAnalysisAI/InferenceX-app/dispatches \
# -d '{"event_type": "ingest-agentic-results",
# "client_payload": {"run-id": "<run_id>", "run-attempt": "<attempt>",
# "client_payload": {"source-run-id": "<benchmark_run_id>",
# "merge-run-id": "<main_run_id>",
# "database-target": "production"}}'
#
# The ingest script (packages/db/src/ingest-ci-run.ts) auto-detects agentic
Expand Down Expand Up @@ -86,7 +87,7 @@ on:
permissions: {}

concurrency:
group: ${{ github.workflow }}-${{ github.event.client_payload.run-id || inputs.run-id }}
group: ${{ github.workflow }}-${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }}
cancel-in-progress: false

jobs:
Expand All @@ -106,7 +107,11 @@ jobs:
- name: Wait for source run to finish
if: >-
github.event_name == 'repository_dispatch' &&
github.event.action == 'ingest-agentic-results'
github.event.action == 'ingest-agentic-results' &&
(
github.event.client_payload.source-run-id == '' ||
github.event.client_payload.source-run-id == github.event.client_payload.merge-run-id
)
run: sleep 300

- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
Expand Down Expand Up @@ -183,68 +188,24 @@ jobs:
echo "Selected ingest target: $REQUESTED_DATABASE_TARGET"
echo "Cache invalidate URL: $cache_invalidate_url"

- name: Run migrations
run: pnpm admin:db:migrate --yes

- name: Download artifacts from InferenceX run
- name: Prepare artifacts from InferenceX
id: artifacts
env:
GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }}
RUN_ID: ${{ github.event.client_payload.run-id || inputs.run-id }}
SOURCE_RUN_ID: ${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }}
MERGE_RUN_ID: ${{ github.event.client_payload.merge-run-id || github.event.client_payload.run-id || inputs.run-id }}
ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
run: |
mkdir -p "$ARTIFACTS_PATH"

# Download all artifacts for the run, deduplicated by name (keep latest).
gh api "repos/SemiAnalysisAI/InferenceX/actions/runs/${RUN_ID}/artifacts" --paginate \
| jq -r '
[.artifacts[]]
| group_by(.name) | map(sort_by(.created_at) | last)[]
| "\(.name)\t\(.archive_download_url)"' \
| while IFS=$'\t' read -r name url; do
echo "Downloading artifact: ${name}"
ok=false
for attempt in 1 2 3; do
if gh api "${url}" > artifact.zip; then
ok=true
break
fi
echo " Attempt ${attempt}/3 failed, retrying in ${attempt}s..."
sleep "$attempt"
done
if [ "$ok" = false ]; then
echo "::warning::Failed to download artifact after 3 attempts: ${name} — skipping"
rm -f artifact.zip
echo "$name" >> "$ARTIFACTS_PATH/.failures"
continue
fi
mkdir -p "${ARTIFACTS_PATH}/${name}"
if ! unzip -o artifact.zip -d "${ARTIFACTS_PATH}/${name}"; then
echo "::warning::Failed to extract artifact: ${name} — skipping"
rm -rf "${ARTIFACTS_PATH:?}/${name}"
echo "$name" >> "$ARTIFACTS_PATH/.failures"
fi
rm -f artifact.zip
done

if [ -f "$ARTIFACTS_PATH/.failures" ]; then
count=$(wc -l < "$ARTIFACTS_PATH/.failures")
rm "$ARTIFACTS_PATH/.failures"
echo "::warning::${count} artifact(s) failed to download; ingesting what's available"
fi

echo "Downloaded artifacts:"
ls "$ARTIFACTS_PATH/"
INGEST_REPO: SemiAnalysisAI/InferenceX
run: pnpm admin:db:prepare:ci

if [ -z "$(ls -A "$ARTIFACTS_PATH")" ]; then
echo "::error::No artifacts could be downloaded from run ${RUN_ID}"
exit 1
fi
- name: Run migrations
run: pnpm admin:db:migrate --yes

- name: Ingest results to DB
env:
GITHUB_TOKEN: ${{ secrets.INFX_MAIN_PAT }}
INGEST_RUN_ID: ${{ github.event.client_payload.run-id || inputs.run-id }}
INGEST_RUN_ATTEMPT: ${{ github.event.client_payload.run-attempt || inputs.run-attempt }}
INGEST_RUN_ID: ${{ steps.artifacts.outputs.merge-run-id }}
INGEST_RUN_ATTEMPT: ${{ steps.artifacts.outputs.merge-run-attempt }}
Comment thread
cquil11 marked this conversation as resolved.
INGEST_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
INGEST_REPO: SemiAnalysisAI/InferenceX
UNMAPPED_ENTITIES_OUTPUT: ${{ github.workspace }}/unmapped-entities.json
Expand Down Expand Up @@ -299,7 +260,7 @@ jobs:
webhook-type: incoming-webhook
payload: |
{
"text": ":warning: *Unrecognized entities during agentic ingest*\nRun ID: ${{ github.event.client_payload.run-id || inputs.run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
"text": ":warning: *Unrecognized entities during agentic ingest*\nRun ID: ${{ steps.artifacts.outputs.source-run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
}

- name: Notify Slack on failure
Expand All @@ -310,5 +271,5 @@ jobs:
webhook-type: incoming-webhook
payload: |
{
"text": ":rotating_light: *Agentic ingest workflow failed*\nRun ID: ${{ github.event.client_payload.run-id || inputs.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
"text": ":rotating_light: *Agentic ingest workflow failed*\nRun ID: ${{ steps.artifacts.outputs.source-run-id || github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
}
105 changes: 18 additions & 87 deletions .github/workflows/ingest-results.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,21 @@ on:
permissions: {}

concurrency:
group: ${{ github.workflow }}-${{ github.event.client_payload.run-id }}
group: ${{ github.workflow }}-${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id }}
cancel-in-progress: false

jobs:
ingest:
name: Ingest benchmark results
timeout-minutes: 15
timeout-minutes: 30
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Wait for source run to finish
if: >-
github.event.client_payload.source-run-id == '' ||
github.event.client_payload.source-run-id == github.event.client_payload.merge-run-id
run: sleep 300

- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
Expand All @@ -34,99 +37,27 @@ jobs:
env:
CYPRESS_INSTALL_BINARY: '0'

- name: Run migrations
env:
DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }}
run: pnpm admin:db:migrate --yes

- name: Download artifacts from InferenceX run
- name: Prepare artifacts from InferenceX
id: artifacts
env:
GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }}
RUN_ID: ${{ github.event.client_payload.run-id }}
SOURCE_RUN_ID: ${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id }}
MERGE_RUN_ID: ${{ github.event.client_payload.merge-run-id || github.event.client_payload.run-id }}
ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
run: |
mkdir -p "$ARTIFACTS_PATH"

# Download all artifacts for the run, deduplicated by name (keep latest).
gh api "repos/SemiAnalysisAI/InferenceX/actions/runs/${RUN_ID}/artifacts" --paginate \
| jq -r '
[.artifacts[]]
| group_by(.name) | map(sort_by(.created_at) | last)[]
| "\(.name)\t\(.archive_download_url)"' \
| while IFS=$'\t' read -r name url; do
echo "Downloading artifact: ${name}"
ok=false
for attempt in 1 2 3; do
if gh api "${url}" > artifact.zip; then
ok=true
break
fi
echo " Attempt ${attempt}/3 failed, retrying in ${attempt}s..."
sleep "$attempt"
done
if [ "$ok" = false ]; then
echo "::warning::Failed to download artifact after 3 attempts: ${name} — skipping"
rm -f artifact.zip
echo "$name" >> "$ARTIFACTS_PATH/.failures"
continue
fi
mkdir -p "${ARTIFACTS_PATH}/${name}"
if ! unzip -o artifact.zip -d "${ARTIFACTS_PATH}/${name}"; then
echo "::warning::Failed to extract artifact: ${name} — skipping"
rm -rf "${ARTIFACTS_PATH:?}/${name}"
echo "$name" >> "$ARTIFACTS_PATH/.failures"
fi
rm -f artifact.zip
done

if [ -f "$ARTIFACTS_PATH/.failures" ]; then
count=$(wc -l < "$ARTIFACTS_PATH/.failures")
rm "$ARTIFACTS_PATH/.failures"
echo "::warning::${count} artifact(s) failed to download; ingesting what's available"
fi

echo "Downloaded artifacts:"
ls "$ARTIFACTS_PATH/"

if [ -z "$(ls -A "$ARTIFACTS_PATH")" ]; then
echo "::error::No artifacts could be downloaded from run ${RUN_ID}"
exit 1
fi
INGEST_REPO: SemiAnalysisAI/InferenceX
run: pnpm admin:db:prepare:ci

- name: Flatten reused ingest artifact bundle
- name: Run migrations
env:
ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
run: |
bundle="$ARTIFACTS_PATH/reused-ingest-artifacts"
if [ ! -d "$bundle" ]; then
echo "No reused ingest artifact bundle found."
exit 0
fi

echo "Flattening reused ingest artifact bundle:"
for child in "$bundle"/*; do
[ -e "$child" ] || continue
name=$(basename "$child")
dest="$ARTIFACTS_PATH/$name"
if [ -e "$dest" ]; then
echo "::warning::Skipping reused artifact '$name'; the run has a fresher copy"
rm -rf "$child"
continue
fi
mv "$child" "$dest"
echo " $name"
done
rmdir "$bundle"

echo "Artifacts after flattening:"
ls "$ARTIFACTS_PATH/"
DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }}
run: pnpm admin:db:migrate --yes

- name: Ingest results to DB
env:
DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }}
GITHUB_TOKEN: ${{ secrets.INFX_MAIN_PAT }}
INGEST_RUN_ID: ${{ github.event.client_payload.run-id }}
INGEST_RUN_ATTEMPT: ${{ github.event.client_payload.run-attempt }}
INGEST_RUN_ID: ${{ steps.artifacts.outputs.merge-run-id }}
INGEST_RUN_ATTEMPT: ${{ steps.artifacts.outputs.merge-run-attempt }}
INGEST_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts
INGEST_REPO: SemiAnalysisAI/InferenceX
UNMAPPED_ENTITIES_OUTPUT: ${{ github.workspace }}/unmapped-entities.json
Expand Down Expand Up @@ -180,7 +111,7 @@ jobs:
webhook-type: incoming-webhook
payload: |
{
"text": ":warning: *Unrecognized entities during ingest*\nRun ID: ${{ github.event.client_payload.run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
"text": ":warning: *Unrecognized entities during ingest*\nRun ID: ${{ steps.artifacts.outputs.source-run-id }}\n```${{ steps.unmapped.outputs.summary }}```\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
}

- name: Notify Slack on failure
Expand All @@ -191,5 +122,5 @@ jobs:
webhook-type: incoming-webhook
payload: |
{
"text": ":rotating_light: *Ingest results workflow failed*\nRun ID: ${{ github.event.client_payload.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
"text": ":rotating_light: *Ingest results workflow failed*\nRun ID: ${{ steps.artifacts.outputs.source-run-id || github.event.client_payload.source-run-id || github.event.client_payload.run-id }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>"
}
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"admin:cache:warmup": "pnpm --filter *app cache:warmup",
"admin:db:ingest:run": "pnpm --filter *db db:ingest:run",
"admin:db:ingest:ci": "pnpm --filter *db db:ingest:ci",
"admin:db:prepare:ci": "pnpm --filter *db db:prepare:ci",
"admin:db:ingest:gcs": "pnpm --filter *db db:ingest:gcs",
"admin:db:ingest:supplemental": "pnpm --filter *db db:ingest:supplemental",
"admin:db:migrate": "pnpm --filter *db db:migrate",
Expand Down
1 change: 1 addition & 0 deletions packages/db/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"./queries/*": "./src/queries/*.ts"
},
"scripts": {
"db:prepare:ci": "tsx src/prepare-ci-artifacts.ts",
"db:ingest:ci": "tsx src/ingest-ci-run.ts",
"db:ingest:run": "dotenv -e ../../.env -- tsx src/ingest-ci-run.ts --download",
"db:ingest:gcs": "dotenv -e ../../.env -- tsx src/ingest-gcs-backup.ts",
Expand Down
58 changes: 58 additions & 0 deletions packages/db/src/lib/ci-artifact-preparation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest';

import { buildArtifactPlan } from './ci-artifact-preparation.js';
import type { ArtifactMeta } from './github-artifacts.js';

const artifact = (id: number, name: string, created_at: string, expired = false): ArtifactMeta => ({
id,
name,
created_at,
expired,
archive_download_url: `https://api.github.com/artifacts/${id}/zip`,
});

describe('buildArtifactPlan', () => {
it('matches normal ingestion by keeping the newest upload for each exact name', () => {
const plan = buildArtifactPlan('100', '100', [
artifact(1, 'bmk_model_runner_01', '2026-01-01T00:00:00Z'),
artifact(2, 'bmk_model_runner_01', '2026-01-02T00:00:00Z'),
artifact(3, 'bmk_model_runner_02', '2026-01-01T00:00:00Z'),
]);

expect(plan.artifacts.map((item) => item.id)).toEqual([2, 3]);
expect(plan.reused).toBe(false);
});

it('uses the merge-run changelog for reused sweeps', () => {
const plan = buildArtifactPlan(
'100',
'200',
[
artifact(1, 'bmk_model', '2026-01-01T00:00:00Z'),
artifact(2, 'changelog-metadata', '2026-01-01T00:01:00Z'),
],
[
artifact(3, 'changelog-metadata', '2026-01-02T00:00:00Z'),
artifact(4, 'changelog-metadata', '2026-01-03T00:00:00Z'),
],
);

expect(plan.artifacts.map((item) => item.id)).toEqual([1, 4]);
expect(plan.reused).toBe(true);
});

it('rejects reuse without source artifacts or a merge changelog', () => {
expect(() =>
buildArtifactPlan(
'100',
'200',
[artifact(1, 'changelog-metadata', '2026-01-01T00:00:00Z')],
[artifact(2, 'changelog-metadata', '2026-01-02T00:00:00Z', true)],
),
).toThrow('No unexpired artifacts found on source run 100');

expect(() =>
buildArtifactPlan('100', '200', [artifact(1, 'bmk_model', '2026-01-01T00:00:00Z')], []),
).toThrow('No changelog-metadata artifact found on merge run 200');
});
});
Loading
Loading