[CI] Diff translator-lit failures vs amd-staging baseline

.github/workflows/spirv-ci.yml

@@ -102,32 +102,98 @@ jobs:
       # Each suite is its own step so failures are individually identifiable
       # in the PR check rollup. Add new suites as new steps below.
 
-      - name: Test - SPIRV Translator (check-amd-llvm-spirv)
+      - name: Test - SPIRV Translator (check-amd-llvm-spirv) [PR head]
         # Non-blocking: upstream Khronos breaks ~1 translator lit test per
         # week (spirv-val drift, LLVM IR changes vs DebugInfo tests, DCE).
         # Their fixes typically land within a day; our daily upstream-merge
         # cron pulls them in. Blocking here would gate unrelated PRs during
-        # those windows. The next step posts a sticky PR comment listing
-        # any failing tests so AMD-side regressions stay visible.
+        # those windows. The baseline run below partitions failures into
+        # new / fixed / pre-existing so AMD-side regressions stay visible.
         id: check_spirv_xlator
         continue-on-error: true
         run: |
           set -o pipefail
           ninja -C build check-amd-llvm-spirv 2>&1 | tee build/check-amd-llvm-spirv.log
 
-      - name: Post translator lit failure summary to PR
+      - name: Capture PR head translator failures
+        if: github.event_name == 'pull_request' && always()
+        run: |
+          grep -oE '^FAIL: LLVM_SPIRV :: \S+' build/check-amd-llvm-spirv.log \
+            | sort -u > build/spirv-fails-pr.txt || true
+          echo "PR head failures:"; cat build/spirv-fails-pr.txt
+
+      - name: Switch translator to amd-staging tip for baseline
+        if: github.event_name == 'pull_request' && always()
+        run: |
+          cd llvm-project/llvm/projects/SPIRV-LLVM-Translator
+          git fetch --depth=1 origin amd-staging
+          git checkout FETCH_HEAD
+
+      - name: Test - SPIRV Translator [baseline amd-staging]
+        # Re-run the lit suite with the translator at amd-staging tip so the
+        # comment script can compute the per-PR delta (new vs fixed vs
+        # pre-existing). Incremental rebuild — only translator objects change.
+        if: github.event_name == 'pull_request' && always()
+        id: check_spirv_baseline
+        continue-on-error: true
+        run: |
+          set -o pipefail
+          # Re-configure to pick up any CMakeLists / file-list changes in the swap
+          cmake -G Ninja -S llvm-project/llvm -B build
+          ninja -C build check-amd-llvm-spirv 2>&1 \
+            | tee build/check-amd-llvm-spirv-baseline.log
+          grep -oE '^FAIL: LLVM_SPIRV :: \S+' build/check-amd-llvm-spirv-baseline.log \
+            | sort -u > build/spirv-fails-baseline.txt || true
+
+      - name: Post translator lit summary to PR
         if: github.event_name == 'pull_request' && always()
         uses: actions/github-script@v7
         with:
           script: |
             const fs = require('fs');
             const marker = '<!-- spirv-ci:translator-lit -->';
-            const log = fs.readFileSync('build/check-amd-llvm-spirv.log', 'utf8');
-            const failed = [...log.matchAll(/^FAIL: (LLVM_SPIRV :: \S+)/gm)].map(m => m[1]);
+            const read = (p) => {
+              try { return fs.readFileSync(p, 'utf8').split('\n').filter(Boolean); }
+              catch { return null; }
+            };
+            const prList = read('build/spirv-fails-pr.txt');
+            const baseList = read('build/spirv-fails-baseline.txt');
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
-            const body = failed.length === 0
-              ? `${marker}\n✅ **SPIRV translator lit suite (\`check-amd-llvm-spirv\`)**: all tests passing.`
-              : `${marker}\n⚠️ **SPIRV translator lit suite (\`check-amd-llvm-spirv\`)**: ${failed.length} failing — non-blocking (see [run](${runUrl})).\n\nUpstream Khronos churn breaks these intermittently; check whether the failure is also on \`amd-staging\` tip before assuming this PR caused it.\n\n<details><summary>Failing tests</summary>\n\n\`\`\`\n${failed.join('\n')}\n\`\`\`\n</details>`;
+            const fmt = (xs) => xs.length ? '```\n' + xs.join('\n') + '\n```' : '_(none)_';
+
+            let body;
+            if (!prList) {
+              body = `${marker}\n⚠️ **SPIRV translator lit suite**: PR run did not produce a result (see [run](${runUrl})).`;
+            } else if (!baseList) {
+              body = prList.length === 0
+                ? `${marker}\n✅ **SPIRV translator lit suite**: all tests passing on PR head. (Baseline comparison unavailable.)`
+                : `${marker}\n⚠️ **SPIRV translator lit suite**: ${prList.length} failing on PR head; baseline comparison unavailable (see [run](${runUrl})).\n\n<details><summary>Failing tests</summary>\n\n${fmt(prList)}\n</details>`;
+            } else {
+              const prSet = new Set(prList);
+              const baseSet = new Set(baseList);
+              const newFails = prList.filter(t => !baseSet.has(t));
+              const fixed = baseList.filter(t => !prSet.has(t));
+              const common = prList.filter(t => baseSet.has(t));
+
+              let headline;
+              if (newFails.length === 0 && fixed.length === 0 && common.length === 0) {
+                headline = `✅ **SPIRV translator lit suite**: clean on both PR head and \`amd-staging\` baseline.`;
+              } else if (newFails.length > 0) {
+                headline = `🔴 **${newFails.length} new translator lit failure${newFails.length === 1 ? '' : 's'}** introduced by this PR (non-blocking; see [run](${runUrl})).`;
+              } else if (fixed.length > 0 && common.length === 0) {
+                headline = `🟢 **SPIRV translator lit suite**: this PR fixes ${fixed.length} test${fixed.length === 1 ? '' : 's'} that fail on \`amd-staging\`. No remaining failures.`;
+              } else if (fixed.length > 0) {
+                headline = `🟢 This PR fixes ${fixed.length} translator lit test${fixed.length === 1 ? '' : 's'} (vs \`amd-staging\` baseline). ${common.length} pre-existing failure${common.length === 1 ? '' : 's'} remain.`;
+              } else {
+                headline = `⚠️ **${common.length} pre-existing translator lit failure${common.length === 1 ? '' : 's'}** on baseline; not caused by this PR (see [run](${runUrl})).`;
+              }
+
+              body = `${marker}\n${headline}\n\n` +
+                `<details${newFails.length > 0 ? ' open' : ''}><summary>🔴 New failures (${newFails.length}) — likely caused by this PR</summary>\n\n${fmt(newFails)}\n</details>\n\n` +
+                `<details><summary>🟢 Fixed by this PR (${fixed.length}) — failing on baseline, passing here</summary>\n\n${fmt(fixed)}\n</details>\n\n` +
+                `<details><summary>⚠️ Pre-existing on \`amd-staging\` (${common.length})</summary>\n\n${fmt(common)}\n</details>`;
+            }
+
             const { data: comments } = await github.rest.issues.listComments({
               owner: context.repo.owner, repo: context.repo.repo,
               issue_number: context.payload.pull_request.number,