fix: guard map --baseline against a moved rule set

.changeset/calm-baselines-check.md

@@ -0,0 +1,5 @@
+---
+'@evlog/cli': patch
+---
+
+fix: `evlog map --baseline` records the CLI and rule-set versions in `evlog.map.json` and refuses to diff (exit 2) when the committed rule set does not match the running CLI, instead of reporting regressions caused by a rule change

apps/docs/content/3.cli/2.map.md

@@ -285,7 +285,7 @@ The check becomes `n/a` with your reason attached, so it stops costing score —
 
 ## evlog.map.json
 
-Every run writes `evlog.map.json` to the project root: the score, the framework, and every entry point with its checks, its suggestions, its sensitivity, and its own score. It is the same data `--json` prints.
+Every run writes `evlog.map.json` to the project root: the score, the framework, the CLI version and rule-set version that wrote it, and every entry point with its checks, its suggestions, its sensitivity, and its own score. It is the same data `--json` prints.
 
 Use `--no-write` when you do not want the file — CI runs, or a quick look at somebody else's project. The file is a build artifact, so [gitignore it](/cli/ci#the-map-file) unless you have a reason to track it.
 

apps/docs/content/3.cli/5.ci.md

@@ -126,13 +126,24 @@ evlog map --baseline ../base-map.json   # or any path
 
 With a bare `--baseline` and no map on disk, the CLI falls back to `git:HEAD` on its own, so the answer stays "what did the last commit say" rather than "what did I say a minute ago".
 
+### When the rule set moves
+
+`evlog.map.json` records the CLI version that wrote it and a separate **rule-set version** that only changes when a rule's semantics change. On `--baseline`, the CLI compares the committed rule-set version against its own. If they differ, it refuses to diff and exits `2`: a rule tightened between the two versions would show up as a `pass` to `fail` transition on code the pull request did not touch, and a gate that blames the wrong thing is worse than one that admits it cannot run.
+
+```
+baseline was written by @evlog/cli 0.3.0, running @evlog/cli 0.4.1 (rule set 1 → 2)
+regenerate the baseline: evlog map && git add evlog.map.json
+```
+
+A release that ships a feature but no rule change keeps the rule-set version, so upgrading the CLI does not force everyone to regenerate. A map written before version reporting has no version fields; the CLI treats it as unknown and warns once instead of failing every project on upgrade.
+
 ## Exit codes
 
 | Code | Meaning |
 | --- | --- |
 | `0` | Score met the threshold, no regression against the baseline, or neither was requested |
 | `1` | Score below `--min-score`, a regression against `--baseline`, or the scan could not run |
-| `2` | Usage error — unknown flag, or an invalid `--framework` |
+| `2` | Usage error — unknown flag, invalid `--framework`, or a baseline whose rule set does not match the running CLI |
 
 ::warning
 A pipe replaces `$?` with the exit code of the last command in it, so `evlog map --min-score 90 \| tee map.log` always looks green. Add `set -o pipefail` to the step.

packages/cli/src/commands/map.ts

@@ -1,13 +1,13 @@
 import { EvlogError } from 'evlog'
 import type { CliContext } from '../core/context'
-import { EXIT_FAIL } from '../core/output'
+import { EXIT_FAIL, EXIT_USAGE } from '../core/output'
 import { defineEvlogCommand } from '../lib/command'
 import type { CliDebug } from '../lib/debug'
 import { createNoopCliDebug } from '../lib/debug'
 import { cliErrors } from '../lib/errors'
 import { resolveEvlog, resolveProject } from '../lib/project'
 import type { ProjectInfo } from '../lib/project'
-import { compareToBaseline, hasRegressed, loadBaseline } from '../lib/map/baseline'
+import { checkBaselineVersion, compareToBaseline, hasRegressed, loadBaseline } from '../lib/map/baseline'
 import type { BaselineComparison } from '../lib/map/baseline'
 import { detectFramework } from '../lib/map/detect'
 import {
@@ -42,6 +42,8 @@ export interface MapResult {
   mapPath: string | null
   /** Diff against the committed map, when `--baseline` was passed. */
   baseline: BaselineComparison | null
+  /** Baseline problems that do not stop the run — a map that predates version reporting. */
+  baselineWarnings: string[]
 }
 
 /**
@@ -93,6 +95,16 @@ export async function runMap(
     )
     : null
 
+  /* A map written before version reporting cannot prove its rule set matches
+     the running one, so it gets a warning instead of a gate: hard-failing every
+     project on upgrade would punish the ones that never saw the feature. */
+  const baselineWarnings: string[] = []
+  if (baselineMap && checkBaselineVersion(baselineMap.map) === 'unknown') {
+    baselineWarnings.push(
+      `the baseline ${baselineMap.source.label} predates map version reporting, so its rule set cannot be verified; regenerate it with evlog map`,
+    )
+  }
+
   const scanResult = await log.step(
     'scan',
     () => scan(scanCtx),
@@ -122,6 +134,7 @@ export async function runMap(
     scan: scanResult,
     mapPath,
     baseline,
+    baselineWarnings,
   }
 }
 
@@ -143,7 +156,7 @@ export function formatMapReport(
   /* Framework detection and disable-comment problems share one channel: both
      mean "the numbers below were produced under an assumption you should see",
      and both have to appear above every view rather than only the default one. */
-  const warnings = [...result.frameworkWarnings, ...result.scan.warnings]
+  const warnings = [...result.frameworkWarnings, ...result.baselineWarnings, ...result.scan.warnings]
   if (warnings.length > 0) {
     sections.push(formatMapWarnings(ctx, warnings))
   }
@@ -260,7 +273,9 @@ export default defineEvlogCommand('map', {
           json: { error: { code: error.code, message: error.message, why: error.why, fix: error.fix } },
           human: error.fix ? `${error.message}\n→ ${error.fix}` : error.message,
         })
-        ui.exit(EXIT_FAIL)
+        /* A baseline whose rule set does not match is a usage error, not a
+           check failure: the app did not get worse, the comparison is invalid. */
+        ui.exit(error.code === cliErrors.MAP_BASELINE_VERSION_MISMATCH.code ? EXIT_USAGE : EXIT_FAIL)
         return
       }
       throw error

packages/cli/src/lib/errors.ts

@@ -177,6 +177,15 @@ export const cliErrors = defineErrorCatalog('cli', {
     link: 'https://evlog.dev/cli/ci',
     tags: ['map', 'baseline'],
   },
+  MAP_BASELINE_VERSION_MISMATCH: {
+    status: 400,
+    message: ({ baselineCli, runningCli, baselineRuleSet, runningRuleSet }: { baselineCli: string, runningCli: string, baselineRuleSet: number, runningRuleSet: number }) =>
+      `baseline was written by @evlog/cli ${baselineCli}, running @evlog/cli ${runningCli} (rule set ${baselineRuleSet} \u2192 ${runningRuleSet})`,
+    why: 'The rule set changed between the two versions, so a per-check diff could blame code the PR did not touch',
+    fix: 'Regenerate the baseline: evlog map && git add evlog.map.json',
+    link: 'https://evlog.dev/cli/ci',
+    tags: ['map', 'baseline'],
+  },
   MAP_INVALID_MIN_SCORE: {
     status: 400,
     message: ({ value }: { value: string }) =>

packages/cli/src/lib/map/baseline.ts

@@ -2,7 +2,9 @@ import { execFileSync } from 'node:child_process'
 import { readFileSync } from 'node:fs'
 import { isAbsolute, resolve } from 'node:path'
 import { cliErrors } from '../errors'
+import { version as CLI_VERSION } from '../../../package.json'
 import { classifyRouteObservability, scoreGlobal } from './score'
+import { RULE_SET_VERSION } from './rules/index'
 import type { CheckId, MapFile, RouteEntry } from './types'
 import { MAP_FILE_NAME } from './write'
 
@@ -207,3 +209,25 @@ export function compareToBaseline(baseline: MapFile, current: MapFile, source: B
 export function hasRegressed(comparison: BaselineComparison): boolean {
   return comparison.regressions.length > 0 || comparison.delta < 0
 }
+
+/**
+ * Whether a committed baseline is comparable to the running CLI.
+ *
+ * Returns `'unknown'` (the caller warns rather than fails) when the map
+ * predates version reporting, and throws a usage error when the rule set moved
+ * underneath the committed file. Same reasoning as a malformed `--min-score`:
+ * a gate that reports a regression it cannot justify is worse than one that
+ * admits it cannot run.
+ */
+export type BaselineVersionStatus = 'ok' | 'unknown'
+
+export function checkBaselineVersion(baseline: Pick<MapFile, 'cliVersion' | 'ruleSetVersion'>): BaselineVersionStatus {
+  if (baseline.ruleSetVersion === undefined) return 'unknown'
+  if (baseline.ruleSetVersion === RULE_SET_VERSION) return 'ok'
+  throw cliErrors.MAP_BASELINE_VERSION_MISMATCH({
+    baselineCli: baseline.cliVersion ?? 'unknown',
+    runningCli: CLI_VERSION,
+    baselineRuleSet: baseline.ruleSetVersion,
+    runningRuleSet: RULE_SET_VERSION,
+  })
+}

packages/cli/src/lib/map/rules/index.ts

@@ -64,6 +64,18 @@ void idsMatch
  */
 export const RULES: readonly MapRule[] = REGISTRY
 
+/**
+ * Version of the rule set as written into `evlog.map.json`.
+ *
+ * Bump this only when a rule's semantics change in a way that could flip a
+ * verdict for code a PR did not touch: a tightened check, a new requirement, a
+ * reweighting. A release that adds a map feature without moving any verdict
+ * must leave it alone, or every project would be forced to regenerate its
+ * baseline for nothing. Written beside the CLI version so `--baseline` can
+ * tell a stale committed map apart from a merely older one.
+ */
+export const RULE_SET_VERSION = 1
+
 const RULES_BY_ID = new Map<CheckId, MapRule>(RULES.map(rule => [rule.id, rule]))
 
 /** Look up a rule's metadata — weight, title, docs link, suggested fix. */

packages/cli/src/lib/map/scan.ts

@@ -1,11 +1,12 @@
 import { join } from 'node:path'
+import { version as CLI_VERSION } from '../../../package.json'
 import { getAdapter } from './adapters/index'
 import { countSuppressed } from './directives'
 import { buildFileFacts } from './facts'
 import { createParseCache, parseFile } from './parse'
 import { collectProjectFacts, readPackageJson } from './project-facts'
 import type { ProjectFacts } from './project-facts'
-import { getRule, runRules } from './rules/index'
+import { RULE_SET_VERSION, getRule, runRules } from './rules/index'
 import type { FrameworkCapabilities } from './rules/index'
 import { classifySensitivity } from './sensitivity'
 import { classifyRouteObservability, gradeFromScore, scoreGlobal, scoreRoute } from './score'
@@ -106,6 +107,8 @@ export async function scan(input: ScanContext): Promise<ScanResult> {
   const map: MapFile = {
     version: 1,
     generatedAt: new Date().toISOString(),
+    cliVersion: CLI_VERSION,
+    ruleSetVersion: RULE_SET_VERSION,
     framework: ctx.framework,
     projectName: ctx.projectName,
     score: globalScore,

packages/cli/src/lib/map/types.ts

@@ -85,6 +85,21 @@ export interface RouteEntry extends RawRouteEntry {
 export interface MapFile {
   version: 1
   generatedAt: string
+  /**
+   * The CLI version that wrote this map, for the `--baseline` compatibility
+   * check. Absent on maps written before version reporting existed, which the
+   * baseline check treats as "unknown" rather than failing.
+   */
+  cliVersion?: string
+  /**
+   * Rule-set version, bumped only when a rule's semantics change.
+   *
+   * A package release that adds an unrelated feature must not force every
+   * project to regenerate its baseline, so this is not the package version: it
+   * moves only when a verdict this map records could change for code the PR
+   * did not touch. Absent on pre-versioning maps, treated as "unknown".
+   */
+  ruleSetVersion?: number
   framework: Framework
   projectName: string
   score: number

packages/cli/src/lib/map/write.ts

@@ -35,10 +35,11 @@ export function serializeMapFile(map: MapFile): string {
   return JSON.stringify(sortedRoutes(map), null, 2)
 }
 
-/** {@link MapFile} with `generatedAt` redacted — stable across test runs for snapshotting. */
-export function mapForSnapshot(map: MapFile): Omit<MapFile, 'generatedAt'> & { generatedAt: '[REDACTED]' } {
+/** {@link MapFile} with `generatedAt` and `cliVersion` redacted — stable across test runs for snapshotting. Both churn on every release, while `ruleSetVersion` stays, so a snapshot only moves when the rule set actually changes. */
+export function mapForSnapshot(map: MapFile): Omit<MapFile, 'generatedAt' | 'cliVersion'> & { generatedAt: '[REDACTED]', cliVersion: '[REDACTED]' } {
   return {
     ...sortedRoutes(map),
     generatedAt: '[REDACTED]',
+    cliVersion: '[REDACTED]',
   }
 }

packages/cli/test/map.command.test.ts

@@ -1,17 +1,18 @@
 import { existsSync } from 'node:fs'
-import { cp, mkdtemp, readFile, rm } from 'node:fs/promises'
+import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { runCommand } from 'citty'
 import { afterEach, describe, expect, it, vi } from 'vitest'
+import { version } from '../package.json'
 import map, { formatMapReport, runMap } from '../src/commands/map'
 import type { MapResult } from '../src/commands/map'
 import { createContext } from '../src/core/context'
 import type { CliContext } from '../src/core/context'
 import { SCHEMA_VERSION } from '../src/core/output'
 import { resolveCliEnvironment } from '../src/lib/environment'
 import { MIN_WIDTH, formatMapInspect } from '../src/lib/map/report'
-import { REQUIREMENTS } from '../src/lib/map/rules/index'
+import { REQUIREMENTS, RULE_SET_VERSION } from '../src/lib/map/rules/index'
 import type { RouteEntry } from '../src/lib/map/types'
 
 const FIXTURES = join(import.meta.dirname, 'map/fixtures')
@@ -71,6 +72,15 @@ describe('runMap', () => {
     expect(result.framework).toBe('tanstack-start')
   })
 
+  it('writes the CLI and rule set versions that wrote the map', async () => {
+    const cwd = await copyFixture('nuxt-basic')
+    const result = await runMap(fakeContext(cwd))
+
+    const written = JSON.parse(await readFile(result.mapPath!, 'utf-8')) as { cliVersion: string, ruleSetVersion: number }
+    expect(written.ruleSetVersion).toBe(RULE_SET_VERSION)
+    expect(written.cliVersion).toBe(version)
+  })
+
   it('throws a catalog error for an unsupported project', async () => {
     const cwd = await mkdtemp(join(tmpdir(), 'evlog-cli-map-unsupported-'))
     tempDirs.push(cwd)
@@ -353,6 +363,27 @@ describe('map command', () => {
     expect(existsSync(join(cwd, 'evlog.map.json'))).toBe(false)
   })
 
+  it('refuses a --baseline written by a different rule set (exit 2) instead of diffing', async () => {
+    const cwd = await copyFixture('nuxt-basic')
+    vi.spyOn(process.stdout, 'write').mockImplementation(() => true)
+    vi.spyOn(process.stderr, 'write').mockImplementation(() => true)
+    await writeFile(join(cwd, 'evlog.map.json'), JSON.stringify({
+      version: 1,
+      generatedAt: '2026-01-01T00:00:00.000Z',
+      cliVersion: '0.3.0',
+      ruleSetVersion: RULE_SET_VERSION - 1,
+      framework: 'nuxt',
+      projectName: 'test',
+      score: 100,
+      routes: [],
+    }), 'utf8')
+
+    await runCommand(map, { rawArgs: ['--cwd', cwd, '--json', '--no-header', '--no-write', '--baseline'] })
+
+    /* A stale rule set is a usage error, not a check failure. */
+    expect(process.exitCode).toBe(2)
+  })
+
   it('leaves the exit code untouched without --min-score', async () => {
     const cwd = join(FIXTURES, 'nuxt-basic')
     vi.spyOn(process.stdout, 'write').mockImplementation(() => true)

packages/cli/test/map/__snapshots__/scan.test.ts.snap

@@ -2,6 +2,7 @@
 
 exports[`full scan snapshots > next-app-router map snapshot 1`] = `
 {
+  "cliVersion": "[REDACTED]",
   "framework": "next",
   "generatedAt": "[REDACTED]",
   "projectName": "next-app-router-fixture",
@@ -360,13 +361,15 @@ exports[`full scan snapshots > next-app-router map snapshot 1`] = `
       "suggestions": {},
     },
   ],
+  "ruleSetVersion": 1,
   "score": 52,
   "version": 1,
 }
 `;
 
 exports[`full scan snapshots > nuxt-basic map snapshot 1`] = `
 {
+  "cliVersion": "[REDACTED]",
   "framework": "nuxt",
   "generatedAt": "[REDACTED]",
   "projectName": "nuxt-basic-fixture",
@@ -901,13 +904,15 @@ exports[`full scan snapshots > nuxt-basic map snapshot 1`] = `
       "suggestions": {},
     },
   ],
+  "ruleSetVersion": 1,
   "score": 56,
   "version": 1,
 }
 `;
 
 exports[`full scan snapshots > tanstack-basic map snapshot 1`] = `
 {
+  "cliVersion": "[REDACTED]",
   "framework": "tanstack-start",
   "generatedAt": "[REDACTED]",
   "projectName": "tanstack-basic-fixture",
@@ -1070,6 +1075,7 @@ exports[`full scan snapshots > tanstack-basic map snapshot 1`] = `
       "suggestions": {},
     },
   ],
+  "ruleSetVersion": 1,
   "score": 69,
   "version": 1,
 }

packages/cli/test/map/baseline.test.ts

@@ -3,7 +3,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { afterEach, describe, expect, it } from 'vitest'
-import { compareToBaseline, hasRegressed, loadBaseline } from '../../src/lib/map/baseline'
+import { checkBaselineVersion, compareToBaseline, hasRegressed, loadBaseline } from '../../src/lib/map/baseline'
+import { RULE_SET_VERSION } from '../../src/lib/map/rules/index'
 import type { BaselineSource } from '../../src/lib/map/baseline'
 import type { CheckId, CheckResult, MapFile, RouteEntry } from '../../src/lib/map/types'
 
@@ -203,3 +204,25 @@ describe('loadBaseline', () => {
     expect(() => loadBaseline(dir, 'missing.json')).toThrow(/missing\.json/)
   })
 })
+
+describe('checkBaselineVersion', () => {
+  it('accepts a baseline whose rule set matches the running CLI', () => {
+    expect(checkBaselineVersion({ ruleSetVersion: RULE_SET_VERSION, cliVersion: '0.5.1' })).toBe('ok')
+  })
+
+  it('does not gate on a CLI bump that left the rule set alone', () => {
+    /* A release that ships a new feature but no rule change must not force
+       every project to regenerate. */
+    expect(checkBaselineVersion({ ruleSetVersion: RULE_SET_VERSION, cliVersion: '0.1.0' })).toBe('ok')
+  })
+
+  it('treats a map that predates version reporting as unknown, not broken', () => {
+    expect(checkBaselineVersion({})).toBe('unknown')
+  })
+
+  it('refuses a baseline written by a different rule set', () => {
+    expect(() => checkBaselineVersion({ ruleSetVersion: RULE_SET_VERSION - 1, cliVersion: '0.3.0' }))
+      .toThrow(/rule set/)
+  })
+})
+