From e7ecb14a72a6b8f9ae59d711f48550e6c63c8630 Mon Sep 17 00:00:00 2001 From: "evlogai[bot]" Date: Tue, 11 Aug 2026 18:52:52 +0000 Subject: [PATCH] fix: guard map --baseline against a moved rule set evlog.map.json now records the CLI version and a separate rule-set version that only changes when a rule's semantics change. On --baseline, a committed rule set that differs from the running one is a usage error (exit 2) rather than a misleading per-check diff that blames code the PR did not touch. A map written before version reporting has no version fields; it warns once instead of failing every project on upgrade. --- .changeset/calm-baselines-check.md | 5 +++ apps/docs/content/3.cli/2.map.md | 2 +- apps/docs/content/3.cli/5.ci.md | 13 ++++++- packages/cli/src/commands/map.ts | 23 +++++++++--- packages/cli/src/lib/errors.ts | 9 +++++ packages/cli/src/lib/map/baseline.ts | 24 +++++++++++++ packages/cli/src/lib/map/rules/index.ts | 12 +++++++ packages/cli/src/lib/map/scan.ts | 5 ++- packages/cli/src/lib/map/types.ts | 15 ++++++++ packages/cli/src/lib/map/write.ts | 5 +-- packages/cli/test/map.command.test.ts | 35 +++++++++++++++++-- .../test/map/__snapshots__/scan.test.ts.snap | 6 ++++ packages/cli/test/map/baseline.test.ts | 25 ++++++++++++- 13 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 .changeset/calm-baselines-check.md diff --git a/.changeset/calm-baselines-check.md b/.changeset/calm-baselines-check.md new file mode 100644 index 00000000..49c47fab --- /dev/null +++ b/.changeset/calm-baselines-check.md @@ -0,0 +1,5 @@ +--- +'@evlog/cli': patch +--- + +fix: `evlog map --baseline` records the CLI and rule-set versions in `evlog.map.json` and refuses to diff (exit 2) when the committed rule set does not match the running CLI, instead of reporting regressions caused by a rule change diff --git a/apps/docs/content/3.cli/2.map.md b/apps/docs/content/3.cli/2.map.md index 490903f9..f7947bbe 100644 --- a/apps/docs/content/3.cli/2.map.md +++ b/apps/docs/content/3.cli/2.map.md @@ -285,7 +285,7 @@ The check becomes `n/a` with your reason attached, so it stops costing score — ## evlog.map.json -Every run writes `evlog.map.json` to the project root: the score, the framework, and every entry point with its checks, its suggestions, its sensitivity, and its own score. It is the same data `--json` prints. +Every run writes `evlog.map.json` to the project root: the score, the framework, the CLI version and rule-set version that wrote it, and every entry point with its checks, its suggestions, its sensitivity, and its own score. It is the same data `--json` prints. Use `--no-write` when you do not want the file — CI runs, or a quick look at somebody else's project. The file is a build artifact, so [gitignore it](/cli/ci#the-map-file) unless you have a reason to track it. diff --git a/apps/docs/content/3.cli/5.ci.md b/apps/docs/content/3.cli/5.ci.md index e8f7ce93..6a14f88b 100644 --- a/apps/docs/content/3.cli/5.ci.md +++ b/apps/docs/content/3.cli/5.ci.md @@ -126,13 +126,24 @@ evlog map --baseline ../base-map.json # or any path With a bare `--baseline` and no map on disk, the CLI falls back to `git:HEAD` on its own, so the answer stays "what did the last commit say" rather than "what did I say a minute ago". +### When the rule set moves + +`evlog.map.json` records the CLI version that wrote it and a separate **rule-set version** that only changes when a rule's semantics change. On `--baseline`, the CLI compares the committed rule-set version against its own. If they differ, it refuses to diff and exits `2`: a rule tightened between the two versions would show up as a `pass` to `fail` transition on code the pull request did not touch, and a gate that blames the wrong thing is worse than one that admits it cannot run. + +``` +baseline was written by @evlog/cli 0.3.0, running @evlog/cli 0.4.1 (rule set 1 → 2) +regenerate the baseline: evlog map && git add evlog.map.json +``` + +A release that ships a feature but no rule change keeps the rule-set version, so upgrading the CLI does not force everyone to regenerate. A map written before version reporting has no version fields; the CLI treats it as unknown and warns once instead of failing every project on upgrade. + ## Exit codes | Code | Meaning | | --- | --- | | `0` | Score met the threshold, no regression against the baseline, or neither was requested | | `1` | Score below `--min-score`, a regression against `--baseline`, or the scan could not run | -| `2` | Usage error — unknown flag, or an invalid `--framework` | +| `2` | Usage error — unknown flag, invalid `--framework`, or a baseline whose rule set does not match the running CLI | ::warning A pipe replaces `$?` with the exit code of the last command in it, so `evlog map --min-score 90 \| tee map.log` always looks green. Add `set -o pipefail` to the step. diff --git a/packages/cli/src/commands/map.ts b/packages/cli/src/commands/map.ts index 33612df0..ba53745c 100644 --- a/packages/cli/src/commands/map.ts +++ b/packages/cli/src/commands/map.ts @@ -1,13 +1,13 @@ import { EvlogError } from 'evlog' import type { CliContext } from '../core/context' -import { EXIT_FAIL } from '../core/output' +import { EXIT_FAIL, EXIT_USAGE } from '../core/output' import { defineEvlogCommand } from '../lib/command' import type { CliDebug } from '../lib/debug' import { createNoopCliDebug } from '../lib/debug' import { cliErrors } from '../lib/errors' import { resolveEvlog, resolveProject } from '../lib/project' import type { ProjectInfo } from '../lib/project' -import { compareToBaseline, hasRegressed, loadBaseline } from '../lib/map/baseline' +import { checkBaselineVersion, compareToBaseline, hasRegressed, loadBaseline } from '../lib/map/baseline' import type { BaselineComparison } from '../lib/map/baseline' import { detectFramework } from '../lib/map/detect' import { @@ -42,6 +42,8 @@ export interface MapResult { mapPath: string | null /** Diff against the committed map, when `--baseline` was passed. */ baseline: BaselineComparison | null + /** Baseline problems that do not stop the run — a map that predates version reporting. */ + baselineWarnings: string[] } /** @@ -93,6 +95,16 @@ export async function runMap( ) : null + /* A map written before version reporting cannot prove its rule set matches + the running one, so it gets a warning instead of a gate: hard-failing every + project on upgrade would punish the ones that never saw the feature. */ + const baselineWarnings: string[] = [] + if (baselineMap && checkBaselineVersion(baselineMap.map) === 'unknown') { + baselineWarnings.push( + `the baseline ${baselineMap.source.label} predates map version reporting, so its rule set cannot be verified; regenerate it with evlog map`, + ) + } + const scanResult = await log.step( 'scan', () => scan(scanCtx), @@ -122,6 +134,7 @@ export async function runMap( scan: scanResult, mapPath, baseline, + baselineWarnings, } } @@ -143,7 +156,7 @@ export function formatMapReport( /* Framework detection and disable-comment problems share one channel: both mean "the numbers below were produced under an assumption you should see", and both have to appear above every view rather than only the default one. */ - const warnings = [...result.frameworkWarnings, ...result.scan.warnings] + const warnings = [...result.frameworkWarnings, ...result.baselineWarnings, ...result.scan.warnings] if (warnings.length > 0) { sections.push(formatMapWarnings(ctx, warnings)) } @@ -260,7 +273,9 @@ export default defineEvlogCommand('map', { json: { error: { code: error.code, message: error.message, why: error.why, fix: error.fix } }, human: error.fix ? `${error.message}\n→ ${error.fix}` : error.message, }) - ui.exit(EXIT_FAIL) + /* A baseline whose rule set does not match is a usage error, not a + check failure: the app did not get worse, the comparison is invalid. */ + ui.exit(error.code === cliErrors.MAP_BASELINE_VERSION_MISMATCH.code ? EXIT_USAGE : EXIT_FAIL) return } throw error diff --git a/packages/cli/src/lib/errors.ts b/packages/cli/src/lib/errors.ts index 99c35bc0..847ecb93 100644 --- a/packages/cli/src/lib/errors.ts +++ b/packages/cli/src/lib/errors.ts @@ -177,6 +177,15 @@ export const cliErrors = defineErrorCatalog('cli', { link: 'https://evlog.dev/cli/ci', tags: ['map', 'baseline'], }, + MAP_BASELINE_VERSION_MISMATCH: { + status: 400, + message: ({ baselineCli, runningCli, baselineRuleSet, runningRuleSet }: { baselineCli: string, runningCli: string, baselineRuleSet: number, runningRuleSet: number }) => + `baseline was written by @evlog/cli ${baselineCli}, running @evlog/cli ${runningCli} (rule set ${baselineRuleSet} \u2192 ${runningRuleSet})`, + why: 'The rule set changed between the two versions, so a per-check diff could blame code the PR did not touch', + fix: 'Regenerate the baseline: evlog map && git add evlog.map.json', + link: 'https://evlog.dev/cli/ci', + tags: ['map', 'baseline'], + }, MAP_INVALID_MIN_SCORE: { status: 400, message: ({ value }: { value: string }) => diff --git a/packages/cli/src/lib/map/baseline.ts b/packages/cli/src/lib/map/baseline.ts index fbf31cec..68a757e2 100644 --- a/packages/cli/src/lib/map/baseline.ts +++ b/packages/cli/src/lib/map/baseline.ts @@ -2,7 +2,9 @@ import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { isAbsolute, resolve } from 'node:path' import { cliErrors } from '../errors' +import { version as CLI_VERSION } from '../../../package.json' import { classifyRouteObservability, scoreGlobal } from './score' +import { RULE_SET_VERSION } from './rules/index' import type { CheckId, MapFile, RouteEntry } from './types' import { MAP_FILE_NAME } from './write' @@ -207,3 +209,25 @@ export function compareToBaseline(baseline: MapFile, current: MapFile, source: B export function hasRegressed(comparison: BaselineComparison): boolean { return comparison.regressions.length > 0 || comparison.delta < 0 } + +/** + * Whether a committed baseline is comparable to the running CLI. + * + * Returns `'unknown'` (the caller warns rather than fails) when the map + * predates version reporting, and throws a usage error when the rule set moved + * underneath the committed file. Same reasoning as a malformed `--min-score`: + * a gate that reports a regression it cannot justify is worse than one that + * admits it cannot run. + */ +export type BaselineVersionStatus = 'ok' | 'unknown' + +export function checkBaselineVersion(baseline: Pick): BaselineVersionStatus { + if (baseline.ruleSetVersion === undefined) return 'unknown' + if (baseline.ruleSetVersion === RULE_SET_VERSION) return 'ok' + throw cliErrors.MAP_BASELINE_VERSION_MISMATCH({ + baselineCli: baseline.cliVersion ?? 'unknown', + runningCli: CLI_VERSION, + baselineRuleSet: baseline.ruleSetVersion, + runningRuleSet: RULE_SET_VERSION, + }) +} diff --git a/packages/cli/src/lib/map/rules/index.ts b/packages/cli/src/lib/map/rules/index.ts index 0d8400f4..67389c55 100644 --- a/packages/cli/src/lib/map/rules/index.ts +++ b/packages/cli/src/lib/map/rules/index.ts @@ -64,6 +64,18 @@ void idsMatch */ export const RULES: readonly MapRule[] = REGISTRY +/** + * Version of the rule set as written into `evlog.map.json`. + * + * Bump this only when a rule's semantics change in a way that could flip a + * verdict for code a PR did not touch: a tightened check, a new requirement, a + * reweighting. A release that adds a map feature without moving any verdict + * must leave it alone, or every project would be forced to regenerate its + * baseline for nothing. Written beside the CLI version so `--baseline` can + * tell a stale committed map apart from a merely older one. + */ +export const RULE_SET_VERSION = 1 + const RULES_BY_ID = new Map(RULES.map(rule => [rule.id, rule])) /** Look up a rule's metadata — weight, title, docs link, suggested fix. */ diff --git a/packages/cli/src/lib/map/scan.ts b/packages/cli/src/lib/map/scan.ts index 9615be0b..df2732c8 100644 --- a/packages/cli/src/lib/map/scan.ts +++ b/packages/cli/src/lib/map/scan.ts @@ -1,11 +1,12 @@ import { join } from 'node:path' +import { version as CLI_VERSION } from '../../../package.json' import { getAdapter } from './adapters/index' import { countSuppressed } from './directives' import { buildFileFacts } from './facts' import { createParseCache, parseFile } from './parse' import { collectProjectFacts, readPackageJson } from './project-facts' import type { ProjectFacts } from './project-facts' -import { getRule, runRules } from './rules/index' +import { RULE_SET_VERSION, getRule, runRules } from './rules/index' import type { FrameworkCapabilities } from './rules/index' import { classifySensitivity } from './sensitivity' import { classifyRouteObservability, gradeFromScore, scoreGlobal, scoreRoute } from './score' @@ -106,6 +107,8 @@ export async function scan(input: ScanContext): Promise { const map: MapFile = { version: 1, generatedAt: new Date().toISOString(), + cliVersion: CLI_VERSION, + ruleSetVersion: RULE_SET_VERSION, framework: ctx.framework, projectName: ctx.projectName, score: globalScore, diff --git a/packages/cli/src/lib/map/types.ts b/packages/cli/src/lib/map/types.ts index 51ff832d..f85e4488 100644 --- a/packages/cli/src/lib/map/types.ts +++ b/packages/cli/src/lib/map/types.ts @@ -85,6 +85,21 @@ export interface RouteEntry extends RawRouteEntry { export interface MapFile { version: 1 generatedAt: string + /** + * The CLI version that wrote this map, for the `--baseline` compatibility + * check. Absent on maps written before version reporting existed, which the + * baseline check treats as "unknown" rather than failing. + */ + cliVersion?: string + /** + * Rule-set version, bumped only when a rule's semantics change. + * + * A package release that adds an unrelated feature must not force every + * project to regenerate its baseline, so this is not the package version: it + * moves only when a verdict this map records could change for code the PR + * did not touch. Absent on pre-versioning maps, treated as "unknown". + */ + ruleSetVersion?: number framework: Framework projectName: string score: number diff --git a/packages/cli/src/lib/map/write.ts b/packages/cli/src/lib/map/write.ts index 3f6b7fe8..a4d5efcf 100644 --- a/packages/cli/src/lib/map/write.ts +++ b/packages/cli/src/lib/map/write.ts @@ -35,10 +35,11 @@ export function serializeMapFile(map: MapFile): string { return JSON.stringify(sortedRoutes(map), null, 2) } -/** {@link MapFile} with `generatedAt` redacted — stable across test runs for snapshotting. */ -export function mapForSnapshot(map: MapFile): Omit & { generatedAt: '[REDACTED]' } { +/** {@link MapFile} with `generatedAt` and `cliVersion` redacted — stable across test runs for snapshotting. Both churn on every release, while `ruleSetVersion` stays, so a snapshot only moves when the rule set actually changes. */ +export function mapForSnapshot(map: MapFile): Omit & { generatedAt: '[REDACTED]', cliVersion: '[REDACTED]' } { return { ...sortedRoutes(map), generatedAt: '[REDACTED]', + cliVersion: '[REDACTED]', } } diff --git a/packages/cli/test/map.command.test.ts b/packages/cli/test/map.command.test.ts index d2634e98..bcc55b51 100644 --- a/packages/cli/test/map.command.test.ts +++ b/packages/cli/test/map.command.test.ts @@ -1,9 +1,10 @@ import { existsSync } from 'node:fs' -import { cp, mkdtemp, readFile, rm } from 'node:fs/promises' +import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { runCommand } from 'citty' import { afterEach, describe, expect, it, vi } from 'vitest' +import { version } from '../package.json' import map, { formatMapReport, runMap } from '../src/commands/map' import type { MapResult } from '../src/commands/map' import { createContext } from '../src/core/context' @@ -11,7 +12,7 @@ import type { CliContext } from '../src/core/context' import { SCHEMA_VERSION } from '../src/core/output' import { resolveCliEnvironment } from '../src/lib/environment' import { MIN_WIDTH, formatMapInspect } from '../src/lib/map/report' -import { REQUIREMENTS } from '../src/lib/map/rules/index' +import { REQUIREMENTS, RULE_SET_VERSION } from '../src/lib/map/rules/index' import type { RouteEntry } from '../src/lib/map/types' const FIXTURES = join(import.meta.dirname, 'map/fixtures') @@ -71,6 +72,15 @@ describe('runMap', () => { expect(result.framework).toBe('tanstack-start') }) + it('writes the CLI and rule set versions that wrote the map', async () => { + const cwd = await copyFixture('nuxt-basic') + const result = await runMap(fakeContext(cwd)) + + const written = JSON.parse(await readFile(result.mapPath!, 'utf-8')) as { cliVersion: string, ruleSetVersion: number } + expect(written.ruleSetVersion).toBe(RULE_SET_VERSION) + expect(written.cliVersion).toBe(version) + }) + it('throws a catalog error for an unsupported project', async () => { const cwd = await mkdtemp(join(tmpdir(), 'evlog-cli-map-unsupported-')) tempDirs.push(cwd) @@ -353,6 +363,27 @@ describe('map command', () => { expect(existsSync(join(cwd, 'evlog.map.json'))).toBe(false) }) + it('refuses a --baseline written by a different rule set (exit 2) instead of diffing', async () => { + const cwd = await copyFixture('nuxt-basic') + vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + await writeFile(join(cwd, 'evlog.map.json'), JSON.stringify({ + version: 1, + generatedAt: '2026-01-01T00:00:00.000Z', + cliVersion: '0.3.0', + ruleSetVersion: RULE_SET_VERSION - 1, + framework: 'nuxt', + projectName: 'test', + score: 100, + routes: [], + }), 'utf8') + + await runCommand(map, { rawArgs: ['--cwd', cwd, '--json', '--no-header', '--no-write', '--baseline'] }) + + /* A stale rule set is a usage error, not a check failure. */ + expect(process.exitCode).toBe(2) + }) + it('leaves the exit code untouched without --min-score', async () => { const cwd = join(FIXTURES, 'nuxt-basic') vi.spyOn(process.stdout, 'write').mockImplementation(() => true) diff --git a/packages/cli/test/map/__snapshots__/scan.test.ts.snap b/packages/cli/test/map/__snapshots__/scan.test.ts.snap index c103a8d2..be2b3293 100644 --- a/packages/cli/test/map/__snapshots__/scan.test.ts.snap +++ b/packages/cli/test/map/__snapshots__/scan.test.ts.snap @@ -2,6 +2,7 @@ exports[`full scan snapshots > next-app-router map snapshot 1`] = ` { + "cliVersion": "[REDACTED]", "framework": "next", "generatedAt": "[REDACTED]", "projectName": "next-app-router-fixture", @@ -360,6 +361,7 @@ exports[`full scan snapshots > next-app-router map snapshot 1`] = ` "suggestions": {}, }, ], + "ruleSetVersion": 1, "score": 52, "version": 1, } @@ -367,6 +369,7 @@ exports[`full scan snapshots > next-app-router map snapshot 1`] = ` exports[`full scan snapshots > nuxt-basic map snapshot 1`] = ` { + "cliVersion": "[REDACTED]", "framework": "nuxt", "generatedAt": "[REDACTED]", "projectName": "nuxt-basic-fixture", @@ -901,6 +904,7 @@ exports[`full scan snapshots > nuxt-basic map snapshot 1`] = ` "suggestions": {}, }, ], + "ruleSetVersion": 1, "score": 56, "version": 1, } @@ -908,6 +912,7 @@ exports[`full scan snapshots > nuxt-basic map snapshot 1`] = ` exports[`full scan snapshots > tanstack-basic map snapshot 1`] = ` { + "cliVersion": "[REDACTED]", "framework": "tanstack-start", "generatedAt": "[REDACTED]", "projectName": "tanstack-basic-fixture", @@ -1070,6 +1075,7 @@ exports[`full scan snapshots > tanstack-basic map snapshot 1`] = ` "suggestions": {}, }, ], + "ruleSetVersion": 1, "score": 69, "version": 1, } diff --git a/packages/cli/test/map/baseline.test.ts b/packages/cli/test/map/baseline.test.ts index 9e7e2b3a..7b0d93c4 100644 --- a/packages/cli/test/map/baseline.test.ts +++ b/packages/cli/test/map/baseline.test.ts @@ -3,7 +3,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { compareToBaseline, hasRegressed, loadBaseline } from '../../src/lib/map/baseline' +import { checkBaselineVersion, compareToBaseline, hasRegressed, loadBaseline } from '../../src/lib/map/baseline' +import { RULE_SET_VERSION } from '../../src/lib/map/rules/index' import type { BaselineSource } from '../../src/lib/map/baseline' import type { CheckId, CheckResult, MapFile, RouteEntry } from '../../src/lib/map/types' @@ -203,3 +204,25 @@ describe('loadBaseline', () => { expect(() => loadBaseline(dir, 'missing.json')).toThrow(/missing\.json/) }) }) + +describe('checkBaselineVersion', () => { + it('accepts a baseline whose rule set matches the running CLI', () => { + expect(checkBaselineVersion({ ruleSetVersion: RULE_SET_VERSION, cliVersion: '0.5.1' })).toBe('ok') + }) + + it('does not gate on a CLI bump that left the rule set alone', () => { + /* A release that ships a new feature but no rule change must not force + every project to regenerate. */ + expect(checkBaselineVersion({ ruleSetVersion: RULE_SET_VERSION, cliVersion: '0.1.0' })).toBe('ok') + }) + + it('treats a map that predates version reporting as unknown, not broken', () => { + expect(checkBaselineVersion({})).toBe('unknown') + }) + + it('refuses a baseline written by a different rule set', () => { + expect(() => checkBaselineVersion({ ruleSetVersion: RULE_SET_VERSION - 1, cliVersion: '0.3.0' })) + .toThrow(/rule set/) + }) +}) +