diff --git a/benchmarks/lens/fault-injections.json b/benchmarks/lens/fault-injections.json new file mode 100644 index 00000000..b7710ca3 --- /dev/null +++ b/benchmarks/lens/fault-injections.json @@ -0,0 +1,37 @@ +{ + "schemaVersion": 1, + "cases": [ + { + "id": "stale-provider-result", + "expected": "stale result rejected and no Pass receipt" + }, + { + "id": "provider-timeout", + "expected": "inconclusive verdict and no Pass receipt" + }, + { + "id": "provider-silence", + "expected": "inconclusive verdict and no Pass receipt" + }, + { + "id": "verifier-conflict", + "expected": "conflicting Observation and inconclusive verdict" + }, + { + "id": "file-cas-race", + "expected": "promotion rejected without partial write" + }, + { + "id": "daemon-restart", + "expected": "providers reaped and pre-restart results rejected" + }, + { + "id": "orphan-provider", + "expected": "orphan process group recovered or terminated" + }, + { + "id": "stale-receipt-gate", + "expected": "Ready and Task completion gates reject old receipt" + } + ] +} diff --git a/benchmarks/lens/fixtures/python/app.py b/benchmarks/lens/fixtures/python/app.py new file mode 100644 index 00000000..071c7ec9 --- /dev/null +++ b/benchmarks/lens/fixtures/python/app.py @@ -0,0 +1,5 @@ +from models import User, display_name + + +def greeting(user: User) -> str: + return f"Hello, {display_name(user)}" diff --git a/benchmarks/lens/fixtures/python/models.py b/benchmarks/lens/fixtures/python/models.py new file mode 100644 index 00000000..ed4320c5 --- /dev/null +++ b/benchmarks/lens/fixtures/python/models.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class User: + name: str + nickname: str | None = None + + +def display_name(user: User) -> str: + return user.name diff --git a/benchmarks/lens/fixtures/python/pyproject.toml b/benchmarks/lens/fixtures/python/pyproject.toml new file mode 100644 index 00000000..bdce10c9 --- /dev/null +++ b/benchmarks/lens/fixtures/python/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "spark-lens-python-fixture" +version = "0.0.0" +requires-python = ">=3.12" + +[tool.ruff] +target-version = "py312" diff --git a/benchmarks/lens/fixtures/rust/Cargo.toml b/benchmarks/lens/fixtures/rust/Cargo.toml new file mode 100644 index 00000000..0df18506 --- /dev/null +++ b/benchmarks/lens/fixtures/rust/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "spark-lens-rust-fixture" +version = "0.0.0" +edition = "2024" + +[lib] +path = "src/lib.rs" diff --git a/benchmarks/lens/fixtures/rust/src/lib.rs b/benchmarks/lens/fixtures/rust/src/lib.rs new file mode 100644 index 00000000..c555c07e --- /dev/null +++ b/benchmarks/lens/fixtures/rust/src/lib.rs @@ -0,0 +1,12 @@ +pub struct User { + pub name: String, + pub nickname: Option, +} + +pub fn display_name(user: &User) -> &str { + &user.name +} + +pub fn greeting(user: &User) -> String { + format!("Hello, {}", display_name(user)) +} diff --git a/benchmarks/lens/fixtures/typescript/package.json b/benchmarks/lens/fixtures/typescript/package.json new file mode 100644 index 00000000..66d7dd41 --- /dev/null +++ b/benchmarks/lens/fixtures/typescript/package.json @@ -0,0 +1,5 @@ +{ + "name": "spark-lens-typescript-fixture", + "private": true, + "type": "module" +} diff --git a/benchmarks/lens/fixtures/typescript/src/index.ts b/benchmarks/lens/fixtures/typescript/src/index.ts new file mode 100644 index 00000000..9e43c4b0 --- /dev/null +++ b/benchmarks/lens/fixtures/typescript/src/index.ts @@ -0,0 +1,5 @@ +import { displayName, type User } from "./model.js"; + +export function greeting(user: User): string { + return `Hello, ${displayName(user)}`; +} diff --git a/benchmarks/lens/fixtures/typescript/src/model.ts b/benchmarks/lens/fixtures/typescript/src/model.ts new file mode 100644 index 00000000..64ceb5ed --- /dev/null +++ b/benchmarks/lens/fixtures/typescript/src/model.ts @@ -0,0 +1,8 @@ +export interface User { + name: string; + nickname?: string; +} + +export function displayName(user: User): string { + return user.name; +} diff --git a/benchmarks/lens/fixtures/typescript/tsconfig.json b/benchmarks/lens/fixtures/typescript/tsconfig.json new file mode 100644 index 00000000..7aa8834f --- /dev/null +++ b/benchmarks/lens/fixtures/typescript/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "strict": true, + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noEmit": true + }, + "include": ["src"] +} diff --git a/benchmarks/lens/measurements.json b/benchmarks/lens/measurements.json new file mode 100644 index 00000000..58694d85 --- /dev/null +++ b/benchmarks/lens/measurements.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-07-31T00:00:00.000Z", + "baseline": { + "package": "pi-lens", + "version": "3.8.73", + "commit": "dc4d6f4d5dfd0d5ddbc6a473efac9e9d1ea84d57" + }, + "protocol": { + "sameMachine": null, + "fixedModel": null, + "fixedPrompt": null, + "repetitions": null + }, + "correctness": { + "acceptedStaleResults": null, + "passesWithoutAffirmativeClean": null, + "staleReceiptBypasses": null + }, + "capabilities": { + "typescript": { + "covered": [], + "independentDiagnosticProviders": null + }, + "python": { + "covered": [], + "independentDiagnosticProviders": null + }, + "rust": { + "covered": [], + "independentDiagnosticProviders": null + } + }, + "concurrency": { + "maxProcessesPerProviderWorktree": null, + "fourSessionRssRatio": null, + "orphanProvidersAfterRestart": null, + "acceptedPreRestartResults": null + }, + "performance": { + "warmP95Ratio": { + "typescript": null, + "python": null, + "rust": null + }, + "warmGeomeanSpeedup": null, + "coldP95Ratio": null, + "discoveryHitRate": null, + "visibleTokenRatio": null + }, + "agentEffect": { + "sparkCompletionRate": null, + "baselineCompletionRate": null, + "sparkDefectRecall": null, + "baselineDefectRecall": null, + "sparkFalsePositiveRate": null, + "baselineFalsePositiveRate": null, + "invalidWorkReduction": null + }, + "evidenceRefs": [] +} diff --git a/benchmarks/lens/scorecard.json b/benchmarks/lens/scorecard.json new file mode 100644 index 00000000..3311573a --- /dev/null +++ b/benchmarks/lens/scorecard.json @@ -0,0 +1,272 @@ +{ + "schemaVersion": 1, + "generator": "scripts/run-lens-scorecard.mts", + "generatedAt": "2026-07-31T12:10:41.304Z", + "fixtureDigest": "a481d67971029670ae967f8be6db9565654072061d3247658c7a5b6159ac2305", + "measurements": { + "schemaVersion": 1, + "generatedAt": "2026-07-31T00:00:00.000Z", + "baseline": { + "package": "pi-lens", + "version": "3.8.73", + "commit": "dc4d6f4d5dfd0d5ddbc6a473efac9e9d1ea84d57" + }, + "protocol": { + "sameMachine": null, + "fixedModel": null, + "fixedPrompt": null, + "repetitions": null + }, + "correctness": { + "acceptedStaleResults": null, + "passesWithoutAffirmativeClean": null, + "staleReceiptBypasses": null + }, + "capabilities": { + "typescript": { + "covered": [], + "independentDiagnosticProviders": null + }, + "python": { + "covered": [], + "independentDiagnosticProviders": null + }, + "rust": { + "covered": [], + "independentDiagnosticProviders": null + } + }, + "concurrency": { + "maxProcessesPerProviderWorktree": null, + "fourSessionRssRatio": null, + "orphanProvidersAfterRestart": null, + "acceptedPreRestartResults": null + }, + "performance": { + "warmP95Ratio": { + "typescript": null, + "python": null, + "rust": null + }, + "warmGeomeanSpeedup": null, + "coldP95Ratio": null, + "discoveryHitRate": null, + "visibleTokenRatio": null + }, + "agentEffect": { + "sparkCompletionRate": null, + "baselineCompletionRate": null, + "sparkDefectRecall": null, + "baselineDefectRecall": null, + "sparkFalsePositiveRate": null, + "baselineFalsePositiveRate": null, + "invalidWorkReduction": null + }, + "evidenceRefs": [] + }, + "gates": { + "correctness": { + "status": "pending", + "checks": [ + { + "id": "correctness.accepted_stale_results", + "status": "pending", + "threshold": "<= 0", + "value": null + }, + { + "id": "correctness.pass_without_affirmative_clean", + "status": "pending", + "threshold": "<= 0", + "value": null + }, + { + "id": "correctness.stale_receipt_bypasses", + "status": "pending", + "threshold": "<= 0", + "value": null + } + ] + }, + "capabilities": { + "status": "pending", + "checks": [ + { + "id": "capabilities.typescript.surface", + "status": "pending", + "threshold": "includes diagnostics, navigation, format, structural_search, impact, triage, patch_proposal, project_verification", + "value": [] + }, + { + "id": "capabilities.typescript.diagnostic_providers", + "status": "pending", + "threshold": ">= 2", + "value": null + }, + { + "id": "capabilities.python.surface", + "status": "pending", + "threshold": "includes diagnostics, navigation, format, structural_search, impact, triage, patch_proposal, project_verification", + "value": [] + }, + { + "id": "capabilities.python.diagnostic_providers", + "status": "pending", + "threshold": ">= 2", + "value": null + }, + { + "id": "capabilities.rust.surface", + "status": "pending", + "threshold": "includes diagnostics, navigation, format, structural_search, impact, triage, patch_proposal, project_verification", + "value": [] + }, + { + "id": "capabilities.rust.diagnostic_providers", + "status": "pending", + "threshold": ">= 2", + "value": null + } + ] + }, + "concurrency": { + "status": "pending", + "checks": [ + { + "id": "concurrency.processes_per_provider_worktree", + "status": "pending", + "threshold": "<= 1", + "value": null + }, + { + "id": "concurrency.four_session_rss_ratio", + "status": "pending", + "threshold": "<= 0.6", + "value": null + }, + { + "id": "concurrency.orphan_providers_after_restart", + "status": "pending", + "threshold": "<= 0", + "value": null + }, + { + "id": "concurrency.accepted_pre_restart_results", + "status": "pending", + "threshold": "<= 0", + "value": null + } + ] + }, + "performance": { + "status": "pending", + "checks": [ + { + "id": "performance.typescript.warm_p95_ratio", + "status": "pending", + "threshold": "<= 1", + "value": null + }, + { + "id": "performance.python.warm_p95_ratio", + "status": "pending", + "threshold": "<= 1", + "value": null + }, + { + "id": "performance.rust.warm_p95_ratio", + "status": "pending", + "threshold": "<= 1", + "value": null + }, + { + "id": "performance.warm_geomean_speedup", + "status": "pending", + "threshold": ">= 0.2", + "value": null + }, + { + "id": "performance.cold_p95_ratio", + "status": "pending", + "threshold": "<= 1.1", + "value": null + }, + { + "id": "performance.discovery_hit_rate", + "status": "pending", + "threshold": ">= 0.95", + "value": null + }, + { + "id": "performance.visible_token_ratio", + "status": "pending", + "threshold": "<= 0.7", + "value": null + } + ] + }, + "agentEffect": { + "status": "pending", + "checks": [ + { + "id": "agent.protocol_same_machine", + "status": "pending", + "threshold": "true", + "value": null + }, + { + "id": "agent.protocol_fixed_model", + "status": "pending", + "threshold": "true", + "value": null + }, + { + "id": "agent.protocol_fixed_prompt", + "status": "pending", + "threshold": "true", + "value": null + }, + { + "id": "agent.protocol_repetitions", + "status": "pending", + "threshold": ">= 3", + "value": null + }, + { + "id": "agent.completion_rate", + "status": "pending", + "threshold": "Spark >= pi-lens", + "value": { + "spark": null, + "piLens": null + } + }, + { + "id": "agent.defect_recall", + "status": "pending", + "threshold": "Spark >= pi-lens", + "value": { + "spark": null, + "piLens": null + } + }, + { + "id": "agent.false_positive_rate", + "status": "pending", + "threshold": "Spark <= pi-lens", + "value": { + "spark": null, + "piLens": null + } + }, + { + "id": "agent.invalid_work_reduction", + "status": "pending", + "threshold": ">= 0.2", + "value": null + } + ] + } + }, + "overall": "pending" +} diff --git a/benchmarks/lens/tasks.json b/benchmarks/lens/tasks.json new file mode 100644 index 00000000..71d60269 --- /dev/null +++ b/benchmarks/lens/tasks.json @@ -0,0 +1,111 @@ +{ + "schemaVersion": 1, + "promptTemplate": "Repair the injected defect, minimize source reads and edits, and finish only after project verification passes.", + "tasks": [ + { + "id": "typescript-null-boundary", + "language": "typescript", + "fixture": "fixtures/typescript", + "mutation": { + "path": "src/model.ts", + "find": "return user.name;", + "replace": "return user.nickname;" + }, + "capabilities": [ + "diagnostics", + "navigation", + "format", + "patch_proposal", + "project_verification" + ] + }, + { + "id": "typescript-impact-rename", + "language": "typescript", + "fixture": "fixtures/typescript", + "mutation": { + "path": "src/model.ts", + "find": "displayName", + "replace": "label" + }, + "capabilities": ["structural_search", "impact", "triage", "patch_proposal"] + }, + { + "id": "typescript-stale-fix", + "language": "typescript", + "fixture": "fixtures/typescript", + "faultInjection": "stale-provider-result", + "capabilities": ["diagnostics", "patch_proposal", "project_verification"] + }, + { + "id": "python-optional-boundary", + "language": "python", + "fixture": "fixtures/python", + "mutation": { + "path": "app.py", + "find": "return user.name", + "replace": "return user.nickname" + }, + "capabilities": [ + "diagnostics", + "navigation", + "format", + "patch_proposal", + "project_verification" + ] + }, + { + "id": "python-impact-rename", + "language": "python", + "fixture": "fixtures/python", + "mutation": { + "path": "models.py", + "find": "display_name", + "replace": "label" + }, + "capabilities": ["structural_search", "impact", "triage", "patch_proposal"] + }, + { + "id": "python-provider-conflict", + "language": "python", + "fixture": "fixtures/python", + "faultInjection": "verifier-conflict", + "capabilities": ["diagnostics", "triage", "project_verification"] + }, + { + "id": "rust-option-boundary", + "language": "rust", + "fixture": "fixtures/rust", + "mutation": { + "path": "src/lib.rs", + "find": "&user.name", + "replace": "user.nickname.as_deref().unwrap()" + }, + "capabilities": [ + "diagnostics", + "navigation", + "format", + "patch_proposal", + "project_verification" + ] + }, + { + "id": "rust-impact-rename", + "language": "rust", + "fixture": "fixtures/rust", + "mutation": { + "path": "src/lib.rs", + "find": "display_name", + "replace": "label" + }, + "capabilities": ["structural_search", "impact", "triage", "patch_proposal"] + }, + { + "id": "rust-provider-restart", + "language": "rust", + "fixture": "fixtures/rust", + "faultInjection": "daemon-restart", + "capabilities": ["diagnostics", "patch_proposal", "project_verification"] + } + ] +} diff --git a/package.json b/package.json index 202b83aa..ccc1980e 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "type": "module", "scripts": { "check": "pnpm run check:static && pnpm run check:docs && pnpm run test:unit && pnpm run test:process:source", - "check:static": "pnpm --filter @zendev-lab/spark-cockpit exec svelte-kit sync && pnpm --filter @zendev-lab/spark-docs exec astro sync && node scripts/check-architecture-ratchets.mjs && node scripts/check-npm-product.mjs && pnpm run check:evidence-surface && pnpm run check:boundaries && pnpm run check:test-quality && pnpm run check:test-ownership && node scripts/check-doc-terminology.mjs && vp fmt . --check && vp lint --quiet && pnpm run typecheck", + "check:static": "pnpm --filter @zendev-lab/spark-cockpit exec svelte-kit sync && pnpm --filter @zendev-lab/spark-docs exec astro sync && node scripts/check-architecture-ratchets.mjs && node scripts/check-npm-product.mjs && node --experimental-strip-types scripts/check-lens-release.mts && pnpm run check:evidence-surface && pnpm run check:boundaries && pnpm run check:test-quality && pnpm run check:test-ownership && node scripts/check-doc-terminology.mjs && vp fmt . --check && vp lint --quiet && pnpm run typecheck", "check:evidence-surface": "node scripts/check-evidence-surface.mjs", "check:boundaries": "depcruise --config .dependency-cruiser.cjs apps packages test", "fix": "pnpm --filter @zendev-lab/spark-cockpit exec svelte-kit sync && vp fmt . --write && vp lint --fix --quiet && pnpm run typecheck", @@ -92,6 +92,7 @@ "@j178/prek": "0.4.11", "@types/node": "^26.1.1", "@zendev-lab/spark-daemon-client": "workspace:*", + "@zendev-lab/spark-lens": "workspace:^", "@zendev-lab/spark-system": "workspace:*", "dependency-cruiser": "^18.1.0", "es-toolkit": "catalog:", diff --git a/packages/spark-lens/src/index.ts b/packages/spark-lens/src/index.ts index 3aaad6c6..bc8e60a7 100644 --- a/packages/spark-lens/src/index.ts +++ b/packages/spark-lens/src/index.ts @@ -1,5 +1,6 @@ export * from "./revision.ts"; export * from "./routes.ts"; +export * from "./scorecard.ts"; export * from "./language-profiles.ts"; export * from "./intelligence.ts"; export * from "./patch-proposal.ts"; diff --git a/packages/spark-lens/src/scorecard.test.ts b/packages/spark-lens/src/scorecard.test.ts new file mode 100644 index 00000000..dd18b649 --- /dev/null +++ b/packages/spark-lens/src/scorecard.test.ts @@ -0,0 +1,115 @@ +import { expect, test } from "vitest"; + +import { + evaluateLensScorecard, + LENS_REQUIRED_CAPABILITIES, + type LensScorecardMeasurements, +} from "./scorecard.ts"; + +test("keeps unmeasured results pending and fails any observed threshold violation", () => { + const pending = measurements(); + expect(evaluateLensScorecard(pending, "fixture").overall).toBe("pending"); + + const failed = measurements(); + failed.correctness.acceptedStaleResults = 1; + expect(evaluateLensScorecard(failed, "fixture")).toMatchObject({ + overall: "fail", + gates: { correctness: { status: "fail" } }, + }); +}); + +test("passes only when every capability, comparison, and protocol threshold passes", () => { + const complete = measurements(); + complete.protocol = { + sameMachine: true, + fixedModel: true, + fixedPrompt: true, + repetitions: 3, + }; + complete.correctness = { + acceptedStaleResults: 0, + passesWithoutAffirmativeClean: 0, + staleReceiptBypasses: 0, + }; + for (const language of ["typescript", "python", "rust"] as const) { + complete.capabilities[language] = { + covered: [...LENS_REQUIRED_CAPABILITIES], + independentDiagnosticProviders: 2, + }; + complete.performance.warmP95Ratio[language] = 1; + } + complete.concurrency = { + maxProcessesPerProviderWorktree: 1, + fourSessionRssRatio: 0.6, + orphanProvidersAfterRestart: 0, + acceptedPreRestartResults: 0, + }; + complete.performance = { + ...complete.performance, + warmGeomeanSpeedup: 0.2, + coldP95Ratio: 1.1, + discoveryHitRate: 0.95, + visibleTokenRatio: 0.7, + }; + complete.agentEffect = { + sparkCompletionRate: 0.8, + baselineCompletionRate: 0.8, + sparkDefectRecall: 0.9, + baselineDefectRecall: 0.9, + sparkFalsePositiveRate: 0.1, + baselineFalsePositiveRate: 0.1, + invalidWorkReduction: 0.2, + }; + expect(evaluateLensScorecard(complete, "fixture").overall).toBe("pass"); +}); + +function measurements(): LensScorecardMeasurements { + return { + schemaVersion: 1, + generatedAt: "2026-07-31T00:00:00.000Z", + baseline: { + package: "pi-lens", + version: "3.8.73", + commit: "dc4d6f4d5dfd0d5ddbc6a473efac9e9d1ea84d57", + }, + protocol: { + sameMachine: null, + fixedModel: null, + fixedPrompt: null, + repetitions: null, + }, + correctness: { + acceptedStaleResults: null, + passesWithoutAffirmativeClean: null, + staleReceiptBypasses: null, + }, + capabilities: { + typescript: { covered: [], independentDiagnosticProviders: null }, + python: { covered: [], independentDiagnosticProviders: null }, + rust: { covered: [], independentDiagnosticProviders: null }, + }, + concurrency: { + maxProcessesPerProviderWorktree: null, + fourSessionRssRatio: null, + orphanProvidersAfterRestart: null, + acceptedPreRestartResults: null, + }, + performance: { + warmP95Ratio: { typescript: null, python: null, rust: null }, + warmGeomeanSpeedup: null, + coldP95Ratio: null, + discoveryHitRate: null, + visibleTokenRatio: null, + }, + agentEffect: { + sparkCompletionRate: null, + baselineCompletionRate: null, + sparkDefectRecall: null, + baselineDefectRecall: null, + sparkFalsePositiveRate: null, + baselineFalsePositiveRate: null, + invalidWorkReduction: null, + }, + evidenceRefs: [], + }; +} diff --git a/packages/spark-lens/src/scorecard.ts b/packages/spark-lens/src/scorecard.ts new file mode 100644 index 00000000..06dd4367 --- /dev/null +++ b/packages/spark-lens/src/scorecard.ts @@ -0,0 +1,268 @@ +export const LENS_REQUIRED_CAPABILITIES = [ + "diagnostics", + "navigation", + "format", + "structural_search", + "impact", + "triage", + "patch_proposal", + "project_verification", +] as const; + +export type LensScorecardStatus = "pass" | "fail" | "pending"; +export type LensScorecardLanguage = "typescript" | "python" | "rust"; + +export interface LensScorecardMeasurements { + schemaVersion: 1; + generatedAt: string; + baseline: { + package: "pi-lens"; + version: "3.8.73"; + commit: "dc4d6f4d5dfd0d5ddbc6a473efac9e9d1ea84d57"; + }; + protocol: { + sameMachine: boolean | null; + fixedModel: boolean | null; + fixedPrompt: boolean | null; + repetitions: number | null; + }; + correctness: { + acceptedStaleResults: number | null; + passesWithoutAffirmativeClean: number | null; + staleReceiptBypasses: number | null; + }; + capabilities: Record< + LensScorecardLanguage, + { + covered: string[]; + independentDiagnosticProviders: number | null; + } + >; + concurrency: { + maxProcessesPerProviderWorktree: number | null; + fourSessionRssRatio: number | null; + orphanProvidersAfterRestart: number | null; + acceptedPreRestartResults: number | null; + }; + performance: { + warmP95Ratio: Record; + warmGeomeanSpeedup: number | null; + coldP95Ratio: number | null; + discoveryHitRate: number | null; + visibleTokenRatio: number | null; + }; + agentEffect: { + sparkCompletionRate: number | null; + baselineCompletionRate: number | null; + sparkDefectRecall: number | null; + baselineDefectRecall: number | null; + sparkFalsePositiveRate: number | null; + baselineFalsePositiveRate: number | null; + invalidWorkReduction: number | null; + }; + evidenceRefs: string[]; +} + +export interface LensScorecardCheck { + id: string; + status: LensScorecardStatus; + threshold: string; + value: unknown; +} + +export interface LensScorecardGate { + status: LensScorecardStatus; + checks: LensScorecardCheck[]; +} + +export interface LensReleaseScorecard { + schemaVersion: 1; + generator: "scripts/run-lens-scorecard.mts"; + generatedAt: string; + fixtureDigest: string; + measurements: LensScorecardMeasurements; + gates: { + correctness: LensScorecardGate; + capabilities: LensScorecardGate; + concurrency: LensScorecardGate; + performance: LensScorecardGate; + agentEffect: LensScorecardGate; + }; + overall: LensScorecardStatus; +} + +export function evaluateLensScorecard( + measurements: LensScorecardMeasurements, + fixtureDigest: string, +): LensReleaseScorecard { + const gates = { + correctness: gate([ + maximum( + "correctness.accepted_stale_results", + measurements.correctness.acceptedStaleResults, + 0, + ), + maximum( + "correctness.pass_without_affirmative_clean", + measurements.correctness.passesWithoutAffirmativeClean, + 0, + ), + maximum( + "correctness.stale_receipt_bypasses", + measurements.correctness.staleReceiptBypasses, + 0, + ), + ]), + capabilities: gate( + (["typescript", "python", "rust"] as const).flatMap((language) => { + const value = measurements.capabilities[language]; + return [ + check( + `capabilities.${language}.surface`, + value.covered.length === 0 + ? null + : LENS_REQUIRED_CAPABILITIES.every((capability) => + value.covered.includes(capability), + ), + `includes ${LENS_REQUIRED_CAPABILITIES.join(", ")}`, + value.covered, + ), + minimum( + `capabilities.${language}.diagnostic_providers`, + value.independentDiagnosticProviders, + 2, + ), + ]; + }), + ), + concurrency: gate([ + maximum( + "concurrency.processes_per_provider_worktree", + measurements.concurrency.maxProcessesPerProviderWorktree, + 1, + ), + maximum( + "concurrency.four_session_rss_ratio", + measurements.concurrency.fourSessionRssRatio, + 0.6, + ), + maximum( + "concurrency.orphan_providers_after_restart", + measurements.concurrency.orphanProvidersAfterRestart, + 0, + ), + maximum( + "concurrency.accepted_pre_restart_results", + measurements.concurrency.acceptedPreRestartResults, + 0, + ), + ]), + performance: gate([ + ...(["typescript", "python", "rust"] as const).map((language) => + maximum( + `performance.${language}.warm_p95_ratio`, + measurements.performance.warmP95Ratio[language], + 1, + ), + ), + minimum("performance.warm_geomean_speedup", measurements.performance.warmGeomeanSpeedup, 0.2), + maximum("performance.cold_p95_ratio", measurements.performance.coldP95Ratio, 1.1), + minimum("performance.discovery_hit_rate", measurements.performance.discoveryHitRate, 0.95), + maximum("performance.visible_token_ratio", measurements.performance.visibleTokenRatio, 0.7), + ]), + agentEffect: gate([ + check( + "agent.protocol_same_machine", + measurements.protocol.sameMachine, + "true", + measurements.protocol.sameMachine, + ), + check( + "agent.protocol_fixed_model", + measurements.protocol.fixedModel, + "true", + measurements.protocol.fixedModel, + ), + check( + "agent.protocol_fixed_prompt", + measurements.protocol.fixedPrompt, + "true", + measurements.protocol.fixedPrompt, + ), + minimum("agent.protocol_repetitions", measurements.protocol.repetitions, 3), + compare( + "agent.completion_rate", + measurements.agentEffect.sparkCompletionRate, + measurements.agentEffect.baselineCompletionRate, + ">=", + ), + compare( + "agent.defect_recall", + measurements.agentEffect.sparkDefectRecall, + measurements.agentEffect.baselineDefectRecall, + ">=", + ), + compare( + "agent.false_positive_rate", + measurements.agentEffect.sparkFalsePositiveRate, + measurements.agentEffect.baselineFalsePositiveRate, + "<=", + ), + minimum("agent.invalid_work_reduction", measurements.agentEffect.invalidWorkReduction, 0.2), + ]), + }; + const statuses = Object.values(gates).map((value) => value.status); + return { + schemaVersion: 1, + generator: "scripts/run-lens-scorecard.mts", + generatedAt: new Date().toISOString(), + fixtureDigest, + measurements, + gates, + overall: statuses.includes("fail") ? "fail" : statuses.includes("pending") ? "pending" : "pass", + }; +} + +function maximum(id: string, value: number | null, limit: number): LensScorecardCheck { + return check(id, value === null ? null : value <= limit, `<= ${String(limit)}`, value); +} + +function minimum(id: string, value: number | null, limit: number): LensScorecardCheck { + return check(id, value === null ? null : value >= limit, `>= ${String(limit)}`, value); +} + +function compare( + id: string, + left: number | null, + right: number | null, + operator: ">=" | "<=", +): LensScorecardCheck { + const passed = + left === null || right === null ? null : operator === ">=" ? left >= right : left <= right; + return check(id, passed, `Spark ${operator} pi-lens`, { spark: left, piLens: right }); +} + +function check( + id: string, + passed: boolean | null, + threshold: string, + value: unknown, +): LensScorecardCheck { + return { + id, + status: passed === null ? "pending" : passed ? "pass" : "fail", + threshold, + value, + }; +} + +function gate(checks: LensScorecardCheck[]): LensScorecardGate { + return { + status: checks.some((item) => item.status === "fail") + ? "fail" + : checks.some((item) => item.status === "pending") + ? "pending" + : "pass", + checks, + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a7459b34..77305ee3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -157,6 +157,9 @@ importers: '@zendev-lab/spark-daemon-client': specifier: workspace:* version: link:packages/spark-daemon-client + '@zendev-lab/spark-lens': + specifier: workspace:^ + version: link:packages/spark-lens '@zendev-lab/spark-system': specifier: workspace:* version: link:packages/spark-system diff --git a/scripts/check-lens-release.mts b/scripts/check-lens-release.mts new file mode 100644 index 00000000..db0549a4 --- /dev/null +++ b/scripts/check-lens-release.mts @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +import { readFile, readdir } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { fileURLToPath } from "node:url"; + +import { + evaluateLensScorecard, + type LensReleaseScorecard, +} from "../packages/spark-lens/src/scorecard.ts"; +import { jsonFile, lensFixtureDigest } from "./lens-scorecard-io.mts"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const scorecardPath = resolve( + root, + process.env.SPARK_LENS_SCORECARD ?? "benchmarks/lens/scorecard.json", +); +const scorecard = await jsonFile(scorecardPath); +const fixtureDigest = await lensFixtureDigest(root); +const evaluated = evaluateLensScorecard(scorecard.measurements, fixtureDigest); +const violations: string[] = []; + +if ( + scorecard.measurements.baseline.package !== "pi-lens" || + scorecard.measurements.baseline.version !== "3.8.73" || + scorecard.measurements.baseline.commit !== "dc4d6f4d5dfd0d5ddbc6a473efac9e9d1ea84d57" +) { + violations.push("scorecard baseline is not pi-lens@3.8.73 dc4d6f4"); +} +if (scorecard.fixtureDigest !== fixtureDigest) { + violations.push("scorecard fixture digest is stale"); +} +if ( + !isDeepStrictEqual(scorecard.gates, evaluated.gates) || + scorecard.overall !== evaluated.overall +) { + violations.push("scorecard gates do not match the machine evaluator"); +} + +const extensionSource = await readFile( + resolve( + root, + process.env.SPARK_LENS_REGISTRATION_SOURCE ?? "packages/spark-extension/src/extension/index.ts", + ), + "utf8", +); +const publicRegistration = /registerSparkTool\(\s*createSparkLensToolConfig\(\)\s*\)/u.test( + extensionSource, +); +const internalRegistration = + /registerSparkImplementationTool\(\s*createSparkLensToolConfig\(\)\s*\)/u.test(extensionSource); +if (!internalRegistration && !publicRegistration) { + violations.push("Lens tool registration is missing"); +} + +const publicFiles = [ + resolve(root, "README.md"), + ...(await filesUnder(resolve(root, "apps/spark-docs/src/content/docs"))), +]; +const publicText = ( + await Promise.all(publicFiles.map(async (path) => await readFile(path, "utf8"))) +).join("\n"); +const publicDocumentation = /\bSpark Lens\b|spark\s+lens\b|lens\s*\(\s*\{/iu.test(publicText); +const leadingClaim = + /(?:outperform(?:s|ed|ing)?|faster than|better than)\s+pi-lens|(?:超越|领先于)\s*pi-lens/iu.test( + publicText, + ); + +if (scorecard.overall !== "pass") { + if (publicRegistration) violations.push("Lens is public before every scorecard gate passes"); + if (publicDocumentation) { + violations.push("Lens is documented as public before every scorecard gate passes"); + } + if (leadingClaim) violations.push("Lens claims leadership before every scorecard gate passes"); +} + +if (violations.length > 0) { + throw new Error(`Spark Lens release gate failed:\n- ${violations.join("\n- ")}`); +} +process.stdout.write( + scorecard.overall === "pass" + ? "Spark Lens release scorecard passes; public enablement is permitted.\n" + : `Spark Lens release gate remains closed (${scorecard.overall}); internal registration verified.\n`, +); + +async function filesUnder(path: string): Promise { + const output: string[] = []; + for (const entry of await readdir(path, { withFileTypes: true })) { + const child = resolve(path, entry.name); + if (entry.isDirectory()) output.push(...(await filesUnder(child))); + else if (entry.isFile() && /\.mdx?$/u.test(entry.name)) output.push(child); + } + return output; +} diff --git a/scripts/lens-scorecard-io.mts b/scripts/lens-scorecard-io.mts new file mode 100644 index 00000000..fdc75008 --- /dev/null +++ b/scripts/lens-scorecard-io.mts @@ -0,0 +1,42 @@ +import { createHash } from "node:crypto"; +import { readFile, readdir } from "node:fs/promises"; +import { relative, resolve } from "node:path"; + +const FIXTURE_INPUTS = [ + "benchmarks/lens/tasks.json", + "benchmarks/lens/fault-injections.json", + "benchmarks/lens/fixtures", +] as const; + +export async function lensFixtureDigest(root: string): Promise { + const paths: string[] = []; + for (const input of FIXTURE_INPUTS) { + await collectFiles(resolve(root, input), paths); + } + paths.sort((left, right) => left.localeCompare(right)); + const hash = createHash("sha256"); + for (const path of paths) { + hash.update(relative(root, path).replaceAll("\\", "/")); + hash.update("\0"); + hash.update(await readFile(path)); + hash.update("\0"); + } + return hash.digest("hex"); +} + +export async function jsonFile(path: string): Promise { + return JSON.parse(await readFile(path, "utf8")) as T; +} + +async function collectFiles(path: string, output: string[]): Promise { + const entries = await readdir(path, { withFileTypes: true }).catch(() => undefined); + if (!entries) { + output.push(path); + return; + } + for (const entry of entries) { + const child = resolve(path, entry.name); + if (entry.isDirectory()) await collectFiles(child, output); + else if (entry.isFile()) output.push(child); + } +} diff --git a/scripts/run-lens-scorecard.mts b/scripts/run-lens-scorecard.mts new file mode 100644 index 00000000..3291cae3 --- /dev/null +++ b/scripts/run-lens-scorecard.mts @@ -0,0 +1,21 @@ +#!/usr/bin/env node + +import { writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + evaluateLensScorecard, + type LensScorecardMeasurements, +} from "../packages/spark-lens/src/scorecard.ts"; +import { jsonFile, lensFixtureDigest } from "./lens-scorecard-io.mts"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const measurementsPath = resolve(root, process.argv[2] ?? "benchmarks/lens/measurements.json"); +const outputPath = resolve(root, process.argv[3] ?? "benchmarks/lens/scorecard.json"); +const measurements = await jsonFile(measurementsPath); +const scorecard = evaluateLensScorecard(measurements, await lensFixtureDigest(root)); +await writeFile(outputPath, `${JSON.stringify(scorecard, null, 2)}\n`); +process.stdout.write( + `Spark Lens scorecard ${scorecard.overall}: ${outputPath.replace(`${root}/`, "")}\n`, +); diff --git a/test/lens-release-gate.test.ts b/test/lens-release-gate.test.ts new file mode 100644 index 00000000..685798f4 --- /dev/null +++ b/test/lens-release-gate.test.ts @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { test } from "vitest"; + +const execFileAsync = promisify(execFile); + +test("Lens release ratchet rejects public registration while the scorecard is pending", async () => { + const directory = await mkdtemp(join(tmpdir(), "spark-lens-release-")); + const source = join(directory, "registration.ts"); + await writeFile(source, "registerSparkTool(createSparkLensToolConfig());\n"); + + await assert.rejects( + execFileAsync( + process.execPath, + ["--experimental-strip-types", "scripts/check-lens-release.mts"], + { + cwd: resolve("."), + env: { + ...process.env, + SPARK_LENS_REGISTRATION_SOURCE: source, + }, + }, + ), + (error: unknown) => { + const failure = error as { stderr?: string }; + assert.match(failure.stderr ?? "", /Lens is public before every scorecard gate passes/u); + return true; + }, + ); +}); diff --git a/test/lens-scorecard-fixtures.test.ts b/test/lens-scorecard-fixtures.test.ts new file mode 100644 index 00000000..0345db38 --- /dev/null +++ b/test/lens-scorecard-fixtures.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { test } from "vitest"; + +import { + LENS_REQUIRED_CAPABILITIES, + evaluateLensScorecard, + type LensReleaseScorecard, +} from "@zendev-lab/spark-lens"; +import { lensFixtureDigest } from "../scripts/lens-scorecard-io.mts"; + +test("Lens benchmark fixtures cover every language surface and declared fault", async () => { + const tasks = JSON.parse(await readFile("benchmarks/lens/tasks.json", "utf8")) as { + tasks: { + id: string; + language: "typescript" | "python" | "rust"; + capabilities: string[]; + faultInjection?: string; + }[]; + }; + const faults = JSON.parse(await readFile("benchmarks/lens/fault-injections.json", "utf8")) as { + cases: { id: string }[]; + }; + const faultIds = new Set(faults.cases.map((item) => item.id)); + for (const language of ["typescript", "python", "rust"] as const) { + const languageTasks = tasks.tasks.filter((task) => task.language === language); + assert.ok(languageTasks.length >= 3, `${language} requires at least three fixed tasks`); + const capabilities = new Set(languageTasks.flatMap((task) => task.capabilities)); + assert.deepEqual( + LENS_REQUIRED_CAPABILITIES.filter((capability) => !capabilities.has(capability)), + [], + `${language} benchmark capability gap`, + ); + } + for (const task of tasks.tasks) { + if (task.faultInjection) { + assert.ok(faultIds.has(task.faultInjection), `${task.id} references an unknown fault`); + } + } + for (const required of [ + "stale-provider-result", + "provider-timeout", + "provider-silence", + "verifier-conflict", + "file-cas-race", + "daemon-restart", + "orphan-provider", + "stale-receipt-gate", + ]) { + assert.ok(faultIds.has(required), `missing fault injection ${required}`); + } +}); + +test("committed Lens scorecard is machine reproducible and remains closed", async () => { + const scorecard = JSON.parse( + await readFile("benchmarks/lens/scorecard.json", "utf8"), + ) as LensReleaseScorecard; + const fixtureDigest = await lensFixtureDigest(resolve(".")); + const evaluated = evaluateLensScorecard(scorecard.measurements, fixtureDigest); + assert.equal(scorecard.fixtureDigest, fixtureDigest); + assert.deepEqual(scorecard.gates, evaluated.gates); + assert.equal(scorecard.overall, "pending"); +}); diff --git a/test/package-script-surface.test.ts b/test/package-script-surface.test.ts index 19b58049..510c37cd 100644 --- a/test/package-script-surface.test.ts +++ b/test/package-script-surface.test.ts @@ -83,6 +83,7 @@ test("root package exposes one compact validation and release surface", async () "pnpm --filter @zendev-lab/spark-docs exec astro sync", "node scripts/check-architecture-ratchets.mjs", "node scripts/check-npm-product.mjs", + "node --experimental-strip-types scripts/check-lens-release.mts", "pnpm run check:evidence-surface", "pnpm run check:boundaries", "pnpm run check:test-quality",