From 840491059e6c31cf7b075c7665c2ecd8212241de Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 14 Jul 2026 08:49:31 +0100 Subject: [PATCH 1/3] feat(labeler): publisher-history context stage (W8.4 slice 1) Add a `history`-source assessment stage that produces bounded factual context from the labeler's own D1: prior releases from the publishing DID, the same artifact checksum submitted under other DIDs (global, cross-publisher), and active manual labels on the subject. History findings are operator-only context, never labels. The resolver already drops every `source: "history"` finding before any category to label mapping, and this amends the W8.1 finding contract (D4) so history findings cite a dedicated `HISTORY_FINDING_CATEGORIES` set, disjoint from the automated-block union warning vocabulary; the non-history validation path is unchanged. An orchestrator test proves a history finding flows through validation and resolution but never becomes an issued label. The stage is best-effort: because it can never affect labels, a failure to gather context must never fail the run and discard the other stages' findings, so it swallows its own errors and returns no findings. Sensitive specifics (cross-publisher checksum reuse, active manual-label identities) stay in `privateDetail`, never in the public-facing title/summary. Adds migration 0006 (partial index on `assessments.artifact_checksum`) for the cross-DID checksum lookup. The two aggregator-sourced inputs (handle/profile changes, verification state) are deferred to a later slice gated on a ratified read path. Claude-Session: https://claude.ai/code/session_014rikrGMh25h1rJczUQyTMM --- .../implementation-plan.md | 2 +- .../0006_assessment_checksum_index.sql | 9 + apps/labeler/src/assessment-orchestrator.ts | 17 +- apps/labeler/src/assessment-store.ts | 48 +++ apps/labeler/src/findings.ts | 33 +- apps/labeler/src/history-context.ts | 168 ++++++++++ .../test/assessment-orchestrator.test.ts | 37 ++- apps/labeler/test/findings.test.ts | 46 ++- apps/labeler/test/history-context.test.ts | 290 ++++++++++++++++++ 9 files changed, 642 insertions(+), 8 deletions(-) create mode 100644 apps/labeler/migrations/0006_assessment_checksum_index.sql create mode 100644 apps/labeler/src/history-context.ts create mode 100644 apps/labeler/test/history-context.test.ts diff --git a/.opencode/plans/plugin-registry-labelling-service/implementation-plan.md b/.opencode/plans/plugin-registry-labelling-service/implementation-plan.md index 8b82ba06f..a96d20b94 100644 --- a/.opencode/plans/plugin-registry-labelling-service/implementation-plan.md +++ b/.opencode/plans/plugin-registry-labelling-service/implementation-plan.md @@ -1045,7 +1045,7 @@ History may recommend operator review but cannot automatically produce package/p Dependencies: `W2.3`, registry publisher/profile data. -Decisions (2026-07-14): history is not a new contract surface — `FindingSource` already includes `"history"` and the resolver already drops it (`policy-resolver.ts` excludes `source: "history"` before any category→label mapping; `isBlockingFinding` returns false for it), so "never auto-label" is structural, not a new guard. "Recommend review" is passive: a history finding surfaces through the existing assessment-detail projection an operator already opens; no `operational_events` push and no new "flag-for-review" operator action (D3). Two of the five inputs — recent handle/profile changes and verification state — live only in the aggregator's D1, which the labeler has no binding to reach (and `constants.ts` deliberately avoids ingesting profiles); they are deferred to a later slice gated on a ratified read-only labeler→aggregator read path (service-binding preferred over a second D1 binding), not built now (D1). Repeated-checksum detection is global (cross-publisher): a checksum reused under a different DID is the sharper signal, needing a new `idx_assessments_artifact_checksum` and a cross-DID lookup (D2). Prior releases, active manual labels (reuse `getActiveLabelState`, filter `!automated && active`), and repeated checksums/findings come from the labeler's own D1. The finding contract is amended (touches W8.1): `source: "history"` findings cite a dedicated history/review category set exempt from the automated-block ∪ warning constraint that `validateFinding` enforces, rather than borrowing a warn category (D4). Neutral display facts (prior-release count, verification badge) are assembled at console read-time in the existing subject-history endpoint, not persisted as a pipeline `ContextBundle` (D5). The stage is DB-bound (`analyzeHistory(db, assessment, opts) → NormalizedFinding[]`), wired into `OrchestratorStages`/`STAGE_ORDER`/`stubStages` and run last, wrapping D1 failures as `StageTransientError`. Sliced: (1) own-D1 history findings + the W8.1 amendment + the checksum index — unblocked; (2) console neutral-context surfacing — unblocked, optional; (3) the two aggregator-sourced inputs — blocked on D1's read path. +Decisions (2026-07-14): history is not a new contract surface — `FindingSource` already includes `"history"` and the resolver already drops it (`policy-resolver.ts` excludes `source: "history"` before any category→label mapping; `isBlockingFinding` returns false for it), so "never auto-label" is structural, not a new guard. "Recommend review" is passive: a history finding surfaces through the existing assessment-detail projection an operator already opens; no `operational_events` push and no new "flag-for-review" operator action (D3). Two of the five inputs — recent handle/profile changes and verification state — live only in the aggregator's D1, which the labeler has no binding to reach (and `constants.ts` deliberately avoids ingesting profiles); they are deferred to a later slice gated on a ratified read-only labeler→aggregator read path (service-binding preferred over a second D1 binding), not built now (D1). Repeated-checksum detection is global (cross-publisher): a checksum reused under a different DID is the sharper signal, needing a new `idx_assessments_artifact_checksum` and a cross-DID lookup (D2). Prior releases, active manual labels (reuse `getActiveLabelState`, filter `!automated && active`), and repeated checksums/findings come from the labeler's own D1. The finding contract is amended (touches W8.1): `source: "history"` findings cite a dedicated history/review category set exempt from the automated-block ∪ warning constraint that `validateFinding` enforces, rather than borrowing a warn category (D4). Neutral display facts (prior-release count, verification badge) are assembled at console read-time in the existing subject-history endpoint, not persisted as a pipeline `ContextBundle` (D5). The stage is DB-bound (`analyzeHistory(db, assessment, opts) → NormalizedFinding[]`), wired into `OrchestratorStages`/`STAGE_ORDER`/`stubStages` and run last. Because history is operator-only context that never becomes a label, the stage is best-effort — on any error it logs and returns no findings rather than throwing, so it can never fail the run and discard the decision-relevant findings from the other stages (adversary-flagged 2026-07-14). History findings keep sensitive specifics (cross-publisher checksum reuse, active manual-label identities/existence including redactions) in `privateDetail` only, never in the public-facing `title`/`publicSummary`; slice (2)'s surfacing must additionally render history findings in the operator projection only and exclude them from any public serializer. Sliced: (1) own-D1 history findings + the W8.1 amendment + the checksum index — unblocked; (2) console neutral-context surfacing — unblocked, optional; (3) the two aggregator-sourced inputs — blocked on D1's read path. ### `W8.5` Implement versioned policy resolver diff --git a/apps/labeler/migrations/0006_assessment_checksum_index.sql b/apps/labeler/migrations/0006_assessment_checksum_index.sql new file mode 100644 index 000000000..0abf5d5b2 --- /dev/null +++ b/apps/labeler/migrations/0006_assessment_checksum_index.sql @@ -0,0 +1,9 @@ +-- Indexes the artifact checksum so the publisher-history stage (plan W8.4) can +-- run its global, cross-publisher checksum-repeat lookup — "this exact artifact +-- was also submitted under another DID" — without scanning the assessments +-- table. Partial (`WHERE artifact_checksum IS NOT NULL`) because pre-artifact +-- lifecycle states leave the column null. + +CREATE INDEX idx_assessments_artifact_checksum + ON assessments(artifact_checksum) + WHERE artifact_checksum IS NOT NULL; diff --git a/apps/labeler/src/assessment-orchestrator.ts b/apps/labeler/src/assessment-orchestrator.ts index cbbccdcb4..3cfc23d66 100644 --- a/apps/labeler/src/assessment-orchestrator.ts +++ b/apps/labeler/src/assessment-orchestrator.ts @@ -60,9 +60,23 @@ export interface OrchestratorStages { dependency: StageAdapter; codeAi: StageAdapter; imageAi: StageAdapter; + /** Publisher-history context (plan W8.4). Runs last: its findings are + * context the resolver drops, never labels, so nothing downstream depends on + * them. Must be best-effort — a history stage that throws would fail the whole + * run and discard every other stage's findings; `analyzeHistory` swallows its + * own errors and returns `[]`. DB-bound — a real wiring passes a closure over + * `analyzeHistory`. */ + history: StageAdapter; } -const STAGE_ORDER = ["acquire", "deterministic", "dependency", "codeAi", "imageAi"] as const; +const STAGE_ORDER = [ + "acquire", + "deterministic", + "dependency", + "codeAi", + "imageAi", + "history", +] as const; export interface AssessmentOrchestratorOptions { db: D1Database; @@ -336,4 +350,5 @@ export const stubStages: OrchestratorStages = { dependency: () => Promise.resolve([]), codeAi: () => Promise.resolve([]), imageAi: () => Promise.resolve([]), + history: () => Promise.resolve([]), }; diff --git a/apps/labeler/src/assessment-store.ts b/apps/labeler/src/assessment-store.ts index 184011857..e821b3297 100644 --- a/apps/labeler/src/assessment-store.ts +++ b/apps/labeler/src/assessment-store.ts @@ -1128,6 +1128,54 @@ export async function getAllLabelsForAssessment( })); } +/** + * Distinct release URIs previously observed under a publishing DID, newest + * observation first, excluding the URI under assessment — the "prior releases + * from the DID" input to the publisher-history stage (plan W8.4). Capped at + * `limit`; the caller reports the count as bounded rather than exact. + */ +export async function getPriorReleaseUrisForDid( + db: D1Database, + input: { did: string; excludeUri: string; limit: number }, +): Promise { + const rows = await db + .prepare( + `SELECT uri, MAX(observed_at_epoch_ms) AS observed + FROM subjects + WHERE did = ? AND uri != ? + GROUP BY uri + ORDER BY observed DESC + LIMIT ?`, + ) + .bind(input.did, input.excludeUri, input.limit) + .all<{ uri: string; observed: number }>(); + return (rows.results ?? []).map((row) => row.uri); +} + +/** + * Distinct OTHER publishing DIDs that have submitted an assessment for the same + * exact artifact checksum (plan W8.4 D2) — the cross-publisher reuse signal. + * Global: it spans every publisher's assessments, not just the queried + * subject's. Capped at `limit`. + */ +export async function getPublishersSharingChecksum( + db: D1Database, + input: { checksum: string; excludeDid: string; limit: number }, +): Promise { + const rows = await db + .prepare( + `SELECT DISTINCT s.did + FROM assessments a + JOIN subjects s ON s.uri = a.uri AND s.cid = a.cid + WHERE a.artifact_checksum = ? AND s.did != ? + ORDER BY s.did + LIMIT ?`, + ) + .bind(input.checksum, input.excludeDid, input.limit) + .all<{ did: string }>(); + return (rows.results ?? []).map((row) => row.did); +} + function rowToAssessment(row: AssessmentRow): Assessment { return { id: row.id, diff --git a/apps/labeler/src/findings.ts b/apps/labeler/src/findings.ts index 4983ca648..961c5e969 100644 --- a/apps/labeler/src/findings.ts +++ b/apps/labeler/src/findings.ts @@ -35,11 +35,30 @@ export interface ModelFindingMetadata { * finding; model/image stages report the model and prompt version. */ export type FindingSourceMetadata = ToolFindingMetadata | ModelFindingMetadata; +/** + * The category vocabulary a `source: "history"` finding may cite (plan W8.4 + * D4). History findings are context, never labels — the resolver drops them + * before any category→label mapping — so they cite this dedicated set rather + * than the policy's automated-block ∪ warning label values, and + * `validateFinding` holds them to it exclusively. + */ +export type HistoryFindingCategory = + | "publisher-history" + | "shared-artifact" + | "active-manual-label"; + +export const HISTORY_FINDING_CATEGORIES: ReadonlySet = new Set([ + "publisher-history", + "shared-artifact", + "active-manual-label", +]); + export interface NormalizedFinding { source: FindingSource; - /** A label value from the policy vocabulary — validated against - * `allowedFindingCategories` (automated-block ∪ warning), never the - * eligibility or manual-system label values. */ + /** For non-history sources, a label value from the policy vocabulary — + * validated against `allowedFindingCategories` (automated-block ∪ warning), + * never the eligibility or manual-system label values. For `source: + * "history"`, one of `HISTORY_FINDING_CATEGORIES` instead (plan W8.4 D4). */ category: string; severity: FindingSeverity; confidence?: number; @@ -77,8 +96,14 @@ export function validateFinding(finding: unknown, opts: ValidateFindingOptions): if (typeof source !== "string" || !isFindingSource(source)) throw new FindingValidationError(`finding.source is invalid: ${String(source)}`); + // History findings are exempt from the automated-block ∪ warning constraint + // (plan W8.4 D4): they cite the dedicated `HISTORY_FINDING_CATEGORIES` set, + // and only that set — a history finding may not borrow a block/warn label + // value, nor a non-history finding a history category. + const allowedCategories = + source === "history" ? HISTORY_FINDING_CATEGORIES : opts.allowedCategories; const category = finding.category; - if (typeof category !== "string" || !opts.allowedCategories.has(category)) + if (typeof category !== "string" || !allowedCategories.has(category)) throw new FindingValidationError( `finding.category is not an allowed finding category: ${String(category)}`, ); diff --git a/apps/labeler/src/history-context.ts b/apps/labeler/src/history-context.ts new file mode 100644 index 000000000..b8227d9c0 --- /dev/null +++ b/apps/labeler/src/history-context.ts @@ -0,0 +1,168 @@ +/** + * Publisher-history context stage (plan W8.4, slice 1). Produces + * `source: "history"` normalized findings from the labeler's OWN D1 — prior + * releases from the publishing DID, the same artifact checksum submitted under + * other DIDs, and existing active manual labels on the subject. + * + * These findings are bounded, factual context an operator reads through the + * assessment projection. They are structurally never turned into labels: the + * resolver drops every `source: "history"` finding before any category→label + * mapping (`policy-resolver.ts`), and `validateFinding` holds them to the + * dedicated `HISTORY_FINDING_CATEGORIES` set (`findings.ts`), disjoint from the + * policy's label vocabulary. + * + * The two deferred inputs — recent handle/profile changes and verification + * state — live only in the aggregator's D1, which the labeler has no binding to + * reach; they are gated on a ratified read path (plan W8.4 D1) and not built + * here. + */ + +import { + getActiveLabelState, + getCurrentSubjectByUri, + getPriorReleaseUrisForDid, + getPublishersSharingChecksum, + type Assessment, +} from "./assessment-store.js"; +import type { NormalizedFinding } from "./findings.js"; + +const HISTORY_TOOL = "publisher-history"; +const HISTORY_TOOL_VERSION = "1"; + +const DEFAULT_PRIOR_RELEASE_LIMIT = 20; +const DEFAULT_SHARED_PUBLISHER_LIMIT = 20; +/** How many URIs/DIDs to name in a finding's private detail. */ +const SAMPLE_SIZE = 5; + +export interface HistoryContextOptions { + /** The labeler's own DID (`src`) — the stream whose active manual labels + * count as context. */ + src: string; + priorReleaseLimit?: number; + sharedPublisherLimit?: number; + now?: Date; +} + +/** + * DB-bound stage adapter matching the orchestrator's stage contract: returns + * `NormalizedFinding[]`, one per input that has something to report. History is + * operator-only context that never becomes a label, so a failure to gather it + * must never fail the assessment run — that would discard the decision-relevant + * findings from every other stage. The stage is best-effort: on any error it + * logs and returns no findings rather than throwing, so it can never gate the + * run regardless of its position in `STAGE_ORDER`. + */ +export async function analyzeHistory( + db: D1Database, + assessment: Assessment, + opts: HistoryContextOptions, +): Promise { + const priorReleaseLimit = opts.priorReleaseLimit ?? DEFAULT_PRIOR_RELEASE_LIMIT; + const sharedPublisherLimit = opts.sharedPublisherLimit ?? DEFAULT_SHARED_PUBLISHER_LIMIT; + + try { + const findings: NormalizedFinding[] = []; + const subject = await getCurrentSubjectByUri(db, assessment.uri); + + if (subject) { + const priorUris = await getPriorReleaseUrisForDid(db, { + did: subject.did, + excludeUri: assessment.uri, + limit: priorReleaseLimit, + }); + if (priorUris.length > 0) + findings.push(priorReleasesFinding(subject.did, priorUris, priorReleaseLimit)); + + if (assessment.artifactChecksum) { + const otherDids = await getPublishersSharingChecksum(db, { + checksum: assessment.artifactChecksum, + excludeDid: subject.did, + limit: sharedPublisherLimit, + }); + if (otherDids.length > 0) + findings.push( + sharedArtifactFinding(assessment.artifactChecksum, otherDids, sharedPublisherLimit), + ); + } + } + + const labelState = await getActiveLabelState(db, { + src: opts.src, + uri: assessment.uri, + cid: assessment.cid, + ...(opts.now !== undefined ? { now: opts.now } : {}), + }); + const activeManualLabels = [...labelState.values()] + .filter((winner) => !winner.automated && winner.active) + .map((winner) => winner.val); + if (activeManualLabels.length > 0) findings.push(activeManualLabelFinding(activeManualLabels)); + + return findings; + } catch (err) { + console.error( + `[history-context] lookup failed, skipping context for this run: ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } +} + +function toolMetadata() { + return { kind: "tool", tool: HISTORY_TOOL, version: HISTORY_TOOL_VERSION } as const; +} + +function countText(count: number, capped: boolean): string { + return capped ? `at least ${count}` : `${count}`; +} + +function pluralize(count: number, noun: string): string { + return count === 1 ? noun : `${noun}s`; +} + +function priorReleasesFinding(did: string, uris: string[], limit: number): NormalizedFinding { + const capped = uris.length >= limit; + const count = countText(uris.length, capped); + return { + source: "history", + category: "publisher-history", + severity: "info", + title: `Publisher has ${count} prior ${pluralize(uris.length, "release")}`, + publicSummary: `The publishing account has ${count} other ${pluralize(uris.length, "release")} known to the labeler.`, + privateDetail: `Publisher ${did} has ${count} prior ${pluralize(uris.length, "release")}. Sample: ${uris.slice(0, SAMPLE_SIZE).join(", ")}.`, + evidenceRefs: [], + sourceMetadata: toolMetadata(), + }; +} + +function sharedArtifactFinding(checksum: string, dids: string[], limit: number): NormalizedFinding { + const capped = dids.length >= limit; + const count = countText(dids.length, capped); + // Cross-publisher artifact reuse is a correlation/deanonymization signal, so + // the title and publicSummary stay non-revealing — the specifics live only in + // privateDetail. + return { + source: "history", + category: "shared-artifact", + severity: "low", + title: "Artifact provenance context", + publicSummary: "Operator-only artifact-provenance context is recorded for this release.", + privateDetail: `Artifact checksum ${checksum} also submitted by ${count} other ${pluralize(dids.length, "publisher")}: ${dids.slice(0, SAMPLE_SIZE).join(", ")}.`, + evidenceRefs: [], + sourceMetadata: toolMetadata(), + }; +} + +function activeManualLabelFinding(vals: string[]): NormalizedFinding { + // The existence and identity of manual labels (including redactions) must not + // leak into a public projection, so the title and publicSummary reveal + // nothing — the label values live only in privateDetail. + return { + source: "history", + category: "active-manual-label", + severity: "info", + title: "Operator label context", + publicSummary: "Operator-only label context is recorded for this subject.", + privateDetail: `Active manual labels: ${vals.join(", ")}.`, + evidenceRefs: [], + sourceMetadata: toolMetadata(), + }; +} diff --git a/apps/labeler/test/assessment-orchestrator.test.ts b/apps/labeler/test/assessment-orchestrator.test.ts index 45c1c43de..bb93c5430 100644 --- a/apps/labeler/test/assessment-orchestrator.test.ts +++ b/apps/labeler/test/assessment-orchestrator.test.ts @@ -23,7 +23,8 @@ import { getCurrentAssessment, transitionAssessmentState, } from "../src/assessment-store.js"; -import { FindingValidationError } from "../src/findings.js"; +import { FindingValidationError, HISTORY_FINDING_CATEGORIES } from "../src/findings.js"; +import { analyzeHistory } from "../src/history-context.js"; import { MODERATION_POLICY } from "../src/policy.js"; import { issueManualLabel } from "../src/service.js"; import { initializeSigningState } from "../src/signing-rotation.js"; @@ -699,6 +700,40 @@ describe("AssessmentOrchestrator: invalid findings", () => { }); }); +describe("AssessmentOrchestrator: history stage never auto-labels (W8.4)", () => { + it("runs the history stage, whose finding surfaces but is never turned into an issued label", async () => { + const run = await pendingRun({ + name: "history-invariant", + cidValue: await cid("history-invariant"), + }); + + // Prior releases exist under PUBLISHER_DID (every run in this suite shares + // it), so the history stage genuinely produces a publisher-history finding + // — the invariant below is non-vacuous. + const assessment = await getAssessment(testEnv.DB, run.id); + const produced = await analyzeHistory(testEnv.DB, assessment!, { src: LABELER_DID }); + expect(produced.some((f) => HISTORY_FINDING_CATEGORIES.has(f.category))).toBe(true); + + const stages: OrchestratorStages = { + ...stubStages, + history: (ctx) => analyzeHistory(testEnv.DB, ctx.assessment, { src: LABELER_DID }), + }; + const result = await (await buildOrchestrator(stages)).runAssessment(run.id); + + // The history finding flowed through validation (it did not abort the run) + // and resolution, yet produced no blocking or warning outcome. + expect(result.state).toBe("passed"); + + // No history-category value is ever issued as a label — the resolver drops + // every history-source finding before any category→label mapping. + const labels = await testEnv.DB.prepare(`SELECT val FROM issued_labels WHERE uri = ?`) + .bind(run.uri) + .all<{ val: string }>(); + const issued = (labels.results ?? []).map((row) => row.val); + for (const category of HISTORY_FINDING_CATEGORIES) expect(issued).not.toContain(category); + }); +}); + describe("AssessmentOrchestrator: supersession negation provenance (decision 6)", () => { it("negates the prior run's automated block label but never a manually-issued label", async () => { const name = "supersede-manual-survives"; diff --git a/apps/labeler/test/findings.test.ts b/apps/labeler/test/findings.test.ts index 4b55ef357..50ed70348 100644 --- a/apps/labeler/test/findings.test.ts +++ b/apps/labeler/test/findings.test.ts @@ -4,6 +4,7 @@ import type { PublicFindingView } from "../src/evidence.js"; import { allowedFindingCategories, FindingValidationError, + HISTORY_FINDING_CATEGORIES, toPublicFindingView, validateFinding, validateFindings, @@ -62,7 +63,9 @@ describe("allowedFindingCategories", () => { describe("validateFinding", () => { it("passes a valid finding for each FindingSource", () => { for (const source of ["deterministic", "capability", "model", "image", "history"]) { - const finding = validateFinding(baseFinding({ source }), { + // History findings cite their own category set, disjoint from block/warn. + const category = source === "history" ? "publisher-history" : "obfuscated-code"; + const finding = validateFinding(baseFinding({ source, category }), { allowedCategories: ALLOWED_CATEGORIES, resolvableEvidenceIds: NO_EVIDENCE, }); @@ -208,6 +211,47 @@ describe("validateFinding", () => { }); }); +describe("validateFinding: history-source categories (W8.1 D4)", () => { + const opts = { allowedCategories: ALLOWED_CATEGORIES, resolvableEvidenceIds: NO_EVIDENCE }; + + it("accepts a history finding for every history category, none of which are block/warn values", () => { + for (const category of HISTORY_FINDING_CATEGORIES) { + expect(ALLOWED_CATEGORIES.has(category)).toBe(false); + const finding = validateFinding(baseFinding({ source: "history", category }), opts); + expect(finding.source).toBe("history"); + expect(finding.category).toBe(category); + } + }); + + it("rejects a history finding that cites a block/warn category, holding history to its own set", () => { + for (const category of ["malware", "obfuscated-code"]) { + expect(() => validateFinding(baseFinding({ source: "history", category }), opts)).toThrow( + FindingValidationError, + ); + } + }); + + it("rejects a NON-history finding that cites a history category", () => { + for (const source of ["deterministic", "capability", "model", "image"]) { + expect(() => + validateFinding(baseFinding({ source, category: "publisher-history" }), opts), + ).toThrow(FindingValidationError); + } + }); + + it("leaves non-history validation unchanged: a deterministic finding still needs a block/warn category", () => { + expect(() => + validateFinding(baseFinding({ source: "deterministic", category: "malware" }), opts), + ).not.toThrow(); + expect(() => + validateFinding( + baseFinding({ source: "deterministic", category: "active-manual-label" }), + opts, + ), + ).toThrow(FindingValidationError); + }); +}); + describe("validateFindings", () => { it("validates every finding and returns them in order", () => { const findings = validateFindings( diff --git a/apps/labeler/test/history-context.test.ts b/apps/labeler/test/history-context.test.ts new file mode 100644 index 000000000..1169b58eb --- /dev/null +++ b/apps/labeler/test/history-context.test.ts @@ -0,0 +1,290 @@ +import { createLabelSigner, type LabelSigner } from "@emdash-cms/registry-moderation"; +import { applyD1Migrations, env } from "cloudflare:test"; +import { beforeAll, describe, expect, it } from "vitest"; + +import { createAssessmentRun, createSubject, type Assessment } from "../src/assessment-store.js"; +import { + allowedFindingCategories, + HISTORY_FINDING_CATEGORIES, + validateFindings, +} from "../src/findings.js"; +import { analyzeHistory } from "../src/history-context.js"; +import { MODERATION_POLICY } from "../src/policy.js"; +import { issueManualLabel } from "../src/service.js"; +import { initializeSigningState } from "../src/signing-rotation.js"; + +interface TestEnv { + DB: D1Database; + TEST_MIGRATIONS: Parameters[1]; +} + +const testEnv = env as unknown as TestEnv; +const LABELER_DID = "did:web:labels.emdashcms.com"; +const PRIVATE_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE"; +const MULTIKEY = "zDnaepsL7AXenJkVYdkh5KuKsSU7Ykh7kyXaLLU7auN9FWSiZ"; +const config = { labelerDid: LABELER_DID, signingKeyVersion: "v1" }; + +let counter = 0; + +beforeAll(async () => { + await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS); + await initializeSigningState(testEnv.DB, { + issuerDid: LABELER_DID, + keyVersion: "v1", + publicKeyMultibase: MULTIKEY, + }); +}); + +function signer(): Promise { + return createLabelSigner({ + issuerDid: LABELER_DID, + privateKey: PRIVATE_KEY, + resolveDid: async () => ({ + id: LABELER_DID, + verificationMethod: [ + { + id: "#atproto_label", + type: "Multikey", + controller: LABELER_DID, + publicKeyMultibase: MULTIKEY, + }, + ], + }), + }); +} + +function uriFor(did: string, name: string): string { + return `at://${did}/com.emdashcms.experimental.package.release/${name}:1.0.0`; +} + +async function seedSubject(did: string, name: string): Promise<{ uri: string; cid: string }> { + const uri = uriFor(did, name); + const cid = `bafkreicid${counter++}00000000000000000000000000000000000000`; + await createSubject(testEnv.DB, { + uri, + cid, + did, + collection: "com.emdashcms.experimental.package.release", + rkey: `${name}:1.0.0`, + }); + return { uri, cid }; +} + +async function seedAssessmentWithChecksum( + subject: { uri: string; cid: string }, + checksum: string, +): Promise { + await createAssessmentRun(testEnv.DB, { + runKey: `rk-${counter++}`, + uri: subject.uri, + cid: subject.cid, + artifactChecksum: checksum, + trigger: "initial", + triggerId: `trigger-${counter++}`, + policyVersion: MODERATION_POLICY.policyVersion, + coverageJson: "{}", + }); +} + +function assessmentFor(input: { + uri: string; + cid: string; + artifactChecksum?: string | null; +}): Assessment { + return { + id: "asmt_under_test", + runKey: "rk_under_test", + uri: input.uri, + cid: input.cid, + artifactId: null, + artifactChecksum: input.artifactChecksum ?? null, + state: "running", + trigger: "initial", + triggerId: "trigger", + policyVersion: MODERATION_POLICY.policyVersion, + modelId: null, + promptHash: null, + publicSummary: null, + coverageJson: "{}", + supersedesAssessmentId: null, + startedAt: null, + completedAt: null, + createdAt: "2026-01-01T00:00:00.000Z", + }; +} + +describe("analyzeHistory", () => { + it("returns no findings for a fresh publisher with no reuse and no manual labels", async () => { + const did = "did:plc:fresh0000000000000000000000"; + const subject = await seedSubject(did, "only"); + const findings = await analyzeHistory( + testEnv.DB, + assessmentFor({ uri: subject.uri, cid: subject.cid }), + { src: LABELER_DID }, + ); + expect(findings).toEqual([]); + }); + + it("reports prior releases from the same publishing DID", async () => { + const did = "did:plc:prior000000000000000000000"; + const current = await seedSubject(did, "current"); + await seedSubject(did, "older-a"); + await seedSubject(did, "older-b"); + + const findings = await analyzeHistory( + testEnv.DB, + assessmentFor({ uri: current.uri, cid: current.cid }), + { src: LABELER_DID }, + ); + + expect(findings).toHaveLength(1); + const finding = findings[0]!; + expect(finding.source).toBe("history"); + expect(finding.category).toBe("publisher-history"); + expect(finding.title).toContain("2 prior releases"); + }); + + it("caps the prior-release count and reports it as bounded", async () => { + const did = "did:plc:manyreleases00000000000000"; + const current = await seedSubject(did, "many-current"); + for (let i = 0; i < 4; i++) await seedSubject(did, `many-${i}`); + + const findings = await analyzeHistory( + testEnv.DB, + assessmentFor({ uri: current.uri, cid: current.cid }), + { src: LABELER_DID, priorReleaseLimit: 2 }, + ); + + expect(findings[0]!.title).toContain("at least 2 prior releases"); + }); + + it("reports the same artifact checksum submitted under a different publisher (global, cross-DID)", async () => { + const checksum = `sha256-shared-${counter++}`; + const subjectA = await seedSubject("did:plc:pubA00000000000000000000000", "reuse-a"); + const subjectB = await seedSubject("did:plc:pubB00000000000000000000000", "reuse-b"); + await seedAssessmentWithChecksum(subjectB, checksum); + + const findings = await analyzeHistory( + testEnv.DB, + assessmentFor({ uri: subjectA.uri, cid: subjectA.cid, artifactChecksum: checksum }), + { src: LABELER_DID }, + ); + + const shared = findings.find((f) => f.category === "shared-artifact"); + expect(shared).toBeDefined(); + expect(shared!.source).toBe("history"); + // Cross-publisher correlation is a deanonymization signal: it must stay out + // of the public-facing title and summary, only in privateDetail. + expect(shared!.title).not.toContain("did:plc:pubB00000000000000000000000"); + expect(shared!.publicSummary).not.toContain("did:plc:pubB00000000000000000000000"); + expect(shared!.publicSummary).not.toContain("1 other"); + expect(shared!.privateDetail).toContain("1 other publisher"); + expect(shared!.privateDetail).toContain("did:plc:pubB00000000000000000000000"); + }); + + it("does not flag the same checksum submitted only under the subject's own DID", async () => { + const did = "did:plc:selfreuse0000000000000000"; + const checksum = `sha256-self-${counter++}`; + const first = await seedSubject(did, "self-a"); + const second = await seedSubject(did, "self-b"); + await seedAssessmentWithChecksum(first, checksum); + + const findings = await analyzeHistory( + testEnv.DB, + assessmentFor({ uri: second.uri, cid: second.cid, artifactChecksum: checksum }), + { src: LABELER_DID }, + ); + + expect(findings.find((f) => f.category === "shared-artifact")).toBeUndefined(); + }); + + it("reports active manual labels on the subject", async () => { + const did = "did:plc:manuallabel00000000000000"; + const subject = await seedSubject(did, "manual"); + await issueManualLabel( + testEnv.DB, + config, + await signer(), + { + actor: LABELER_DID, + type: "manual-label", + reason: "reviewer: confirmed security issue", + idempotencyKey: `manual-${counter++}`, + }, + { uri: subject.uri, val: "security-yanked" }, + ); + + const findings = await analyzeHistory( + testEnv.DB, + assessmentFor({ uri: subject.uri, cid: subject.cid }), + { src: LABELER_DID }, + ); + + const manual = findings.find((f) => f.category === "active-manual-label"); + expect(manual).toBeDefined(); + expect(manual!.source).toBe("history"); + // The existence and identity of manual labels (including redactions) must + // not leak into the public-facing fields. + expect(manual!.title).not.toContain("security-yanked"); + expect(manual!.publicSummary).not.toContain("security-yanked"); + expect(manual!.privateDetail).toContain("security-yanked"); + }); + + it("emits all three context findings together, each valid under the amended finding contract", async () => { + const did = "did:plc:allthree00000000000000000"; + const other = "did:plc:othercombined0000000000000"; + const checksum = `sha256-all-${counter++}`; + const current = await seedSubject(did, "all-current"); + await seedSubject(did, "all-older"); + const otherSubject = await seedSubject(other, "all-other"); + await seedAssessmentWithChecksum(otherSubject, checksum); + await issueManualLabel( + testEnv.DB, + config, + await signer(), + { + actor: LABELER_DID, + type: "manual-label", + reason: "reviewer: disputed", + idempotencyKey: `manual-all-${counter++}`, + }, + { uri: current.uri, val: "security-yanked" }, + ); + + const findings = await analyzeHistory( + testEnv.DB, + assessmentFor({ uri: current.uri, cid: current.cid, artifactChecksum: checksum }), + { src: LABELER_DID }, + ); + + expect(findings.map((f) => f.category).toSorted()).toEqual([ + "active-manual-label", + "publisher-history", + "shared-artifact", + ]); + for (const finding of findings) + expect(HISTORY_FINDING_CATEGORIES.has(finding.category)).toBe(true); + + // The orchestrator validates every stage's output with the block∪warn + // allowed set; the amended validator must still admit these history findings. + const validated = validateFindings(findings, { + allowedCategories: allowedFindingCategories(MODERATION_POLICY), + resolvableEvidenceIds: new Set(), + }); + expect(validated).toHaveLength(3); + }); + + it("is best-effort: a D1 failure yields no findings rather than failing the run", async () => { + const brokenDb = { + prepare() { + throw new Error("D1 unavailable"); + }, + } as unknown as D1Database; + + await expect( + analyzeHistory(brokenDb, assessmentFor({ uri: "at://x/y/z", cid: "cid" }), { + src: LABELER_DID, + }), + ).resolves.toEqual([]); + }); +}); From 197bde568949ef94b0b2824da2bee37407dbb124 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 14 Jul 2026 08:59:14 +0100 Subject: [PATCH 2/3] test(labeler): make the history never-auto-label test self-contained The non-vacuousness assertion relied on prior releases seeded by earlier tests under the shared DID, so it was vacuous when run in isolation. Seed an explicit prior release inside the test. Claude-Session: https://claude.ai/code/session_014rikrGMh25h1rJczUQyTMM --- apps/labeler/test/assessment-orchestrator.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/labeler/test/assessment-orchestrator.test.ts b/apps/labeler/test/assessment-orchestrator.test.ts index bb93c5430..80f1eed2b 100644 --- a/apps/labeler/test/assessment-orchestrator.test.ts +++ b/apps/labeler/test/assessment-orchestrator.test.ts @@ -707,9 +707,16 @@ describe("AssessmentOrchestrator: history stage never auto-labels (W8.4)", () => cidValue: await cid("history-invariant"), }); - // Prior releases exist under PUBLISHER_DID (every run in this suite shares - // it), so the history stage genuinely produces a publisher-history finding - // — the invariant below is non-vacuous. + // Seed an explicit prior release under the same DID so the stage genuinely + // produces a publisher-history finding without relying on other tests' + // state — keeps the non-vacuousness assertion below self-contained. + await createSubject(testEnv.DB, { + uri: releaseUri("history-invariant-prior"), + cid: await cid("history-invariant-prior"), + did: PUBLISHER_DID, + collection: "com.emdashcms.experimental.package.release", + rkey: "history-invariant-prior:1.0.0", + }); const assessment = await getAssessment(testEnv.DB, run.id); const produced = await analyzeHistory(testEnv.DB, assessment!, { src: LABELER_DID }); expect(produced.some((f) => HISTORY_FINDING_CATEGORIES.has(f.category))).toBe(true); From 3f2fe11805e2c6f8ee161c6cdc0905ca8936d306 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 15 Jul 2026 07:43:45 +0100 Subject: [PATCH 3/3] test(labeler): assert the specific publisher-history finding in the invariant test Check for the `publisher-history` category the seeded prior release produces rather than any history category, so a regression in the prior-release path can't pass on an unrelated history finding. Claude-Session: https://claude.ai/code/session_014rikrGMh25h1rJczUQyTMM --- apps/labeler/test/assessment-orchestrator.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/labeler/test/assessment-orchestrator.test.ts b/apps/labeler/test/assessment-orchestrator.test.ts index 80f1eed2b..4c6955c2b 100644 --- a/apps/labeler/test/assessment-orchestrator.test.ts +++ b/apps/labeler/test/assessment-orchestrator.test.ts @@ -719,7 +719,10 @@ describe("AssessmentOrchestrator: history stage never auto-labels (W8.4)", () => }); const assessment = await getAssessment(testEnv.DB, run.id); const produced = await analyzeHistory(testEnv.DB, assessment!, { src: LABELER_DID }); - expect(produced.some((f) => HISTORY_FINDING_CATEGORIES.has(f.category))).toBe(true); + // Assert the specific finding the seeded prior release produces, not just + // "any history category" — so a regression in the prior-release path can't + // pass on the back of an unrelated history finding. + expect(produced.some((f) => f.category === "publisher-history")).toBe(true); const stages: OrchestratorStages = { ...stubStages,