diff --git a/apps/labeler/migrations/0010_assessment_error_escalations.sql b/apps/labeler/migrations/0010_assessment_error_escalations.sql new file mode 100644 index 000000000..bca6e2d81 --- /dev/null +++ b/apps/labeler/migrations/0010_assessment_error_escalations.sql @@ -0,0 +1,36 @@ +-- Prolonged-error escalation tracking (plan W10.5 follow-up, ratified +-- 2026-07-17). Drives the two-stage escalation ladder the reconciliation cron +-- applies to an assessment stuck in the terminal `error` state (the labeler +-- failed to assess, transient retries exhausted) that no newer run has +-- superseded: at 24h an operator alert (an `operational_events` row) so +-- operators can triage an infra-vs-publisher cause, then at 72h the publisher +-- notice if the error is still the live run. +-- +-- One row per escalated assessment makes each stage fire-once across the +-- 5-minute cron ticks: the cron upserts the row, raises the operator alert once +-- when `operator_alerted_at_epoch_ms` is null, and sends the publisher notice +-- once past 72h when `publisher_notified_at_epoch_ms` is null. MUTABLE — the two +-- mark columns flip from null to stamped — so no immutability trigger, mirroring +-- the `notifications`/`notification_outbox` mutable-outbox tables. +-- +-- Timestamp columns queries order/compare on carry an integer `*_epoch_ms` +-- sibling (RFC 3339 strings compare incorrectly across timezone offsets in SQL), +-- matching 0003-0008. + +CREATE TABLE assessment_error_escalations ( + assessment_id TEXT PRIMARY KEY REFERENCES assessments(id), + subject_uri TEXT NOT NULL, + subject_cid TEXT NOT NULL, + operator_alerted_at_epoch_ms INTEGER, + publisher_notified_at_epoch_ms INTEGER, + created_at TEXT NOT NULL, + created_at_epoch_ms INTEGER NOT NULL +); + +-- The escalation scan filters `state = 'error' AND completed_at_epoch_ms <= ?` +-- and orders by `completed_at_epoch_ms`; the existing `idx_assessments_state_created` +-- covers `(state, created_at_epoch_ms)`, not `completed_at`. Partial on the error +-- state keeps this compact (errors are a small minority) and matches the cron's +-- filter and order exactly, so a 5-minute tick seeks instead of full-scanning. +CREATE INDEX idx_assessments_error_completed + ON assessments(completed_at_epoch_ms) WHERE state = 'error'; diff --git a/apps/labeler/src/assessment-error-escalations.ts b/apps/labeler/src/assessment-error-escalations.ts new file mode 100644 index 000000000..f29b7c166 --- /dev/null +++ b/apps/labeler/src/assessment-error-escalations.ts @@ -0,0 +1,183 @@ +/** + * Persistence for the prolonged-error escalation ladder (plan W10.5 follow-up). + * `assessment_error_escalations` (migration 0010) tracks, per errored + * assessment, whether the 24h operator alert and the 72h publisher notice have + * already fired — the tracking table that makes each stage idempotent across the + * 5-minute reconciliation cron ticks. + * + * The escalation query deliberately does NOT reuse the pointer-based + * `isSuperseded` (an `error` run never moves the `current_assessments` pointer, + * spec §10): supersession here means simply "a newer run exists for the same + * `(uri, cid)`", detected by `created_at_epoch_ms`. + */ + +import { PROLONGED_ERROR_OPERATOR_THRESHOLD_MS, PROLONGED_ERROR_SCAN_BATCH } from "./constants.js"; + +export interface EscalatableError { + id: string; + uri: string; + cid: string; + completedAtEpochMs: number; +} + +export interface AssessmentErrorEscalation { + assessmentId: string; + subjectUri: string; + subjectCid: string; + operatorAlertedAtEpochMs: number | null; + publisherNotifiedAtEpochMs: number | null; + createdAtEpochMs: number; +} + +interface EscalatableErrorRow { + id: string; + uri: string; + cid: string; + completed_at_epoch_ms: number; +} + +interface EscalationRow { + assessment_id: string; + subject_uri: string; + subject_cid: string; + operator_alerted_at_epoch_ms: number | null; + publisher_notified_at_epoch_ms: number | null; + created_at_epoch_ms: number; +} + +/** + * Terminal `error` assessments whose `completed_at` is at least the operator + * threshold (24h) old and that no newer run has superseded — the candidate set + * for the escalation ladder. Skips subjects tombstoned at the source (a deleted + * release warrants no publisher chase) and rows whose escalation is already + * complete (both marks stamped), so a fully-escalated row leaves the + * `completed_at`-ordered window and newer errors behind it are reached during a + * backlog. Capped at {@link PROLONGED_ERROR_SCAN_BATCH}; a full page is logged so + * a persistent backlog is visible. + */ +export async function findEscalatableErrors( + db: D1Database, + now: Date, +): Promise { + const operatorBefore = now.getTime() - PROLONGED_ERROR_OPERATOR_THRESHOLD_MS; + const rows = await db + .prepare( + `SELECT a.id, a.uri, a.cid, a.completed_at_epoch_ms + FROM assessments a + JOIN subjects s ON s.uri = a.uri AND s.cid = a.cid + LEFT JOIN assessment_error_escalations e ON e.assessment_id = a.id + WHERE a.state = 'error' + AND a.completed_at_epoch_ms IS NOT NULL + AND a.completed_at_epoch_ms <= ? + AND s.deleted_at IS NULL + AND ( + e.assessment_id IS NULL + OR e.operator_alerted_at_epoch_ms IS NULL + OR e.publisher_notified_at_epoch_ms IS NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM assessments b + WHERE b.uri = a.uri AND b.cid = a.cid AND b.created_at_epoch_ms > a.created_at_epoch_ms + ) + ORDER BY a.completed_at_epoch_ms ASC + LIMIT ?`, + ) + .bind(operatorBefore, PROLONGED_ERROR_SCAN_BATCH) + .all(); + const results = rows.results ?? []; + if (results.length >= PROLONGED_ERROR_SCAN_BATCH) + console.error("[labeler] prolonged-error escalation scan hit the batch cap", { + cap: PROLONGED_ERROR_SCAN_BATCH, + }); + return results.map((row) => ({ + id: row.id, + uri: row.uri, + cid: row.cid, + completedAtEpochMs: row.completed_at_epoch_ms, + })); +} + +/** Guarantee a tracking row exists for an errored assessment, leaving both mark + * columns null. Idempotent: a repeated cron tick keeps the first row's marks. */ +export async function ensureEscalationRow( + db: D1Database, + input: { assessmentId: string; subjectUri: string; subjectCid: string; now: Date }, +): Promise { + await db + .prepare( + `INSERT INTO assessment_error_escalations + (assessment_id, subject_uri, subject_cid, created_at, created_at_epoch_ms) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(assessment_id) DO NOTHING`, + ) + .bind( + input.assessmentId, + input.subjectUri, + input.subjectCid, + input.now.toISOString(), + input.now.getTime(), + ) + .run(); +} + +export async function getEscalation( + db: D1Database, + assessmentId: string, +): Promise { + const row = await db + .prepare( + `SELECT assessment_id, subject_uri, subject_cid, + operator_alerted_at_epoch_ms, publisher_notified_at_epoch_ms, created_at_epoch_ms + FROM assessment_error_escalations WHERE assessment_id = ?`, + ) + .bind(assessmentId) + .first(); + return row + ? { + assessmentId: row.assessment_id, + subjectUri: row.subject_uri, + subjectCid: row.subject_cid, + operatorAlertedAtEpochMs: row.operator_alerted_at_epoch_ms, + publisherNotifiedAtEpochMs: row.publisher_notified_at_epoch_ms, + createdAtEpochMs: row.created_at_epoch_ms, + } + : null; +} + +/** + * The statement that stamps `operator_alerted_at`, guarded on it still being + * null. Returned rather than executed so the caller can batch it atomically with + * the operational-event insert — a crash can then never leave the alert raised + * without the mark (which would re-alert on the next tick). + */ +export function buildMarkOperatorAlerted( + db: D1Database, + assessmentId: string, + now: Date, +): D1PreparedStatement { + return db + .prepare( + `UPDATE assessment_error_escalations + SET operator_alerted_at_epoch_ms = ? + WHERE assessment_id = ? AND operator_alerted_at_epoch_ms IS NULL`, + ) + .bind(now.getTime(), assessmentId); +} + +/** Stamp `publisher_notified_at`, guarded on it still being null. Runs after the + * notice trigger returns; the notice's own `(issuance, id)` dedup makes a crash + * between send and mark self-heal (the next tick re-sends nothing, then marks). */ +export async function markPublisherNotified( + db: D1Database, + assessmentId: string, + now: Date, +): Promise { + await db + .prepare( + `UPDATE assessment_error_escalations + SET publisher_notified_at_epoch_ms = ? + WHERE assessment_id = ? AND publisher_notified_at_epoch_ms IS NULL`, + ) + .bind(now.getTime(), assessmentId) + .run(); +} diff --git a/apps/labeler/src/constants.ts b/apps/labeler/src/constants.ts index 2dce10d4d..66721e739 100644 --- a/apps/labeler/src/constants.ts +++ b/apps/labeler/src/constants.ts @@ -40,3 +40,18 @@ export const NOTIFICATION_MAX_SEND_ATTEMPTS = 5; export const NOTIFICATION_STUCK_PENDING_MS = 15 * 60 * 1000; export const NOTIFICATION_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; export const NOTIFICATION_SWEEP_BATCH = 200; + +/** + * Prolonged-error escalation thresholds (plan W10.5 follow-up). Versioned as a + * set: bump together if the ladder is retuned. A terminal `error` assessment + * (retries exhausted) that no newer run has superseded escalates in two stages + * measured from `completed_at_epoch_ms`: at {@link PROLONGED_ERROR_OPERATOR_THRESHOLD_MS} + * (24h) an operator alert so operators can triage an infra-vs-publisher cause; + * at {@link PROLONGED_ERROR_PUBLISHER_THRESHOLD_MS} (72h) the publisher notice, + * only if the error is still live. {@link PROLONGED_ERROR_SCAN_BATCH} caps one + * cron pass's scan of escalatable errors. + */ +export const PROLONGED_ERROR_ESCALATION_VERSION = 1; +export const PROLONGED_ERROR_OPERATOR_THRESHOLD_MS = 24 * 60 * 60 * 1000; +export const PROLONGED_ERROR_PUBLISHER_THRESHOLD_MS = 72 * 60 * 60 * 1000; +export const PROLONGED_ERROR_SCAN_BATCH = 200; diff --git a/apps/labeler/src/index.ts b/apps/labeler/src/index.ts index d9c7bd096..13b31063d 100644 --- a/apps/labeler/src/index.ts +++ b/apps/labeler/src/index.ts @@ -13,6 +13,7 @@ import { didDocumentResponse, policyDocumentResponse } from "./identity.js"; import { handleNotificationRequest, isNotificationPath } from "./notification-endpoints.js"; import { runNotificationSweep } from "./notification-sweep.js"; import { createNotifyDeps, type NotifyDeps } from "./notification-triggers.js"; +import { runProlongedErrorEscalation } from "./prolonged-error.js"; import { queryLabels } from "./query-labels.js"; import { reconcileAssessments } from "./reconciliation.js"; import { createRuntimeSigner, getRuntimeSigningSecret } from "./signing-runtime.js"; @@ -124,6 +125,24 @@ export default { } })(), ); + + // Prolonged-error escalation (plan W10.5 follow-up): 24h operator alert then + // 72h publisher notice for an errored, unsuperseded run. Its own branch with + // the non-throwing deps builder, so a notify-deps failure skips the pass + // rather than disturbing the sweep or reconciliation passes above. + ctx.waitUntil( + (async () => { + try { + const deps = await safeCreateNotifyDeps(env); + if (!deps) return; + await runProlongedErrorEscalation(deps, new Date()); + } catch (err) { + console.error("[labeler] prolonged-error escalation failed", { + error: err instanceof Error ? err.message : String(err), + }); + } + })(), + ); }, }; diff --git a/apps/labeler/src/notification-triggers.ts b/apps/labeler/src/notification-triggers.ts index 7fc48107a..6726f8978 100644 --- a/apps/labeler/src/notification-triggers.ts +++ b/apps/labeler/src/notification-triggers.ts @@ -231,6 +231,26 @@ function reconsiderationOutcomeNoticeContent( }; } +/** Prolonged-error notice for an assessment stuck in `error` past the 72h + * threshold (plan W10.5 follow-up). Neutral and actionable: it states the + * labeler could not complete its own assessment, that no label change is + * implied, and gives the publisher the one thing they can act on — checking the + * release's artifact URL is reachable. Carries NO findings or private detail. */ +function prolongedErrorNoticeContent( + urls: NoticeUrls, + input: { uri: string; cid: string }, +): NoticeContent { + return { + subject: "We couldn't complete the security assessment of your plugin release", + publicSummary: + "The automated security assessment of this release repeatedly failed to complete.", + effect: + "No label change is implied by this failure. If the release's artifact URL is unavailable or has changed, please verify it is reachable so the assessment can complete.", + assessmentUrl: assessmentUrl(urls.serviceUrl, input.uri, input.cid), + reconsiderationUrl: urls.reconsiderationUrl, + }; +} + // ── Live trigger entry points ─────────────────────────────────────────────── /** Automated block/warning notice from a finalized assessment run. Source is the @@ -326,6 +346,30 @@ export async function notifyReconsiderationOutcome( ); } +/** + * Prolonged-error publisher notice, fired by the reconciliation cron's 72h stage + * (plan W10.5 follow-up) — never at finalization (`assessmentNoticeContent` + * stays null for `error`, so `notifyAssessmentOutcome` no-ops on it). Source is + * the errored assessment id: an errored run never produces a block/warn notice, + * so the `(issuance, id)` key never collides with a finalization notice, and the + * claim dedups a crash-retry between the send and the escalation-row mark. + */ +export async function notifyProlongedError( + deps: NotifyDeps, + assessment: Assessment, +): Promise { + const target = contactTargetFromUri(assessment.uri); + // An unparseable URI is terminal, not transient (the URI never changes), so it + // counts as processed — the cron marks it rather than re-attempting forever. + if (!target) return true; + return runTrigger( + deps, + { type: "issuance", id: assessment.id }, + target, + prolongedErrorNoticeContent(deps, { uri: assessment.uri, cid: assessment.cid }), + ); +} + /** * Rebuild a NOTICE's public content from its source row, for the retry sweep. * Nothing about the notice is persisted (plaintext minimization), so a retry @@ -341,7 +385,10 @@ export async function resolveNoticeForSource( ): Promise { if (sourceType === "issuance") { const assessment = await loadAssessmentSafe(deps.db, sourceId); - return assessment ? assessmentNoticeContent(deps, assessment) : null; + if (!assessment) return null; + if (assessment.state === "error") + return prolongedErrorNoticeContent(deps, { uri: assessment.uri, cid: assessment.cid }); + return assessmentNoticeContent(deps, assessment); } const action = await getOperatorActionById(deps.db, sourceId); if (!action || action.subjectUri === null) return null; @@ -423,18 +470,26 @@ function operatorActionNeg(metadataJson: string): boolean { * Dedup, resolve verification (fail-closed), send, swallow+log. Shared by every * trigger so the dedup and verified-skip policy is applied uniformly and a * notification failure can never escape into the label path. + * + * Returns whether the trigger reached a TERMINAL outcome — a dedup hit or a + * normal `sendNotification` return (sent, confirmation-sent, undeliverable, or a + * claimed-then-failed row the sweep now owns) — versus a thrown TRANSIENT error + * (an aggregator read or a pre-claim D1 write that failed before any row was + * claimed). The prolonged-error cron uses this to decide whether to stamp its + * fire-once mark: a transient failure returns `false` so the next tick retries + * instead of being silently swallowed. Fire-and-forget callers ignore it. */ async function runTrigger( deps: NotifyDeps, source: NotificationSource, target: ContactTarget, notice: NoticeContent, -): Promise { +): Promise { const now = deps.now ?? (() => new Date()); try { if (await sourceAlreadyProcessed(deps.db, source)) { logTrigger(source, target.did, "deduped"); - return; + return true; } const verifiedPublisher = await isVerifiedPublisher(deps.aggregator, target.did, now()); const ctx: SendContext = { @@ -449,6 +504,7 @@ async function runTrigger( const request: NotificationRequest = { source, target, notice }; const outcome = await sendNotification(ctx, request); logTrigger(source, target.did, outcome.status); + return true; } catch (error) { console.error("[notifications] trigger failed", { sourceType: source.type, @@ -456,6 +512,7 @@ async function runTrigger( did: target.did, error: error instanceof Error ? error.message : String(error), }); + return false; } } diff --git a/apps/labeler/src/operational-events.ts b/apps/labeler/src/operational-events.ts index 5f40783ab..557170cb7 100644 --- a/apps/labeler/src/operational-events.ts +++ b/apps/labeler/src/operational-events.ts @@ -13,7 +13,8 @@ export type OperationalEventType = | "dead-letter-retried" | "dead-letter-quarantined" | "reconsideration-opened" - | "reconsideration-resolved"; + | "reconsideration-resolved" + | "assessment-prolonged-error"; export type OperationalEventSeverity = "critical" | "high" | "info"; @@ -26,6 +27,11 @@ export type OperationalEventSeverity = "critical" | "high" | "info"; */ export interface OperationalEventPayload { reason?: string; + /** The escalated run and its release CID for `assessment-prolonged-error`. + * Both are public identifiers (the assessment id the public API returns, the + * record's content hash) — no findings or private detail. */ + assessmentId?: string; + cid?: string; } export interface OperationalEventInsert { @@ -63,6 +69,16 @@ export interface OperationalEventInsert { * phantom resolved-event while its audit row still commits for replay. */ gateOnResolvedReconsideration?: { reconsiderationId: string; actionId: string }; + /** + * When set, the INSERT is gated on the `assessment_error_escalations` row for + * this assessment still being un-alerted. Batched atomically with the + * `operator_alerted_at` mark, it makes the prolonged-error operator alert + * at-most-once even under overlapping cron ticks: a second pass that read the + * row unalerted before the first committed still inserts no event, because its + * EXISTS sees the committed mark (the app-level null read alone cannot — cron + * ticks are not serialized against each other). + */ + gateOnUnalertedEscalation?: { assessmentId: string }; } export interface OutboxInsert { @@ -196,6 +212,19 @@ export function buildOperationalEventInsert( .bind(...values, input.gateOnIssuedLabelActionId); } + if (input.gateOnUnalertedEscalation !== undefined) { + return db + .prepare( + `INSERT INTO operational_events (${columns}) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ? + WHERE EXISTS ( + SELECT 1 FROM assessment_error_escalations + WHERE assessment_id = ? AND operator_alerted_at_epoch_ms IS NULL + )`, + ) + .bind(...values, input.gateOnUnalertedEscalation.assessmentId); + } + return db .prepare( `INSERT INTO operational_events (${columns}) diff --git a/apps/labeler/src/prolonged-error.ts b/apps/labeler/src/prolonged-error.ts new file mode 100644 index 000000000..982468e26 --- /dev/null +++ b/apps/labeler/src/prolonged-error.ts @@ -0,0 +1,74 @@ +/** + * The reconciliation cron's prolonged-error escalation pass (plan W10.5 + * follow-up). For each terminal `error` assessment that has stayed the live, + * unsuperseded run past the thresholds, it walks the two-stage ladder: + * + * - 24h: raise a `assessment-prolonged-error` operational_event (severity + * high, no operator action) so operators can triage an infra-vs-publisher + * cause. The event insert and the `operator_alerted_at` mark commit in one + * `db.batch`, so a crash can never raise the alert without recording it and + * re-alert on the next tick. + * - 72h: send the publisher notice, then stamp `publisher_notified_at`. The + * notice's own `(issuance, id)` dedup makes a crash between send and mark + * self-heal — the next tick re-sends nothing and stamps the mark. + * + * Each stage is fire-once across the 5-minute ticks via + * `assessment_error_escalations` (migration 0010). The pass takes {@link NotifyDeps} + * because the 72h stage needs the sender; the cron builds it with + * `safeCreateNotifyDeps` and skips the pass when it is unavailable. + */ + +import { + buildMarkOperatorAlerted, + ensureEscalationRow, + findEscalatableErrors, + getEscalation, + markPublisherNotified, +} from "./assessment-error-escalations.js"; +import { getAssessment } from "./assessment-store.js"; +import { PROLONGED_ERROR_PUBLISHER_THRESHOLD_MS } from "./constants.js"; +import type { NotifyDeps } from "./notification-triggers.js"; +import { notifyProlongedError } from "./notification-triggers.js"; +import { buildOperationalEventInsert, newOperationalEventId } from "./operational-events.js"; + +export async function runProlongedErrorEscalation(deps: NotifyDeps, now: Date): Promise { + const escalatable = await findEscalatableErrors(deps.db, now); + for (const error of escalatable) { + await ensureEscalationRow(deps.db, { + assessmentId: error.id, + subjectUri: error.uri, + subjectCid: error.cid, + now, + }); + const escalation = await getEscalation(deps.db, error.id); + if (!escalation) continue; + + if (escalation.operatorAlertedAtEpochMs === null) { + const eventInsert = buildOperationalEventInsert(deps.db, { + id: newOperationalEventId(), + eventType: "assessment-prolonged-error", + severity: "high", + actionId: null, + subjectUri: error.uri, + payload: { assessmentId: error.id, cid: error.cid }, + now, + gateOnUnalertedEscalation: { assessmentId: error.id }, + }); + await deps.db.batch([eventInsert, buildMarkOperatorAlerted(deps.db, error.id, now)]); + } + + if ( + escalation.publisherNotifiedAtEpochMs === null && + now.getTime() - error.completedAtEpochMs >= PROLONGED_ERROR_PUBLISHER_THRESHOLD_MS + ) { + const assessment = await getAssessment(deps.db, error.id); + if (!assessment) continue; + // Only stamp the fire-once mark when the trigger reached a terminal + // outcome. A transient failure (aggregator read / pre-claim D1 write threw + // before any notifications row was claimed) returns false — leave the mark + // null so the next tick retries rather than silently dropping the notice. + const processed = await notifyProlongedError(deps, assessment); + if (processed) await markPublisherNotified(deps.db, error.id, now); + } + } +} diff --git a/apps/labeler/test/prolonged-error.test.ts b/apps/labeler/test/prolonged-error.test.ts new file mode 100644 index 000000000..e1db20795 --- /dev/null +++ b/apps/labeler/test/prolonged-error.test.ts @@ -0,0 +1,509 @@ +import { applyD1Migrations, env } from "cloudflare:test"; +import { beforeAll, describe, expect, it } from "vitest"; + +import { AggregatorClient } from "../src/aggregator-client.js"; +import { + buildMarkOperatorAlerted, + ensureEscalationRow, + findEscalatableErrors, + getEscalation, +} from "../src/assessment-error-escalations.js"; +import { computeRunKey, initialTriggerId, intelTriggerId } from "../src/assessment-lifecycle.js"; +import type { Assessment } from "../src/assessment-store.js"; +import { + buildFinalizationStatements, + createAssessmentRun, + createSubject, + deleteSubject, + transitionAssessmentState, +} from "../src/assessment-store.js"; +import { + confirmContact, + ensureContact, + hashConfirmToken, + recipientHash, + recordConfirmSent, +} from "../src/notification-contacts.js"; +import type { ConfirmationPayload, NoticePayload, SendResult } from "../src/notification-send.js"; +import { + notifyAssessmentOutcome, + resolveNoticeForSource, + type NotifyDeps, +} from "../src/notification-triggers.js"; +import { buildOperationalEventInsert, newOperationalEventId } from "../src/operational-events.js"; +import { runProlongedErrorEscalation } from "../src/prolonged-error.js"; + +interface TestEnv { + DB: D1Database; + TEST_MIGRATIONS: Parameters[1]; +} +const testEnv = env as unknown as TestEnv; +const db = () => testEnv.DB; + +const PEPPER = "prolonged-pepper"; +const SERVICE = "https://labels.example"; +const RECON = "https://recon.example/reconsider"; +const SRC = "did:plc:labeler000000000000000000"; +const PUBLISHER_DID = "did:plc:publisher000000000000000000"; +const NOW = new Date("2026-07-17T00:00:00.000Z"); +const H = 60 * 60 * 1000; + +let counter = 0; +const uniq = (p: string) => `${p}-${++counter}`; + +beforeAll(async () => { + await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS); +}); + +function release(): { uri: string; cid: string } { + counter++; + return { + uri: `at://${PUBLISHER_DID}/com.emdashcms.experimental.package.release/pe-${counter}:1.0.0`, + cid: `bafkreipe${counter}0000000000000000000000000000000000000000000`, + }; +} + +async function subject(target: { uri: string; cid: string }): Promise { + await createSubject(db(), { + uri: target.uri, + cid: target.cid, + did: PUBLISHER_DID, + collection: "com.emdashcms.experimental.package.release", + rkey: target.uri.split("/").at(-1)!, + }); +} + +/** Drives a run all the way to the terminal `error` state, controlling both the + * `created_at` (supersession ordering) and `completed_at` (escalation timing). */ +async function errorAssessment( + target: { uri: string; cid: string }, + opts: { createdAt: Date; completedAt: Date; triggerId?: string }, +): Promise { + const triggerId = opts.triggerId ?? initialTriggerId(target.cid); + const runKey = await computeRunKey({ + uri: target.uri, + cid: target.cid, + policyVersion: "v1", + modelId: "m", + promptHash: "p", + scannerSetVersion: "v1", + triggerId, + }); + const { assessment } = await createAssessmentRun(db(), { + runKey, + uri: target.uri, + cid: target.cid, + trigger: "initial", + triggerId, + policyVersion: "v1", + coverageJson: "{}", + now: opts.createdAt, + }); + const id = assessment.id; + await transitionAssessmentState(db(), { + id, + from: "observed", + to: "verifying", + now: opts.createdAt, + }); + await transitionAssessmentState(db(), { + id, + from: "verifying", + to: "pending", + now: opts.createdAt, + }); + await transitionAssessmentState(db(), { + id, + from: "pending", + to: "running", + now: opts.createdAt, + }); + const fin = buildFinalizationStatements(db(), { + assessmentId: id, + fromState: "running", + toState: "error", + src: SRC, + uri: target.uri, + cid: target.cid, + now: opts.completedAt, + }); + await db().batch(fin.statements); + return id; +} + +interface RecordingSender { + confirmations: ConfirmationPayload[]; + notices: NoticePayload[]; + sendConfirmation(p: ConfirmationPayload): Promise; + sendNotice(p: NoticePayload): Promise; +} + +function recordingSender(result: SendResult = { ok: true, providerId: "p" }): RecordingSender { + const confirmations: ConfirmationPayload[] = []; + const notices: NoticePayload[] = []; + return { + confirmations, + notices, + sendConfirmation: async (p) => (confirmations.push(p), result), + sendNotice: async (p) => (notices.push(p), result), + }; +} + +function aggregatorFor(email?: string): AggregatorClient { + const fetcher = { + fetch: async (url: string) => { + if (url.includes("getPublisherVerification")) + return Response.json({ did: PUBLISHER_DID, verifications: [], labels: [] }); + if (url.includes("getPublisher") && email !== undefined) + return Response.json({ + did: PUBLISHER_DID, + profile: { contact: [{ kind: "security", email }] }, + }); + return new Response(JSON.stringify({ error: "NotFound" }), { + status: 404, + headers: { "content-type": "application/json" }, + }); + }, + } as unknown as Fetcher; + return new AggregatorClient(fetcher); +} + +/** An aggregator whose reads THROW — drives a transient pre-claim failure in + * `resolvePublisherContact` (which propagates rather than returning `none`). */ +function throwingAggregator(): AggregatorClient { + return new AggregatorClient({ + fetch: async () => { + throw new Error("aggregator down"); + }, + } as unknown as Fetcher); +} + +function deps(sender: RecordingSender, aggregator: AggregatorClient): NotifyDeps { + return { + db: db(), + aggregator, + sender, + pepper: PEPPER, + serviceUrl: SERVICE, + reconsiderationUrl: RECON, + now: () => NOW, + }; +} + +async function seedConfirmed(email: string): Promise { + const hash = await recipientHash(PEPPER, email); + await ensureContact(db(), hash, "2026-07-16T00:00:00.000Z"); + const th = await hashConfirmToken("seed"); + await recordConfirmSent(db(), hash, th, 1_000); + await confirmContact(db(), hash, th, "2026-07-16T00:00:01.000Z"); +} + +async function operatorAlertCount(uri: string): Promise { + const row = await db() + .prepare( + `SELECT COUNT(*) AS n FROM operational_events + WHERE event_type = 'assessment-prolonged-error' AND subject_uri = ?`, + ) + .bind(uri) + .first<{ n: number }>(); + return row?.n ?? 0; +} + +async function notificationRows(sourceId: string): Promise<{ kind: string; state: string }[]> { + const r = await db() + .prepare(`SELECT kind, state FROM notifications WHERE source_id = ?`) + .bind(sourceId) + .all<{ kind: string; state: string }>(); + return r.results ?? []; +} + +describe("operator alert stage (24h)", () => { + it("fires an operator alert once the error is 24h old, not before", async () => { + const fresh = release(); + await subject(fresh); + await errorAssessment(fresh, { + createdAt: new Date(NOW.getTime() - 30 * H), + completedAt: new Date(NOW.getTime() - 23 * H), + }); + + await runProlongedErrorEscalation(deps(recordingSender(), aggregatorFor()), NOW); + expect(await operatorAlertCount(fresh.uri)).toBe(0); + + const stale = release(); + await subject(stale); + await errorAssessment(stale, { + createdAt: new Date(NOW.getTime() - 30 * H), + completedAt: new Date(NOW.getTime() - 25 * H), + }); + + await runProlongedErrorEscalation(deps(recordingSender(), aggregatorFor()), NOW); + expect(await operatorAlertCount(stale.uri)).toBe(1); + }); + + it("records the alert as severity high with a public-safe payload", async () => { + const target = release(); + await subject(target); + const id = await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 30 * H), + completedAt: new Date(NOW.getTime() - 25 * H), + }); + + await runProlongedErrorEscalation(deps(recordingSender(), aggregatorFor()), NOW); + + const row = await db() + .prepare( + `SELECT severity, action_id, payload_json FROM operational_events + WHERE event_type = 'assessment-prolonged-error' AND subject_uri = ?`, + ) + .bind(target.uri) + .first<{ severity: string; action_id: string | null; payload_json: string }>(); + expect(row?.severity).toBe("high"); + expect(row?.action_id).toBeNull(); + const payload = JSON.parse(row?.payload_json ?? "{}") as Record; + expect(payload).toEqual({ assessmentId: id, cid: target.cid }); + }); + + it("is idempotent across repeated cron ticks (exactly one event)", async () => { + const target = release(); + await subject(target); + await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 30 * H), + completedAt: new Date(NOW.getTime() - 25 * H), + }); + + await runProlongedErrorEscalation(deps(recordingSender(), aggregatorFor()), NOW); + await runProlongedErrorEscalation(deps(recordingSender(), aggregatorFor()), NOW); + await runProlongedErrorEscalation(deps(recordingSender(), aggregatorFor()), NOW); + + expect(await operatorAlertCount(target.uri)).toBe(1); + }); + + it("inserts exactly one operator event when two overlapping passes both read it unalerted", async () => { + const target = release(); + await subject(target); + const id = await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 30 * H), + completedAt: new Date(NOW.getTime() - 25 * H), + }); + await ensureEscalationRow(db(), { + assessmentId: id, + subjectUri: target.uri, + subjectCid: target.cid, + now: NOW, + }); + // Both passes observe operator_alerted null (cron ticks are not serialized), + // then each commits its gated event+mark batch. The in-batch EXISTS gate — not + // the stale app-level read — must admit only the first. + expect((await getEscalation(db(), id))?.operatorAlertedAtEpochMs).toBeNull(); + expect((await getEscalation(db(), id))?.operatorAlertedAtEpochMs).toBeNull(); + const commitAlertBatch = () => + db().batch([ + buildOperationalEventInsert(db(), { + id: newOperationalEventId(), + eventType: "assessment-prolonged-error", + severity: "high", + actionId: null, + subjectUri: target.uri, + payload: { assessmentId: id, cid: target.cid }, + now: NOW, + gateOnUnalertedEscalation: { assessmentId: id }, + }), + buildMarkOperatorAlerted(db(), id, NOW), + ]); + + await commitAlertBatch(); + await commitAlertBatch(); + + expect(await operatorAlertCount(target.uri)).toBe(1); + }); + + it("drops a fully-escalated row from the scan window so newer errors are reached", async () => { + const done = release(); + await subject(done); + const doneId = await errorAssessment(done, { + createdAt: new Date(NOW.getTime() - 100 * H), + completedAt: new Date(NOW.getTime() - 73 * H), + }); + const email = uniq("win") + "@x.test"; + await seedConfirmed(email); + await runProlongedErrorEscalation(deps(recordingSender(), aggregatorFor(email)), NOW); + + // Both marks are now set, so the oldest (by completed_at) row leaves the window. + const escalation = await getEscalation(db(), doneId); + expect(escalation?.operatorAlertedAtEpochMs).not.toBeNull(); + expect(escalation?.publisherNotifiedAtEpochMs).not.toBeNull(); + expect((await findEscalatableErrors(db(), NOW)).some((e) => e.id === doneId)).toBe(false); + + // A newer error behind it in completed_at order is still reached. + const fresh = release(); + await subject(fresh); + const freshId = await errorAssessment(fresh, { + createdAt: new Date(NOW.getTime() - 30 * H), + completedAt: new Date(NOW.getTime() - 25 * H), + }); + const found = await findEscalatableErrors(db(), NOW); + expect(found.some((e) => e.id === freshId)).toBe(true); + expect(found.some((e) => e.id === doneId)).toBe(false); + }); + + it("does not escalate a superseded error at all", async () => { + const target = release(); + await subject(target); + await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 100 * H), + completedAt: new Date(NOW.getTime() - 80 * H), + }); + // A newer run for the SAME (uri, cid) supersedes the error. + await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 10 * H), + completedAt: new Date(NOW.getTime() - 5 * H), + triggerId: intelTriggerId("corpus-v2"), + }); + + const sender = recordingSender(); + await runProlongedErrorEscalation(deps(sender, aggregatorFor("s@x.test")), NOW); + + expect(await operatorAlertCount(target.uri)).toBe(0); + expect(sender.notices).toHaveLength(0); + }); + + it("does not escalate an error whose subject was deleted at the source", async () => { + const target = release(); + await subject(target); + await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 100 * H), + completedAt: new Date(NOW.getTime() - 80 * H), + }); + await deleteSubject(db(), { uri: target.uri, cid: target.cid }); + + const sender = recordingSender(); + await runProlongedErrorEscalation(deps(sender, aggregatorFor("s@x.test")), NOW); + + expect(await operatorAlertCount(target.uri)).toBe(0); + expect(sender.notices).toHaveLength(0); + }); +}); + +describe("publisher notice stage (72h)", () => { + it("does not notify the publisher before 72h", async () => { + const target = release(); + await subject(target); + const id = await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 80 * H), + completedAt: new Date(NOW.getTime() - 71 * H), + }); + const email = uniq("early") + "@x.test"; + await seedConfirmed(email); + const sender = recordingSender(); + + await runProlongedErrorEscalation(deps(sender, aggregatorFor(email)), NOW); + + expect(await operatorAlertCount(target.uri)).toBe(1); + expect(sender.notices).toHaveLength(0); + expect(await notificationRows(id)).toHaveLength(0); + }); + + it("notifies a confirmed publisher once past 72h, exactly once across ticks", async () => { + const target = release(); + await subject(target); + const id = await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 100 * H), + completedAt: new Date(NOW.getTime() - 73 * H), + }); + const email = uniq("late") + "@x.test"; + await seedConfirmed(email); + const sender = recordingSender(); + const d = deps(sender, aggregatorFor(email)); + + await runProlongedErrorEscalation(d, NOW); + await runProlongedErrorEscalation(d, NOW); + + expect(sender.notices).toHaveLength(1); + expect(sender.notices[0]).toMatchObject({ + to: email, + subject: expect.stringContaining("couldn't complete"), + }); + expect(await notificationRows(id)).toEqual([{ kind: "notice", state: "sent" }]); + }); + + it("gates an unconfirmed publisher through double opt-in (confirmation only)", async () => { + const target = release(); + await subject(target); + const id = await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 100 * H), + completedAt: new Date(NOW.getTime() - 73 * H), + }); + const email = uniq("unconf") + "@x.test"; + const sender = recordingSender(); + + await runProlongedErrorEscalation(deps(sender, aggregatorFor(email)), NOW); + + expect(sender.notices).toHaveLength(0); + expect(sender.confirmations).toHaveLength(1); + expect(await notificationRows(id)).toEqual([{ kind: "confirmation", state: "sent" }]); + }); + + it("does not mark on a transient pre-claim failure, and retries on the next tick", async () => { + const target = release(); + await subject(target); + const id = await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 100 * H), + completedAt: new Date(NOW.getTime() - 73 * H), + }); + const email = uniq("retry") + "@x.test"; + await seedConfirmed(email); + + // Tick 1: the aggregator read throws inside resolvePublisherContact, before any + // notifications row is claimed. The trigger swallows it and returns false, so + // nothing is claimed and publisher_notified_at stays null. + await runProlongedErrorEscalation(deps(recordingSender(), throwingAggregator()), NOW); + expect((await getEscalation(db(), id))?.publisherNotifiedAtEpochMs).toBeNull(); + expect(await notificationRows(id)).toHaveLength(0); + + // Tick 2: the aggregator is healthy → the notice is claimed and sent, and the + // mark is stamped. A transient failure retried rather than being swallowed. + const sender = recordingSender(); + await runProlongedErrorEscalation(deps(sender, aggregatorFor(email)), NOW); + expect(sender.notices).toHaveLength(1); + expect(await notificationRows(id)).toEqual([{ kind: "notice", state: "sent" }]); + expect((await getEscalation(db(), id))?.publisherNotifiedAtEpochMs).not.toBeNull(); + }); +}); + +describe("sweep parity and finalization invariant", () => { + it("resolveNoticeForSource re-renders the prolonged-error content for an errored run", async () => { + const target = release(); + await subject(target); + const id = await errorAssessment(target, { + createdAt: new Date(NOW.getTime() - 100 * H), + completedAt: new Date(NOW.getTime() - 73 * H), + }); + + const notice = await resolveNoticeForSource( + deps(recordingSender(), aggregatorFor()), + "issuance", + id, + ); + + expect(notice).not.toBeNull(); + expect(notice?.subject).toContain("couldn't complete"); + expect(notice?.assessmentUrl.startsWith(SERVICE)).toBe(true); + }); + + it("notifyAssessmentOutcome never notifies on an error assessment (finalization stays silent)", async () => { + const sender = recordingSender(); + const assessment = { + id: uniq("asmt"), + uri: `at://${PUBLISHER_DID}/com.emdashcms.experimental.package.release/fin1`, + cid: "bafyerr", + state: "error", + publicSummary: "irrelevant", + } as unknown as Assessment; + + await notifyAssessmentOutcome(deps(sender, aggregatorFor("s@x.test")), assessment); + + expect(sender.notices).toHaveLength(0); + expect(sender.confirmations).toHaveLength(0); + expect(await notificationRows(assessment.id)).toHaveLength(0); + }); +});