From fbee53aae392fa14e5edf87bdbd5fa729df4b7c4 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:15:52 -0300 Subject: [PATCH 1/3] fix: recover reversible inputs under replacement sources Separate confirmed publication progress from ABI coverage and bind new action history plans to retained source context. Preserve legacy identities, full provenance fingerprints and original source/end validators. Fixes #324 Fixes #325 Refs #321 --- .../qualification/reversible-worker-source.md | 29 ++++++-- src/indexer/helpers/reader-progress.ts | 5 ++ .../history/action-history-manifest.ts | 9 ++- .../history/action-source-admission.ts | 18 ++--- src/indexer/workers/deserializer.ts | 2 +- src/indexer/workers/state-reader.ts | 22 ++++-- tests/unit/action-history-manifest.test.ts | 13 ++++ tests/unit/action-source-admission.test.ts | 20 ++++++ tests/unit/action-source-journal.test.ts | 21 ++++++ tests/unit/reader-coordinator-resume.mjs | 67 +++++++++++++++++++ tests/unit/reader-coordinator-resume.test.ts | 12 ++++ tests/unit/reversible-original-attempts.mjs | 45 ++++++++++--- .../unit/reversible-original-attempts.test.ts | 2 + tests/unit/reversible-original-worker.mjs | 12 +++- .../unit/reversible-worker-evidence-check.mjs | 44 +++++++++++- 15 files changed, 284 insertions(+), 37 deletions(-) create mode 100644 tests/unit/reader-coordinator-resume.mjs create mode 100644 tests/unit/reader-coordinator-resume.test.ts diff --git a/docs/qualification/reversible-worker-source.md b/docs/qualification/reversible-worker-source.md index 4adc2f34..0a3131dd 100644 --- a/docs/qualification/reversible-worker-source.md +++ b/docs/qualification/reversible-worker-source.md @@ -40,21 +40,40 @@ selected-source path. - **revoke**: hold the original post-fresh control response, then revoke through the authenticated physical reader. The real master fences the original DS scope. Its proof rejects reuse and its IO remains physically pending. Reconnect with the reader's new - session/generation; an independent DS worker completes block 34 while the old operation - remains held. Late configuration/target reads refuse association. Redelivery of 34 needs + session/generation and the original reader's confirmed-publication request boundary. + Replay that requested prefix 1..33, then recover the original block-33 delivery on an + independent DS worker with its own fresh result/end and current-source association while + the old operation remains held. Block 34 then continues independently. Late + configuration/target reads refuse association. Redelivery of 34 needs its own conditional write, fresh observation and authentic end. Release the original response only afterward; the old attempt drains with no ACK/association or post-fence effects. - **late-rpc**: hold a canonical RPC result after the actual master captures its source, then revoke that source. The same block identity returns, but its issued observation is stale and a JSON copy cannot become issued. The old authentic end remains unresolved - after replacement; a new source/worker can complete block 34. + after replacement; a new source replays its requested prefix and a worker can complete block 34. -Block 34 is current-source continuation and redelivery. These cases do not claim that the -fenced block-33 delivery has recovered or that a full projection/history replay completed. +The recovered block-33 association belongs to a new physical attempt and source selection; +its original failed end remains unresolved. Block 34 is subsequent continuation/redelivery. +These component controls do not establish a completed full projection/history replay. Old failed/unknown outcomes and broker settlement remain explicit. No reader grant, queue budget, deadline, ancestry limit, dependency, public activation or production setting changes. +## Publication and plan identities + +The reader's confirmed, contiguous irreversible publication cursor is independent of canonical +ABI coverage. Reopening selects the earlier required boundary, preserves its anchor when +applicable, and creates a fresh publication epoch. Callbacks from an old checkpoint, source +context or physical socket cannot advance the new source. The strict finalized-prefix rewind +guard remains active; the source replays the actual requested prefix instead of bypassing it. + +Newly produced retained action sources use schema 2. Their history plans include the exact +retained source ID, keeping per-selection output provenance in separate immutable contexts. +Complete document fingerprints still include protocol selection/native-link metadata, and a +different result within the same plan still conflicts. Schema-1 sources keep their original +plan identities and remain readable/retryable. These internal records do not activate the +engineering action-history path or grant readiness, sink completion or capacity authority. + ## Running and retaining observations Use the normal npm lock installation, complete project check and native TypeScript build. diff --git a/src/indexer/helpers/reader-progress.ts b/src/indexer/helpers/reader-progress.ts index 1721cd10..cae4f2f4 100644 --- a/src/indexer/helpers/reader-progress.ts +++ b/src/indexer/helpers/reader-progress.ts @@ -19,6 +19,11 @@ export class LiveReaderCheckpoint { this.anchorId = anchorId; } + /** Publication progress only; ABI coverage and unconfirmed reversible observations cannot advance it. */ + resumePoint(): Readonly<{nextBlock: number; blockId: string}> { + return Object.freeze({nextBlock: this.nextBlock, blockId: this.anchorId}); + } + /** The physical reader's current reversible branch, including an unconfirmed replacement suffix. */ hasCurrentReversible(position: ReaderBlockPosition): boolean { const block = this.blocks.get(position.num); diff --git a/src/indexer/history/action-history-manifest.ts b/src/indexer/history/action-history-manifest.ts index 5a6a19ab..d525d6ca 100644 --- a/src/indexer/history/action-history-manifest.ts +++ b/src/indexer/history/action-history-manifest.ts @@ -13,7 +13,9 @@ export interface ActionHistoryTransactionPlan { /** Expected source membership, prepared before output publication; surviving ES rows cannot construct it. */ export interface ActionHistoryBlockPlan { readonly schemaVersion: 1; - readonly source: Readonly<{chainId: string; blockNum: number; blockId: string; previousBlockId: string; captureSha256: string}>; + readonly source: Readonly<{chainId: string; blockNum: number; blockId: string; previousBlockId: string; captureSha256: string; + /** Exact retained input context for source-scoped plans. Legacy plans omit it. */ + retainedSourceId?: string}>; readonly producer: Readonly<{protocolId: string; profileId: string; codeRevision: string; configurationSha256: string; pluginsSha256: string}>; readonly target: Readonly<{name: string; uuid: string}>; readonly transactions: readonly ActionHistoryTransactionPlan[]; @@ -109,8 +111,11 @@ function freeze(value: T): T { } function validatePlan(record: any): ActionHistoryBlockPlan { + const sourceFields = ['chainId', 'blockNum', 'blockId', 'previousBlockId', 'captureSha256']; + const scoped = Object.hasOwn(record?.source ?? {}, 'retainedSourceId'); if (!keys(record, ['schemaVersion', 'source', 'producer', 'target', 'transactions']) || record.schemaVersion !== 1 || - !keys(record.source, ['chainId', 'blockNum', 'blockId', 'previousBlockId', 'captureSha256']) || + !keys(record.source, scoped ? [...sourceFields, 'retainedSourceId'] : sourceFields) || + (scoped && (typeof record.source.retainedSourceId !== 'string' || !/^h45actionsource1:[0-9a-f]{64}$/.test(record.source.retainedSourceId))) || !digest(record.source.chainId) || !uint32(record.source.blockNum, 1) || !digest(record.source.blockId) || !digest(record.source.previousBlockId) || !digest(record.source.captureSha256) || !keys(record.producer, ['protocolId', 'profileId', 'codeRevision', 'configurationSha256', 'pluginsSha256']) || diff --git a/src/indexer/history/action-source-admission.ts b/src/indexer/history/action-source-admission.ts index 5de513c2..ce395c25 100644 --- a/src/indexer/history/action-source-admission.ts +++ b/src/indexer/history/action-source-admission.ts @@ -24,7 +24,8 @@ export interface ActionSourceIngress { readerSessionId: string; schemaJson: string; frameHex: string; captureSha256: string; } export interface RetainedActionSource { - readonly schemaVersion: 1; + /** Schema 2 scopes the output plan to this immutable input; schema 1 retains its legacy identity. */ + readonly schemaVersion: 1 | 2; /** Original broker frame and handshake, retained before filtering. These bytes are not consensus attestation. */ readonly frameHex: string; readonly schemaJson: string; @@ -133,7 +134,7 @@ export class ActionSourceAdmissionStore { } private validateSource(input: unknown): RetainedActionSource { const source = copy(input) as RetainedActionSource; - if (!source || source.schemaVersion !== 1 || typeof source.frameHex !== 'string' || !/^(?:[0-9a-f]{2})+$/.test(source.frameHex) + if (!source || (source.schemaVersion !== 1 && source.schemaVersion !== 2) || typeof source.frameHex !== 'string' || !/^(?:[0-9a-f]{2})+$/.test(source.frameHex) || typeof source.schemaJson !== 'string' || !source.schemaJson.length || !hex64.test(source.captureSha256) || !hex64.test(source.executableSha256) || (source.previousBlockId !== null && !hex64.test(source.previousBlockId))) throw new Error('Invalid retained action source'); @@ -150,12 +151,13 @@ export class ActionSourceAdmissionStore { if (sourceIdentity(source) !== id) throw new Error('Retained action source identity mismatch'); return source; } - private compile(source: RetainedActionSource, target: ActionHistoryBlockPlan['target']) { + private compile(source: RetainedActionSource, target: ActionHistoryBlockPlan['target'], sourceId: string) { const inventory = deriveActionSourceInventory(source.inventoryInput); if (source.previousBlockId === null) throw new Error('Missing source predecessor cannot establish complete action membership'); const plan = validateActionHistoryBlockPlan({schemaVersion: 1, source: {chainId: inventory.source.chainId, blockNum: inventory.source.blockNum, blockId: inventory.source.blockId, - previousBlockId: source.previousBlockId, captureSha256: source.captureSha256}, producer: source.producer, target, + previousBlockId: source.previousBlockId, captureSha256: source.captureSha256, + ...(source.schemaVersion === 2 ? {retainedSourceId: sourceId} : {})}, producer: source.producer, target, transactions: inventory.transactions.map(({transactionId, traceOrdinal, traceSha256, disposition}) => ({transactionId, traceOrdinal, traceSha256, disposition}))}); return {inventory, plan}; @@ -165,7 +167,7 @@ export class ActionSourceAdmissionStore { return this.bounded(async () => { const source = await this.source(sourceId, signal); let compiled: ReturnType | undefined, incomplete: string | null = null; - try {compiled = this.compile(source, target);} catch (error) { + try {compiled = this.compile(source, target, sourceId);} catch (error) { // Only this pure compilation is classified as source-incomplete. I/O errors must escape without ACK. incomplete = String((error as Error)?.message ?? error).slice(0, 4096).toWellFormed(); } @@ -194,7 +196,7 @@ export class ActionSourceAdmissionStore { } if (link.incomplete !== null) throw new Error('Complete action admission has an incomplete cause'); // Re-issue from original retained source/policy. Stored expected output JSON is never authority. - const {inventory, plan} = this.compile(source, link.target); + const {inventory, plan} = this.compile(source, link.target, link.sourceId); const stored = await this.journal.readPlan(link.planId, signal); if (actionHistoryImmutableJson(stored.plan) !== actionHistoryImmutableJson(plan)) throw new Error('Action admission plan mismatch'); return {id, sourceId: link.sourceId, source, target: link.target, planId: link.planId, inventory, incomplete: null}; @@ -300,7 +302,7 @@ export class ActionSourceAdmissionStore { || !hex64.test(entry.payloadSha256) || !hex64.test(entry.headersSha256)) || outcome.effectRoot !== sha(actionHistoryImmutableJson({fragmentId: outcome.fragmentId, effects: outcome.effects}))) throw new Error('Invalid observed confirmed action handoff'); - const plan = this.compile(current.source, current.target).plan; + const plan = this.compile(current.source, current.target, current.sourceId).plan; const fragment = createActionHistoryFragment(plan, work.transactionId, outcome.effects.map(entry => entry.effect)); if (actionHistoryFragmentId(fragment, plan) !== outcome.fragmentId) throw new Error('Observed action handoff fragment/root mismatch'); signal?.throwIfAborted(); return {id, outcome: copy(outcome)}; @@ -352,7 +354,7 @@ export class ActionSourceAdmissionStore { let reason: ActionAttemptDisposition['reason'] | undefined; if (current.source.inventoryInput.traces === null || current.source.inventoryInput.traces === undefined) reason = 'source-traces-missing'; else if (current.source.previousBlockId === null) reason = 'source-predecessor-missing'; - else try {this.compile(current.source, current.target);} catch (error) {if (isActionSourceValidationFailure(error)) reason = 'source-inventory-invalid';} + else try {this.compile(current.source, current.target, current.sourceId);} catch (error) {if (isActionSourceValidationFailure(error)) reason = 'source-inventory-invalid';} return reason ? [observation(reason, 'source-unprocessable')] : []; } const transactions = current.inventory!.transactions, result: ActionAttemptDisposition[] = []; diff --git a/src/indexer/workers/deserializer.ts b/src/indexer/workers/deserializer.ts index e32c1975..89dc96e4 100644 --- a/src/indexer/workers/deserializer.ts +++ b/src/indexer/workers/deserializer.ts @@ -409,7 +409,7 @@ export default class MainDSWorker extends HyperionWorker { ...(wireActionInputs ? {wireActionInputs} : {}), ...(this.usesExtensionProtocol ? {sourceSelectionId: this.activeSourceSelectionId} : {})}; const store = this.getActionAdmissionStore(); - const sourceId = await store.retainSource({schemaVersion: 1, frameHex: scope.frameHex, + const sourceId = await store.retainSource({schemaVersion: 2, frameHex: scope.frameHex, schemaJson: scope.schemaJson, captureSha256: capture.captureSha256, ingressId: scope.ingressId, previousBlockId: capture.source.previousBlockId, producer: epoch.producer, executableSha256: epoch.executableSha256, inventoryInput}); diff --git a/src/indexer/workers/state-reader.ts b/src/indexer/workers/state-reader.ts index f626d74b..5e0dfc99 100644 --- a/src/indexer/workers/state-reader.ts +++ b/src/indexer/workers/state-reader.ts @@ -866,9 +866,14 @@ export default class StateReader extends HyperionWorker { const lib = res.last_irreversible; let confirmLive: (() => {nextBlock: number; blockId: string} | undefined) | undefined; try { - confirmLive = this.isLiveReader ? this.liveCheckpoint?.observe({ - num: blk_num, id: blk_id, previous: res.prev_block?.block_id.toString() ?? '' - }, protocolNumber(lib.block_num)) : undefined; + const checkpoint = this.isLiveReader ? this.liveCheckpoint : undefined; + if (checkpoint) { + const generation = this.physicalIngressGeneration, context = this.sourceContext; + const confirm = checkpoint.observe({num: blk_num, id: blk_id, + previous: res.prev_block?.block_id.toString() ?? ''}, protocolNumber(lib.block_num)); + confirmLive = () => this.liveCheckpoint === checkpoint && this.physicalIngressGeneration === generation + && this.sourceContext === context ? confirm() : undefined; + } } catch (error: any) { this.failSourceRecovery(error.message); return 0; @@ -1164,10 +1169,15 @@ export default class StateReader extends HyperionWorker { source.generation = opened.generation; this.coordinatorResumeFromBlock = opened.resumeFromBlock; if (opened.resumeFromBlock !== null) { - this.local_block_num = opened.resumeFromBlock - 1; + const published = this.liveCheckpoint?.resumePoint(); + const resume = published ? Math.min(published.nextBlock, opened.resumeFromBlock) : opened.resumeFromBlock; + this.coordinatorResumeFromBlock = resume; + this.local_block_num = resume - 1; // This is a new local publication epoch, not a fabricated master checkpoint. - // Only fresh confirmed irreversible publications may emit live_checkpoint. - this.liveCheckpoint = new LiveReaderCheckpoint(opened.resumeFromBlock, '', this.conf.settings.reader_recovery_window_blocks); + // ABI coverage cannot skip publication recovery; only fresh confirmed irreversible + // publications advance this cursor. Preserve its anchor when both starts agree. + this.liveCheckpoint = new LiveReaderCheckpoint(resume, published?.nextBlock === resume ? published.blockId : '', + this.conf.settings.reader_recovery_window_blocks); } } catch (error) { this.assertCurrentIngress(context.generation); diff --git a/tests/unit/action-history-manifest.test.ts b/tests/unit/action-history-manifest.test.ts index 9dc62ee9..c96af85b 100644 --- a/tests/unit/action-history-manifest.test.ts +++ b/tests/unit/action-history-manifest.test.ts @@ -41,6 +41,19 @@ const complete = (source = plan()) => [ ]; describe('immutable action history source membership', () => { + it('binds an optional retained source identity without changing legacy plan identities', () => { + const legacy = plan(), before = actionHistoryImmutableJson(legacy); + const scoped = {...legacy, source: {...legacy.source, retainedSourceId: 'h45actionsource1:' + hex('a')}}; + const other = {...scoped, source: {...scoped.source, retainedSourceId: 'h45actionsource1:' + hex('b')}}; + expect(actionHistoryPlanId(scoped)).not.toBe(actionHistoryPlanId(other)); + expect(() => validateActionHistoryFragment(complete(scoped)[0], other)).toThrow('fragment binding'); + expect(actionHistoryImmutableJson(validateActionHistoryBlockPlan(legacy))).toBe(before); + expect(actionHistoryPlanId(legacy)).toBe('h45actionplan1:' + hash(actionHistoryImmutableJson(['hyperion-action-history-plan-v1', legacy]))); + for (const invalid of [null, '', 'h45actionsource1:' + hex('A'), 'h45actionsource2:' + hex('a'), hex('a')]) { + const value: any = structuredClone(scoped); value.source.retainedSourceId = invalid; + expect(() => validateActionHistoryBlockPlan(value)).toThrow('Invalid action history source block plan'); + } + }); it('returns separate frozen source plans and deterministic domain-specific identities', () => { const input = plan(), before = structuredClone(input), validated = validateActionHistoryBlockPlan(input); expect(validated).toEqual(input); expect(validated).not.toBe(input); diff --git a/tests/unit/action-source-admission.test.ts b/tests/unit/action-source-admission.test.ts index 5f1d49f1..368cdcaa 100644 --- a/tests/unit/action-source-admission.test.ts +++ b/tests/unit/action-source-admission.test.ts @@ -73,6 +73,26 @@ function searchResponse(ids: string[]): any { } describe('source admission and complete grouped transaction preparation', () => { + it('scopes schema-2 history plans to each retained input context while preserving legacy plans', async () => { + const {store, source, sourceId} = await retained(); + const legacy = await store.admit(sourceId, target); + const legacyPlan = await store.journal.readPlan(legacy.planId!); + expect(Object.hasOwn(legacyPlan.plan.source, 'retainedSourceId')).toBe(false); + const first: RetainedActionSource = {...source, schemaVersion: 2}; + const firstId = await store.retainSource(first), admittedFirst = await store.admit(firstId, target); + const nextIngress = await store.retainIngress({schemaVersion: 1, chainId: hex('1'), protocolId: 'antelope', + profileId: 'antelope-default-v1', readerSessionId: 'reader-session-2', components: [], schemaJson: source.schemaJson, + frameHex: source.frameHex, captureSha256: source.captureSha256}); + const secondId = await store.retainSource({...first, ingressId: nextIngress}); + const admittedSecond = await store.admit(secondId, target); + expect(admittedFirst.planId).not.toBe(admittedSecond.planId); + expect((await store.journal.readPlan(admittedFirst.planId!)).plan.source).toMatchObject({retainedSourceId: firstId}); + expect((await store.journal.readPlan(admittedSecond.planId!)).plan.source).toMatchObject({retainedSourceId: secondId}); + expect((await store.resolve(legacy.id)).planId).toBe(legacy.planId); + expect((await store.admit(sourceId, target)).planId).toBe(legacy.planId); + expect((await store.resolve(admittedFirst.id)).planId).toBe(admittedFirst.planId); + expect((await store.resolve(admittedSecond.id)).planId).toBe(admittedSecond.planId); + }); it('re-derives actual source inventory, persists both groups and seals after restart', async () => { const {memory, store, sourceId, admission, inventory, payload} = await admitted(); expect(inventory.transactions[0].groups.map(value => value.actionOrdinals)).toEqual([[1, 2], [3]]); diff --git a/tests/unit/action-source-journal.test.ts b/tests/unit/action-source-journal.test.ts index 9a6bae5b..1946720a 100644 --- a/tests/unit/action-source-journal.test.ts +++ b/tests/unit/action-source-journal.test.ts @@ -71,6 +71,27 @@ function captureIo(memory: ReturnType, after: (meth } describe('durable action source membership journal', () => { + it('retains complete session provenance in distinct source-scoped plans and refuses changes within one plan', async () => { + const {journal} = fixture(1), legacy = plan(); + const firstPlan = {...legacy, source: {...legacy.source, retainedSourceId: 'h45actionsource1:' + hex('a')}, + transactions: legacy.transactions.slice(0, 1)}; + const secondPlan = {...firstPlan, source: {...firstPlan.source, retainedSourceId: 'h45actionsource1:' + hex('b')}}; + const firstId = (await journal.preparePlan(firstPlan)).id, secondId = (await journal.preparePlan(secondPlan)).id; + // Constructed locator shapes test byte identity only; they confer no native/source authority. + const first = {...action(), protocol: {selectionId: 'h45srcsel1:' + hex('c'), native: {linkId: 'h45srcnat1:' + hex('d')}}}; + const second = {...action(), protocol: {selectionId: 'h45srcsel1:' + hex('e'), native: {linkId: 'h45srcnat1:' + hex('f')}}}; + expect(first.act.data).toEqual(second.act.data); + expect(fingerprintActionHistoryEffect(first).documentSha256).not.toBe(fingerprintActionHistoryEffect(second).documentSha256); + const original = await journal.recordFragment(firstId, txA, [first]); + const recovered = await journal.recordFragment(secondId, txA, [second]); + expect(original.id).not.toBe(recovered.id); + const firstSeal = await journal.seal(firstId), secondSeal = await journal.seal(secondId); + expect(firstSeal.manifest.fragments[0]).toEqual(original.fragment); + expect(secondSeal.manifest.fragments[0]).toEqual(recovered.fragment); + await expect(journal.recordFragment(firstId, txA, [second])).rejects.toThrow('Immutable ABI observation conflict'); + expect(await journal.readManifest(firstId)).toEqual(firstSeal); + expect(await journal.readManifest(secondId)).toEqual(secondSeal); + }); it('stores a strict independent namespace and survives restart with source-ordered complete membership', async () => { const {memory, journal, id, source, fragments} = await prepared(1), sealed = await journal.seal(id); expect(journal.index).toBe('fixture-history-source-journal-v1'); diff --git a/tests/unit/reader-coordinator-resume.mjs b/tests/unit/reader-coordinator-resume.mjs new file mode 100644 index 00000000..16f75d21 --- /dev/null +++ b/tests/unit/reader-coordinator-resume.mjs @@ -0,0 +1,67 @@ +/** Minimized original reader resume/publication callbacks; coordinator and broker transport controls only. */ +import assert from 'node:assert/strict'; +import {mkdtempSync, mkdirSync, writeFileSync, rmSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join, resolve} from 'node:path'; + +const mode = process.argv[2]; +assert(['reversible', 'abi-behind', 'anchor', 'current-confirm', 'retired-socket', 'retired-context', 'retired-checkpoint'].includes(mode)); +const directory = mkdtempSync(join(tmpdir(), 'hyperion-reader-resume-')); +const originalDirectory = process.cwd(), originalConfig = process.env.CONFIG_JSON, originalSend = process.send; +try { + mkdirSync(join(directory, 'config')); + writeFileSync(join(directory, 'chain.json'), JSON.stringify({settings: {chain: 'reader-resume'}, indexer: {}})); + writeFileSync(join(directory, 'config/connections.json'), '{}'); + process.chdir(directory); process.env.CONFIG_JSON = join(directory, 'chain.json'); + process.send = () => true; + const [{default: Reader}, {LiveReaderCheckpoint}] = await Promise.all([ + import('../../build/indexer/workers/state-reader.js'), import('../../build/indexer/helpers/reader-progress.js')]); + const commands = [], payloads = []; + const checkpoint = new LiveReaderCheckpoint(mode === 'anchor' || mode === 'abi-behind' ? 34 : 1, + mode === 'anchor' || mode === 'abi-behind' ? 'thirty-three' : '', 256); + if (mode === 'reversible') for (let number = 1; number <= 33; number++) + assert.equal(checkpoint.observe({num: number, id: String(number), previous: String(number - 1)}, 0)(), undefined); + const reader = Object.assign(Object.create(Reader.prototype), {chain: 'reader-resume', + conf: {settings: {reader_recovery_window_blocks: 256}, indexer: {}}, + protocol: {profileId: 'wire-sysio-6508c9e8-native-engineering-v1'}, + sourceContext: {id: 'controlled-context', readerSessionId: 'new-reader', generation: 1}, + verifiedReaderSessionId: 'new-reader', physicalIngressGeneration: 1, activeIngressGeneration: 1, socketGeneration: 1, + isLiveReader: true, liveCheckpoint: checkpoint, baseRequest: {}, shipRev: 'v0', ship: {connected: true}, + coordinatorRpc: {open: async () => ({generation: 'controlled-generation', resumeFromBlock: mode === 'abi-behind' ? 11 : 34})}, + profiler: {start: () => () => {}}, send(request) {commands.push(structuredClone(request));}, + reversibleBlockMap: new Map(), forkedBlocks: new Map(), local_lib: 1, + stageOneDistQueue: {async push(payload) {payloads.push(payload);}}}); + if (['reversible', 'abi-behind', 'anchor'].includes(mode)) { + await reader.openCoordinatorSource(); + reader.requestBlocks(34); + const expectedStart = mode === 'reversible' ? 1 : mode === 'abi-behind' ? 11 : 34; + assert.equal(commands[0][1].start_block_num, expectedStart, 'ABI coverage must not skip confirmed-publication recovery'); + assert.notEqual(reader.liveCheckpoint, checkpoint, 'A reopened source needs its own publication epoch'); + if (mode === 'reversible') { + assert.equal(reader.liveCheckpoint.observe({num: 33, id: '33', previous: '32'}, 0)(), undefined); + assert(reader.liveCheckpoint.hasCurrentReversible({num: 33, id: '33', previous: '32'})); + } else if (mode === 'anchor') { + const wrong = reader.liveCheckpoint.observe({num: 34, id: 'thirty-four', previous: 'wrong'}, 34); + assert.throws(wrong, /Unlinked live block/); + } + } else { + const frame = {this_block: {block_num: 1, block_id: 'one'}, prev_block: {block_num: 0, block_id: ''}, + last_irreversible: {block_num: 1, block_id: 'one'}, block: '00', traces: '00', deltas: '00'}; + // Only publication ownership is under test here; source binding is covered by the full native fixture. + reader.bindProtocolSource = async () => ({}); + assert.equal(await reader.handleBlockResult(frame, Buffer.from('controlled-frame')), 1); + assert.equal(payloads.length, 1); + if (mode === 'retired-socket') reader.physicalIngressGeneration++; + if (mode === 'retired-context') reader.sourceContext = {...reader.sourceContext, readerSessionId: 'replacement-reader'}; + if (mode === 'retired-checkpoint') reader.liveCheckpoint = new LiveReaderCheckpoint(1, '', 256); + const position = payloads[0].confirmLive(); + assert.deepEqual(position, mode === 'current-confirm' ? {nextBlock: 2, blockId: 'one'} : undefined, + 'Retired publication callbacks must not advance a replacement source'); + } + console.log(JSON.stringify({passed: true, mode, originalReaderMethods: true, controlledCoordinatorAndPublicationTransport: true, + capacityAuthority: false, releaseQualified: false})); +} finally { + process.send = originalSend; process.chdir(originalDirectory); + if (originalConfig === undefined) delete process.env.CONFIG_JSON; else process.env.CONFIG_JSON = originalConfig; + assert.equal(dirname(resolve(directory)), resolve(tmpdir())); rmSync(directory, {recursive: true, force: true}); +} diff --git a/tests/unit/reader-coordinator-resume.test.ts b/tests/unit/reader-coordinator-resume.test.ts new file mode 100644 index 00000000..0d2e921d --- /dev/null +++ b/tests/unit/reader-coordinator-resume.test.ts @@ -0,0 +1,12 @@ +import {expect, test} from 'bun:test'; +import {fileURLToPath} from 'node:url'; + +for (const mode of ['reversible', 'abi-behind', 'anchor', 'current-confirm', 'retired-socket', 'retired-context', 'retired-checkpoint']) + test('original reader keeps publication progress separate from ABI coverage: ' + mode, async () => { + const child = Bun.spawn([process.env.HYPERION_NATIVE_NODE ?? 'node', + fileURLToPath(new URL('./reader-coordinator-resume.mjs', import.meta.url)), mode], + {stdout: 'pipe', stderr: 'pipe', timeout: 10_000}); + const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + expect({code, stdout, stderr}).toMatchObject({code: 0, stderr: ''}); + expect(JSON.parse(stdout.trim().split('\n').at(-1)!)).toMatchObject({passed: true, mode, capacityAuthority: false, releaseQualified: false}); + }, 15_000); diff --git a/tests/unit/reversible-original-attempts.mjs b/tests/unit/reversible-original-attempts.mjs index 29d2934d..448b5f56 100644 --- a/tests/unit/reversible-original-attempts.mjs +++ b/tests/unit/reversible-original-attempts.mjs @@ -140,18 +140,25 @@ try { assert.equal(master.protocolAbiSource.worker, reader.child); assert.equal(master.protocolAbiSource.readerSessionId, sourceOpened.readerSessionId); assert.equal(master.protocolAbiSource.generation, sourceOpened.coordinatorGeneration); - let lastMain = 0; + let lastMain = 0, nextMainTag = 0; async function frame(number) { const before = publications.length; const receipt = await command(reader, {operation: 'frame', frameHex: input.frames[number - 1].frameHex}); const raw = publications.slice(before).filter(p => p.worker === 'reader'); assert.equal(raw.length, 1); assert.equal(raw[0].payloadHex, input.frames[number - 1].frameHex); - await command(main, {operation: 'delivery', delivery: raw[0], tag: number}); + await command(main, {operation: 'delivery', delivery: raw[0], tag: ++nextMainTag}); await coordinator.drain(); const published = publications.slice(lastMain).filter(p => p.worker === 'main' && p.options.headers.block_num === number); lastMain = publications.length; return {receipt, published}; } + async function replayRequestedSourcePrefix(reopened) { + assert.equal(reopened.requestedStartBlock, 1, 'ABI coverage must not skip unconfirmed reader publication recovery'); + let last; + for (let number = reopened.requestedStartBlock; number <= 33; number++) last = await frame(number); + assert(last); + return last.published.find(p => p.options.headers.action_trace_ordinal === 0); + } let selectedDispatch; for (let number = 1; number <= 33; number++) { const result = await frame(number); @@ -227,11 +234,23 @@ try { assert.notEqual(replacement.readerSessionId, sourceOpened.readerSessionId); assert.notEqual(replacement.coordinatorGeneration, sourceOpened.coordinatorGeneration); assert.equal(master.protocolAbiSource.readerSessionId, replacement.readerSessionId); + const recoveredDelivery = await replayRequestedSourcePrefix(replacement); + assert(recoveredDelivery); assert.equal(held.settled, false); assert.equal(ownEnd(first), undefined); + const recoveredSelectionId = recoveredDelivery.options.headers['hyperion-source-selection-v1']; + assert.notEqual(recoveredSelectionId, oldSelectionId); + const recoveredSelection = (await bindings.readSelection(recoveredSelectionId)).selection; + assert.equal(recoveredSelection.readerSessionId, replacement.readerSessionId); + assert.deepEqual(recoveredSelection.source, selected.source); + assert.equal(recoveredSelection.captureSha256, selected.captureSha256); + await command(peer, {operation: 'delivery', delivery: recoveredDelivery, tag: 1}); + const recoveredOriginalDelivery = await associate(peer, 33); + assert.equal(recoveredOriginalDelivery.end.binding.selectionId, recoveredSelectionId); + assert.equal(held.settled, false); assert.equal(ownEnd(first), undefined); const next = await frame(34), delivery = next.published.find(p => p.options.headers.action_trace_ordinal === 0); assert(delivery); - await command(peer, {operation: 'delivery', delivery, tag: 1}); + await command(peer, {operation: 'delivery', delivery, tag: 2}); // The original required IO is still owned while an independent original worker genuinely completes. assert.equal(held.settled, false); assert.equal(ownEnd(first), undefined); - const peerEvidence = ownEnd(peer); assert(peerEvidence); + const peerEvidence = ownEnd(peer, 1); assert(peerEvidence); for (const changed of ['configuration', 'target']) { const originalSettings = memory.client.indices.getSettings; let entered, release, once = false; @@ -245,13 +264,14 @@ try { const pending = adjudicator.adjudicate(peerEvidence); pending.catch(() => {}); await enteredRead; if (changed === 'configuration') config.indexer.disable_indexing = true; else memory.settings.get(chain + '-action-v1-000001')['index.uuid'] = 'replaced-physical-target'; - release(); assert.deepEqual(await pending, {state: 'unresolved'}); assert.equal(associationIds().length, 0); + release(); assert.deepEqual(await pending, {state: 'unresolved'}); + assert.deepEqual(associationIds(), [recoveredOriginalDelivery.associationId]); } finally { release(); memory.client.indices.getSettings = originalSettings; config.indexer.disable_indexing = false; memory.settings.get(chain + '-action-v1-000001')['index.uuid'] = 'owned-reversible-action-target'; } } - result = await associate(peer, 34); + result = await associate(peer, 34, 1); const priorPeerEnd = result.end, conditional = []; memory.faults.beforeWrite = async (request, method) => { if (request.index === store.index && request.id === priorPeerEnd.freshWrites[0].id && method === 'index') { @@ -259,9 +279,9 @@ try { assert(Number.isSafeInteger(request.if_primary_term)); conditional.push(structuredClone(request)); } }; - await command(peer, {operation: 'delivery', delivery, tag: 2, redelivered: true}); + await command(peer, {operation: 'delivery', delivery, tag: 3, redelivered: true}); delete memory.faults.beforeWrite; - const redelivery = await associate(peer, 34, 1); + const redelivery = await associate(peer, 34, 2); assert.equal(conditional.length, 1); assert.equal(redelivery.end.incarnation, priorPeerEnd.incarnation); assert.notEqual(redelivery.end.attemptId, priorPeerEnd.attemptId); assert.equal(redelivery.end.freshWrites[0].id, priorPeerEnd.freshWrites[0].id); @@ -272,6 +292,8 @@ try { const oldEnd = describeEnd(supervisor, evidence); assert.equal(oldEnd.rootResult, 'threw'); assert.equal(oldEnd.status, 'quiescent'); assert.equal(oldEnd.acknowledgementAttempted, false); assert.equal(oldEnd.freshWrites.length, 1); + assert.notEqual(recoveredOriginalDelivery.end.incarnation, oldEnd.incarnation); + assert.notEqual(recoveredOriginalDelivery.end.attemptId, oldEnd.attemptId); assert.deepEqual(await adjudicator.adjudicate(evidence), {state: 'unresolved'}); const after = await command(first, {operation: 'effects'}); assert.equal(after.scope.state, 'drained'); assert.equal(after.scope.pendingOperations, 0); @@ -279,9 +301,11 @@ try { assert.equal(after.effects.storageDispatches, fenced.effects.storageDispatches); assert.equal(after.effects.cacheDispatches, fenced.effects.cacheDispatches); assert.equal(after.effects.streamDispatches, fenced.effects.streamDispatches); - assert.equal(associationIds().length, 2); + assert.equal(associationIds().length, 3); revoked = {oldEnd, pendingAtFence: fenced.scope.pendingOperations, retainedResponseSha256: hash(Buffer.from(JSON.stringify(held.response))), - replacement, peerAssociatedBeforeOriginalSettlement: true, redelivery, conditionalWrites: conditional.length, + replacement, recoveredOriginalDelivery: {...recoveredOriginalDelivery, originalSourceSelectionId: oldSelectionId, + currentSourceSelectionId: recoveredSelectionId, sameCapturedSource: true, associatedBeforeOriginalSettlement: true}, + peerAssociatedBeforeOriginalSettlement: true, redelivery, conditionalWrites: conditional.length, currentConfigurationAndTargetLateReadsRejected: true, oldScopeEffectsAfterFence: after.effects}; } else { await command(first, {operation: 'delivery', delivery: selectedDispatch, tag: 1}); @@ -306,6 +330,7 @@ try { const replacement = await command(reader, {operation: 'reconnect', schemaJson: input.schemaJson, statusHex: input.statusHex}); assert.notEqual(replacement.readerSessionId, sourceOpened.readerSessionId); assert.notEqual(replacement.coordinatorGeneration, sourceOpened.coordinatorGeneration); + await replayRequestedSourcePrefix(replacement); const next = await frame(34), delivery = next.published.find(p => p.options.headers.action_trace_ordinal === 0); assert(delivery); await command(peer, {operation: 'delivery', delivery, tag: 1}); result = await associate(peer, 34); diff --git a/tests/unit/reversible-original-attempts.test.ts b/tests/unit/reversible-original-attempts.test.ts index 3fafd16d..8078f59c 100644 --- a/tests/unit/reversible-original-attempts.test.ts +++ b/tests/unit/reversible-original-attempts.test.ts @@ -15,6 +15,8 @@ for (const mode of ['baseline', 'revoke', 'late-rpc']) test('actual reversible n expect(result.processes.every((p: any) => p.exitCode === 0 && /^v(?:24|26)\./.test(p.node))).toBe(true); if (mode === 'revoke') expect(result.revoked).toMatchObject({peerAssociatedBeforeOriginalSettlement: true, conditionalWrites: 1, currentConfigurationAndTargetLateReadsRejected: true, + recoveredOriginalDelivery: {sameCapturedSource: true, associatedBeforeOriginalSettlement: true, + end: {binding: {blockNum: 33}, rootResult: 'returned', status: 'quiescent', acknowledgementAttempted: true}}, oldEnd: {rootResult: 'threw', acknowledgementAttempted: false, status: 'quiescent'}}); if (mode === 'late-rpc') expect(result.lateRpc).toMatchObject({oldCanonicalObservationRejected: true, sourceStillHasSameCanonicalBlockIdentity: true, oldEndUnresolvedAfterReplacement: true}); diff --git a/tests/unit/reversible-original-worker.mjs b/tests/unit/reversible-original-worker.mjs index ae105df3..47a2fdda 100644 --- a/tests/unit/reversible-original-worker.mjs +++ b/tests/unit/reversible-original-worker.mjs @@ -66,7 +66,9 @@ try { worker.configureWorkerAttemptLifecycle(managed); } if (role === 'continuous_reader') { - Object.assign(worker, {baseRequest: {}, shipRev: 0, socketGeneration: 1, activeIngressGeneration: 1, + Object.assign(worker, {baseRequest: {max_messages_in_flight: 1, have_positions: [], irreversible_only: false, + fetch_block: true, fetch_traces: true, fetch_deltas: true}, shipRev: 'v0', ship: {connected: true}, + socketGeneration: 1, activeIngressGeneration: 1, physicalIngressGeneration: 1, isLiveReader: true, local_block_num: 0, local_lib: 0, receivedFirstBlock: true, reversibleBlockMap: new Map(), forkedBlocks: new Map(), publications: new Map(), sourceProbes: new Set(), liveCheckpoint: new LiveReaderCheckpoint(1, '', 256), @@ -94,7 +96,9 @@ try { if (role === 'continuous_reader') { await worker.onMessage(Buffer.from(input.schemaJson)); await worker.blockReadingQueue.push({kind: 'frame', data: Buffer.from(input.statusHex, 'hex')}); - return {readerSessionId: worker.sourceContext.readerSessionId, coordinatorGeneration: worker.coordinatorSource.generation}; + worker.requestBlocks(1); + return {readerSessionId: worker.sourceContext.readerSessionId, coordinatorGeneration: worker.coordinatorSource.generation, + requestedStartBlock: worker.baseRequest.start_block_num}; } await worker.initializeProtocolShip(input.schemaJson); return {}; } @@ -114,7 +118,9 @@ try { await worker.blockReadingQueue.push({kind: 'connected', generation: worker.physicalIngressGeneration}); await worker.onMessage(Buffer.from(input.schemaJson)); await worker.blockReadingQueue.push({kind: 'frame', data: Buffer.from(input.statusHex, 'hex')}); - return {readerSessionId: worker.sourceContext.readerSessionId, coordinatorGeneration: worker.coordinatorSource.generation}; + worker.requestBlocks(1); + return {readerSessionId: worker.sourceContext.readerSessionId, coordinatorGeneration: worker.coordinatorSource.generation, + requestedStartBlock: worker.baseRequest.start_block_num}; } if (input.operation === 'delivery') { assert.notEqual(role, 'continuous_reader'); diff --git a/tests/unit/reversible-worker-evidence-check.mjs b/tests/unit/reversible-worker-evidence-check.mjs index 0156b0a5..e898ced8 100644 --- a/tests/unit/reversible-worker-evidence-check.mjs +++ b/tests/unit/reversible-worker-evidence-check.mjs @@ -6,6 +6,7 @@ import {fileURLToPath} from 'node:url'; import {ActionSourceAdmissionStore} from '../../build/indexer/history/action-source-admission.js'; import {ActionHistorySourceJournal} from '../../build/indexer/history/action-source-journal.js'; import {ImmutableProtocolBundleStore} from '../../build/indexer/protocol/immutable-bundle-store.js'; +import {ProtocolSourceBindingStore} from '../../build/indexer/protocol/source-bindings.js'; import {frontierMemoryClient} from './protocol-frontier-memory.mjs'; const hash = value => createHash('sha256').update(value).digest('hex'); @@ -81,7 +82,10 @@ export async function checkReversibleWorkerEvidence(path) { store.bundles = new ImmutableProtocolBundleStore(memory.client, 'reversible-worker-original', 1, 'action-source-admission', 'history-source-admission-v1', undefined, {readOnly: true}); store.journal = new ActionHistorySourceJournal(memory.client, 'reversible-worker-original', 1, {readOnly: true}); - const associations = [report.associationId, ...(report.revoked ? [report.revoked.redelivery.associationId] : [])]; + const recovered = report.revoked?.recoveredOriginalDelivery; + if (report.revoked) assert(recovered, 'Revocation evidence lacks recovery of its original delivery'); + const associations = [report.associationId, ...(report.revoked ? [recovered.associationId, report.revoked.redelivery.associationId] : [])]; + assert.equal(new Set(associations).size, associations.length); for (const id of associations) { const observed = (await store.readAttemptAssociation(id)).observation; assert.equal(observed.sourceChecked, true); assert.equal(observed.brokerSettlement, 'unknown'); @@ -97,14 +101,50 @@ export async function checkReversibleWorkerEvidence(path) { const transaction = admission.inventory.transactions.find(t => t.traceOrdinal === old.binding.traceOrdinal); const retained = await store.readConfirmedHandoff(admission.id, old.binding.traceOrdinal, Buffer.from(transaction.dispatch.canonicalJson)); assert.equal(retained.id, old.freshWrites[0].id); + assert.equal(recovered.end.binding.blockNum, old.binding.blockNum); + assert.equal(recovered.end.binding.blockId, old.binding.blockId); + assert.notEqual(recovered.end.incarnation, old.incarnation); assert.notEqual(recovered.end.attemptId, old.attemptId); + assert.equal(recovered.end.rootResult, 'returned'); assert.equal(recovered.end.acknowledgementAttempted, true); + assert.equal(recovered.end.binding.selectionId, recovered.currentSourceSelectionId); + assert.notEqual(recovered.currentSourceSelectionId, recovered.originalSourceSelectionId); + const sources = new ProtocolSourceBindingStore(memory.client, 'reversible-worker-original', 1, {readOnly: true}); + const originalSelection = (await sources.readSelection(recovered.originalSourceSelectionId)).selection; + const currentSelection = (await sources.readSelection(recovered.currentSourceSelectionId)).selection; + assert.deepEqual(currentSelection.source, originalSelection.source); + assert.equal(currentSelection.captureSha256, originalSelection.captureSha256); + assert.equal(currentSelection.readerSessionId, report.revoked.replacement.readerSessionId); + assert.notEqual(currentSelection.readerSessionId, originalSelection.readerSessionId); + const recoveredAdmission = await store.resolve(recovered.end.binding.admissionId); + assert.equal(recoveredAdmission.source.schemaVersion, 2); + assert.equal((await store.journal.readPlan(recoveredAdmission.planId)).plan.source.retainedSourceId, recoveredAdmission.sourceId); + assert.notEqual(recoveredAdmission.planId, admission.planId); + assert.equal(recoveredAdmission.source.captureSha256, admission.source.captureSha256); + const recoveredTransaction = recoveredAdmission.inventory.transactions.find(t => t.traceOrdinal === recovered.end.binding.traceOrdinal); + const recoveredHandoff = await store.readConfirmedHandoff(recoveredAdmission.id, recovered.end.binding.traceOrdinal, + Buffer.from(recoveredTransaction.dispatch.canonicalJson)); + assert.equal(recoveredHandoff.id, recovered.end.freshWrites[0].id); for (const id of associations) { const observed = (await store.readAttemptAssociation(id)).observation; assert.notEqual(observed.end.attemptId, old.attemptId); } } + const readerIndex = data.processes.findIndex(p => p.role === 'continuous_reader'); assert(readerIndex >= 0); + const originalFrames = new Map(data.inputs.frames.map(frame => [frame.controlledFrameSha256, frame.number])); + const observedFrames = [], requestedStarts = []; + for (const [messageIndex, message] of data.processes[readerIndex].sent.entries()) { + if (message.event !== 'qualification_command' || message.input.operation !== 'frame') continue; + const frame = Buffer.from(value(['processes', readerIndex, 'sent', messageIndex, 'input', 'frameHex']), 'hex'); + const number = originalFrames.get(hash(frame)); assert(number !== undefined); observedFrames.push(number); + } + for (const message of data.processes[readerIndex].received) if (message.event === 'qualification_ship_request' + && message.request[0] === 'get_blocks_request_v0') requestedStarts.push(message.request[1].start_block_num); + const prefix = Array.from({length: 33}, (_, index) => index + 1); + assert.deepEqual(observedFrames, data.mode === 'baseline' ? prefix : [...prefix, ...prefix, 34]); + assert.deepEqual(requestedStarts, data.mode === 'baseline' ? [1] : [1, 1]); return {passed: true, mode: data.mode, bytes: bytes.length, sha256: hash(bytes), strings: texts.size, references: paths.size, originalFrames: 34, nativeProcesses: 4, originalServiceRequestsAndReplies: requests.size, authenticEndReplies: acceptedEnds.length, - typedAssociationsRead: associations.length, coldWrites: 0, capacityAuthority: false, releaseQualified: false}; + typedAssociationsRead: associations.length, originalDeliveryRecovered: !!recovered, + originalFrameDeliveries: observedFrames.length, requestedStarts, coldWrites: 0, capacityAuthority: false, releaseQualified: false}; } if (process.argv[1] === fileURLToPath(import.meta.url)) console.log(JSON.stringify(await checkReversibleWorkerEvidence(process.argv[2]))); From ab8781449e54b5479811a7c37d9a415836c090c0 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:36:37 -0300 Subject: [PATCH 2/3] test: bind reported worker ends to original evidence Compare all positive report ends with their typed associations and old ends with exact native replies. Reject contradictory report copies without changing stored evidence or original IPC. Retain the reproduced review counterexample. --- .../qualification/reversible-worker-source.md | 5 ++ tests/unit/reversible-original-attempts.mjs | 3 +- .../unit/reversible-original-attempts.test.ts | 51 ++++++++++++------- .../unit/reversible-worker-evidence-check.mjs | 10 ++++ tests/unit/reversible-worker-report-check.mjs | 36 +++++++++++++ 5 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 tests/unit/reversible-worker-report-check.mjs diff --git a/docs/qualification/reversible-worker-source.md b/docs/qualification/reversible-worker-source.md index 0a3131dd..056d7d7f 100644 --- a/docs/qualification/reversible-worker-source.md +++ b/docs/qualification/reversible-worker-source.md @@ -76,6 +76,11 @@ engineering action-history path or grant readiness, sink completion or capacity ## Running and retaining observations +Cold inspection compares every reported end with its complete retained association or original +native reply. Regression controls change each report copy's payload hash while keeping the +stored records, original IPC and native ends unchanged; those contradictory copies must fail. +These constructed contradictions establish saved-record consistency only. + Use the normal npm lock installation, complete project check and native TypeScript build. Run the focused suite on the selected native Node 24 and 26 runtimes: diff --git a/tests/unit/reversible-original-attempts.mjs b/tests/unit/reversible-original-attempts.mjs index 448b5f56..e92fd963 100644 --- a/tests/unit/reversible-original-attempts.mjs +++ b/tests/unit/reversible-original-attempts.mjs @@ -366,7 +366,8 @@ try { if (originalConfig === undefined) delete process.env.CONFIG_JSON; else process.env.CONFIG_JSON = originalConfig; if (evidencePath) { const fixtureFiles = ['controlled-worker-service-transport.mjs', 'reversible-worker-inputs.mjs', - 'reversible-original-worker.mjs', 'reversible-original-attempts.mjs', 'reversible-worker-evidence.mjs'].map(path => { + 'reversible-original-worker.mjs', 'reversible-original-attempts.mjs', 'reversible-worker-evidence.mjs', + 'reversible-worker-evidence-check.mjs', 'reversible-worker-report-check.mjs'].map(path => { const bytes = readFileSync(new URL(path, import.meta.url)); return {path, bytes: bytes.length, sha256: hash(bytes)}; }); const snapshot = {schemaVersion: 1, mode, report: failure ? null : report, failure, fixtureFiles, diff --git a/tests/unit/reversible-original-attempts.test.ts b/tests/unit/reversible-original-attempts.test.ts index 8078f59c..e9258080 100644 --- a/tests/unit/reversible-original-attempts.test.ts +++ b/tests/unit/reversible-original-attempts.test.ts @@ -1,23 +1,38 @@ import {expect, test} from 'bun:test'; +import assert from 'node:assert/strict'; +import {mkdtempSync, rmSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join, resolve} from 'node:path'; import {fileURLToPath} from 'node:url'; for (const mode of ['baseline', 'revoke', 'late-rpc']) test('actual reversible native worker source survives ' + mode + ' control', async () => { - const child = Bun.spawn([process.env.HYPERION_NATIVE_NODE ?? 'node', - fileURLToPath(new URL('./reversible-original-attempts.mjs', import.meta.url)), mode], - {stdout: 'pipe', stderr: 'pipe', timeout: 30_000}); - const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); - expect({code, stderr, stdout: stdout.slice(-20000)}).toMatchObject({code: 0, stderr: ''}); - const result = JSON.parse(stdout.trim().split('\n').at(-1)!); - expect(result).toMatchObject({passed: true, mode, actualMasterCoordinator: true, originalSeparateNativeWorkers: true, - realReversibleReaderProbe: true, sourcePayloadUnchanged: true, constructedReversibleEnvelopesAndTransports: true, - capacityAuthority: false, releaseQualified: false}); - expect(result.processes).toHaveLength(4); - expect(result.processes.every((p: any) => p.exitCode === 0 && /^v(?:24|26)\./.test(p.node))).toBe(true); - if (mode === 'revoke') expect(result.revoked).toMatchObject({peerAssociatedBeforeOriginalSettlement: true, - conditionalWrites: 1, currentConfigurationAndTargetLateReadsRejected: true, - recoveredOriginalDelivery: {sameCapturedSource: true, associatedBeforeOriginalSettlement: true, - end: {binding: {blockNum: 33}, rootResult: 'returned', status: 'quiescent', acknowledgementAttempted: true}}, - oldEnd: {rootResult: 'threw', acknowledgementAttempted: false, status: 'quiescent'}}); - if (mode === 'late-rpc') expect(result.lateRpc).toMatchObject({oldCanonicalObservationRejected: true, - sourceStillHasSameCanonicalBlockIdentity: true, oldEndUnresolvedAfterReplacement: true}); + const directory = mkdtempSync(join(tmpdir(), 'hyperion-reversible-evidence-')), node = process.env.HYPERION_NATIVE_NODE ?? 'node'; + try { + const evidence = join(directory, mode + '.json'); + const child = Bun.spawn([node, fileURLToPath(new URL('./reversible-original-attempts.mjs', import.meta.url)), mode, evidence], + {stdout: 'pipe', stderr: 'pipe', timeout: 30_000}); + const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + expect({code, stderr, stdout: stdout.slice(-20000)}).toMatchObject({code: 0, stderr: ''}); + const result = JSON.parse(stdout.trim().split('\n').at(-1)!); + expect(result).toMatchObject({passed: true, mode, actualMasterCoordinator: true, originalSeparateNativeWorkers: true, + realReversibleReaderProbe: true, sourcePayloadUnchanged: true, constructedReversibleEnvelopesAndTransports: true, + capacityAuthority: false, releaseQualified: false}); + expect(result.processes).toHaveLength(4); + expect(result.processes.every((p: any) => p.exitCode === 0 && /^v(?:24|26)\./.test(p.node))).toBe(true); + if (mode === 'revoke') expect(result.revoked).toMatchObject({peerAssociatedBeforeOriginalSettlement: true, + conditionalWrites: 1, currentConfigurationAndTargetLateReadsRejected: true, + recoveredOriginalDelivery: {sameCapturedSource: true, associatedBeforeOriginalSettlement: true, + end: {binding: {blockNum: 33}, rootResult: 'returned', status: 'quiescent', acknowledgementAttempted: true}}, + oldEnd: {rootResult: 'threw', acknowledgementAttempted: false, status: 'quiescent'}}); + if (mode === 'late-rpc') expect(result.lateRpc).toMatchObject({oldCanonicalObservationRejected: true, + sourceStillHasSameCanonicalBlockIdentity: true, oldEndUnresolvedAfterReplacement: true}); + const check = Bun.spawn([node, fileURLToPath(new URL('./reversible-worker-report-check.mjs', import.meta.url)), evidence], + {stdout: 'pipe', stderr: 'pipe', timeout: 10_000}); + const [checked, output, errors] = await Promise.all([check.exited, new Response(check.stdout).text(), new Response(check.stderr).text()]); + expect({code: checked, stdout: output.slice(-12000), stderr: errors}).toMatchObject({code: 0, stderr: ''}); + expect(JSON.parse(output.trim())).toMatchObject({passed: true, mode, originalEvidencePassed: true, + originalStoreAndNativePartsUnchanged: true, originalProcessesRerun: false, coldWrites: 0, capacityAuthority: false, releaseQualified: false}); + } finally { + assert.equal(dirname(resolve(directory)), resolve(tmpdir())); rmSync(directory, {recursive: true, force: true}); + } }, 35_000); diff --git a/tests/unit/reversible-worker-evidence-check.mjs b/tests/unit/reversible-worker-evidence-check.mjs index e898ced8..b3e4bc83 100644 --- a/tests/unit/reversible-worker-evidence-check.mjs +++ b/tests/unit/reversible-worker-evidence-check.mjs @@ -86,14 +86,24 @@ export async function checkReversibleWorkerEvidence(path) { if (report.revoked) assert(recovered, 'Revocation evidence lacks recovery of its original delivery'); const associations = [report.associationId, ...(report.revoked ? [recovered.associationId, report.revoked.redelivery.associationId] : [])]; assert.equal(new Set(associations).size, associations.length); + const associationEnds = new Map(); for (const id of associations) { const observed = (await store.readAttemptAssociation(id)).observation; + associationEnds.set(id, observed.end); assert.equal(observed.sourceChecked, true); assert.equal(observed.brokerSettlement, 'unknown'); const originalEnd = acceptedEnds.find(e => e.incarnation === observed.end.incarnation && e.attemptId === observed.end.attemptId); assert(originalEnd, 'Retained association lacks its original native end reply'); assert.deepEqual(observed.end, originalEnd, 'Retained association end differs from its exact original native reply'); assert.equal(observed.results.length, 1); } + for (const result of [report, ...(report.revoked ? [recovered, report.revoked.redelivery] : [])]) + assert.deepEqual(result.end, associationEnds.get(result.associationId), 'Reported association end differs from its retained association'); + for (const end of [...(report.revoked ? [report.revoked.oldEnd] : []), ...(report.lateRpc ? [report.lateRpc.originalEnd] : [])]) { + assert(end && typeof end === 'object', 'Reported original end is absent'); + const original = acceptedEnds.find(e => e.incarnation === end.incarnation && e.attemptId === end.attemptId); + assert(original, 'Reported original end lacks its native reply'); + assert.deepEqual(end, original, 'Reported original end differs from its exact original native reply'); + } if (report.revoked) { assert(report.revoked.pendingAtFence >= 1); assert.equal(report.revoked.oldEnd.rootResult, 'threw'); assert.equal(report.revoked.oldEnd.acknowledgementAttempted, false); diff --git a/tests/unit/reversible-worker-report-check.mjs b/tests/unit/reversible-worker-report-check.mjs new file mode 100644 index 00000000..43ec07cf --- /dev/null +++ b/tests/unit/reversible-worker-report-check.mjs @@ -0,0 +1,36 @@ +/** Construct contradictory report copies while preserving original native/store evidence. */ +import assert from 'node:assert/strict'; +import {readFileSync, writeFileSync, mkdtempSync, rmSync} from 'node:fs'; +import {createHash} from 'node:crypto'; +import {tmpdir} from 'node:os'; +import {basename, dirname, join, resolve} from 'node:path'; +import {checkReversibleWorkerEvidence} from './reversible-worker-evidence-check.mjs'; + +const input = process.argv[2], original = await checkReversibleWorkerEvidence(input); +const envelope = JSON.parse(readFileSync(input)), report = envelope.data.report; +const hash = value => createHash('sha256').update(JSON.stringify(value)).digest('hex'); +const originalParts = Object.fromEntries(['stores', 'processes', 'ends', 'inputs'].map(key => [key, hash(envelope.data[key])])); +const copies = [['report', 'end'], ...(report.revoked ? [ + ['report', 'revoked', 'recoveredOriginalDelivery', 'end'], ['report', 'revoked', 'redelivery', 'end'], + ['report', 'revoked', 'oldEnd']] : []), ...(report.lateRpc ? [['report', 'lateRpc', 'originalEnd']] : [])]; +const directory = mkdtempSync(join(tmpdir(), 'hyperion-evidence-report-')); +try { + for (const [index, path] of copies.entries()) { + const end = path.reduce((value, key) => value[key], envelope.data), before = end.descriptor.payloadSha256; + assert.match(before, /^[0-9a-f]{64}$/); + assert(!envelope.references.some(ref => JSON.stringify(ref.path) === JSON.stringify([...path, 'descriptor', 'payloadSha256']))); + end.descriptor.payloadSha256 = before === '0'.repeat(64) ? '1'.repeat(64) : '0'.repeat(64); + for (const [key, digest] of Object.entries(originalParts)) assert.equal(hash(envelope.data[key]), digest); + const candidate = join(directory, index + '.json'); writeFileSync(candidate, JSON.stringify(envelope), {flag: 'wx'}); + await assert.rejects(checkReversibleWorkerEvidence(candidate), /Reported .*end differs/); + end.descriptor.payloadSha256 = before; + } + assert.equal((await checkReversibleWorkerEvidence(input)).passed, true); + console.log(JSON.stringify({passed: true, mode: envelope.data.mode, originalEvidencePassed: original.passed, + alteredReportCopiesRejected: copies.map(path => path.join('.')), originalStoreAndNativePartsUnchanged: true, + originalProcessesRerun: false, coldWrites: 0, capacityAuthority: false, releaseQualified: false})); +} finally { + assert.equal(dirname(resolve(directory)), resolve(tmpdir())); assert.match(basename(directory), /^hyperion-evidence-report-/); + rmSync(directory, {recursive: true, force: true}); +} + From 418137f46ebc96a28175c556cfdb84dddd43b4e7 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:36:38 -0300 Subject: [PATCH 3/3] test: replay publication gaps in ABI host controls Replay the requested canonical suffix before fresh irreversible confirmation. Preserve the original constructed observation fault during replay and keep publication progress independent of ABI coverage. --- tests/unit/protocol-abi-frontier-worker.mjs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/unit/protocol-abi-frontier-worker.mjs b/tests/unit/protocol-abi-frontier-worker.mjs index 1c547ade..560b953e 100644 --- a/tests/unit/protocol-abi-frontier-worker.mjs +++ b/tests/unit/protocol-abi-frontier-worker.mjs @@ -155,7 +155,8 @@ try { // Explicit incomplete contract observations advance topology with UNKNOWN ABI knowledge. const admittedProvider=a.worker.protocol;assert(Object.isFrozen(admittedProvider)); // Inject a fault into this worker's fixture context without mutating its admitted provider. - a.worker.protocol={...admittedProvider,observeBlockAbi(){throw new Error('Unsupported contract observation');}}; + const incompleteProvider={...admittedProvider,observeBlockAbi(){throw new Error('Unsupported contract observation');}}; + a.worker.protocol=incompleteProvider; try{ resumed.connection.message(frame(104,{prev_block:{block_num:103,block_id:replacementId}}));await resumed.drain();await a.consume(publications.at(-1)); }finally{a.worker.protocol=admittedProvider;} @@ -185,8 +186,18 @@ try { await until(async()=>(await frontier.readControl()).value.source===null); assert.equal(currentMaster.shutdownStarted,false,'Normal physical revocation became a master infrastructure failure'); currentSocket.message(Buffer.from(schemaJson));await resumed.drain();currentSocket.message(status());await resumed.drain(); - assert.equal(resumed.commands.at(-1)[1].start_block_num,106,'Coordinator-ahead resume reused the older Rabbit checkpoint'); + assert.equal(resumed.commands.at(-1)[1].start_block_num,103,'ABI coverage skipped unpublished reversible blocks'); assert.deepEqual(resumed.commands.at(-1)[1].have_positions,[]);assert.equal(currentMaster.liveReader.worker_last_processed_block,103); + // Re-deliver the requested canonical suffix; ABI coverage is not publication completion. + for(const number of [103,104,105]) { + const changes=number===103?{this_block:{block_num:103,block_id:replacementId}} + :number===104?{prev_block:{block_num:103,block_id:replacementId}}:{}; + // Reproduce this source's original constructed observation fault, not a different result under the same pin. + if(number===104)a.worker.protocol=incompleteProvider; + try{currentSocket.message(frame(number,changes));await resumed.drain();await a.consume(publications.at(-1));} + finally{a.worker.protocol=admittedProvider;} + assert.equal(currentMaster.liveReader.worker_last_processed_block,103,'Reversible replay fabricated an irreversible checkpoint'); + } resumed.worker.holdPublisherConfirms=true;const beforeFresh=publications.length; currentSocket.message(frame(106,{last_irreversible:{block_num:106,block_id:blockId(106)}}));await until(()=>publications.length===beforeFresh+1); assert.equal(currentMaster.liveReader.worker_last_processed_block,103,'Opening a source fabricated a broker checkpoint'); @@ -245,7 +256,7 @@ try { assert.equal(rotatedUrls,1);assert.equal(rotationTimers.length,1);assert.equal(resumed.commands.length,rotationCommands); assert.equal(publications.length,rotationPublications); rotated.message(Buffer.from(schemaJson));await resumed.drain();rotated.message(status());await resumed.drain(); - assert.equal(resumed.commands.at(-1)[1].start_block_num,108); + assert.equal(resumed.commands.at(-1)[1].start_block_num,107,'Rotation skipped a reversible publication gap'); assert.notEqual(resumed.worker.coordinatorSource.generation,rotationGeneration); assert.equal((await frontier.readControl()).value.source.readerSessionId,resumed.worker.sourceContext.readerSessionId); assert.equal(rotatedUrls,1);assert.equal(rotationTimers.length,1);