diff --git a/scripts/replay-escrow.mjs b/scripts/replay-escrow.mjs index 228c389..5d8044d 100644 --- a/scripts/replay-escrow.mjs +++ b/scripts/replay-escrow.mjs @@ -2,6 +2,7 @@ import { Networks } from "@stellar/stellar-sdk"; import { getDb } from "../src/lib/mongodb.js"; import { createJsonRpcEventSource } from "../src/lib/indexer/stellarIndexer.js"; import { applyEscrowEvent } from "../src/lib/indexer/escrowIndexer.js"; +import { reconcileEscrowOperations } from "../src/lib/escrow/escrowOperations.js"; const rpcUrl = process.env.NEXT_PUBLIC_STELLAR_RPC_URL; const networkPassphrase = @@ -18,37 +19,76 @@ if (!contractId) { throw new Error("TRUSTLESS_WORK_CONTRACT_ID_TESTNET or NEXT_PUBLIC_TRUSTLESS_WORK_CONTRACT_ID is required to run the escrow indexer"); } -const startLedger = parseInt(process.argv[2], 10); -const limit = parseInt(process.argv[3], 10) || 100; +const mode = process.argv[2] === "reconcile" ? "reconcile" : "replay"; +const startLedger = parseInt(mode === "reconcile" ? process.argv[3] : process.argv[2], 10); +const limit = parseInt(mode === "reconcile" ? process.argv[4] : process.argv[3], 10) || 100; +const actor = process.env.ESCROW_REPLAY_ACTOR || process.env.USER || "admin-cli"; if (!startLedger || isNaN(startLedger)) { console.error("Usage: node scripts/replay-escrow.mjs [limit]"); + console.error(" or: node scripts/replay-escrow.mjs reconcile [limit]"); process.exit(1); } const db = await getDb(); const eventSource = createJsonRpcEventSource({ rpcUrl, contractId, networkPassphrase }); -console.log(`Replaying escrow events from ledger ${startLedger} with limit ${limit}`); +function matchesOperation(event, operation) { + if (operation.transactionHash && (event.transactionHash === operation.transactionHash || event.txHash === operation.transactionHash)) { + return true; + } + if (operation.payload?.escrowId && event.escrowId === operation.payload.escrowId) { + return true; + } + return false; +} try { - const batch = await eventSource.getEvents({ startLedger, limit }); - const events = batch.events || []; - - console.log(`Found ${events.length} events to replay.`); - - let applied = 0; - let skipped = 0; - - for (const event of events) { - const result = await applyEscrowEvent(db, { ...event, source: "escrow" }); - if (result.skipped) skipped += 1; - else applied += 1; + if (mode === "reconcile") { + console.log(`Reconciling escrow operations from ledger ${startLedger} with event limit ${limit}`); + const batch = await eventSource.getEvents({ startLedger, limit }); + const events = batch.events || []; + const result = await reconcileEscrowOperations(db, { + actor, + limit, + queryChainState: async (operation) => { + const event = events.find((candidate) => matchesOperation(candidate, operation)); + if (!event) return { found: false, reason: "no matching escrow event found" }; + return { + found: true, + confirmed: true, + transactionHash: event.transactionHash || event.txHash || operation.transactionHash || null, + ledgerSequence: event.ledger ?? event.ledgerSequence ?? operation.ledgerSequence ?? null, + event, + }; + }, + project: async (operation) => { + const event = operation.onChainState?.event; + if (!event) throw new Error("missing on-chain event for escrow projection"); + await applyEscrowEvent(db, { ...event, source: "escrow" }); + }, + }); + console.log(JSON.stringify({ mode, actor, ...result }, null, 2)); + } else { + console.log(`Replaying escrow events from ledger ${startLedger} with limit ${limit}`); + const batch = await eventSource.getEvents({ startLedger, limit }); + const events = batch.events || []; + + console.log(`Found ${events.length} events to replay.`); + + let applied = 0; + let skipped = 0; + + for (const event of events) { + const result = await applyEscrowEvent(db, { ...event, source: "escrow" }); + if (result.skipped) skipped += 1; + else applied += 1; + } + + console.log(`Replay complete. Applied: ${applied}, Skipped: ${skipped}`); } - - console.log(`Replay complete. Applied: ${applied}, Skipped: ${skipped}`); } catch (err) { - console.error("Replay failed:", err); + console.error(`${mode === "reconcile" ? "Reconcile" : "Replay"} failed:`, err); } process.exit(0); diff --git a/src/app/api/escrow-operations/[idempotencyKey]/route.js b/src/app/api/escrow-operations/[idempotencyKey]/route.js new file mode 100644 index 0000000..d4f2409 --- /dev/null +++ b/src/app/api/escrow-operations/[idempotencyKey]/route.js @@ -0,0 +1,70 @@ +import { NextResponse } from "next/server"; + +import { getUserFromCookie } from "@/lib/api/auth"; +import { withApiHardening } from "@/lib/api/hardening"; +import { COLLECTIONS } from "@/lib/backend/schemaContracts"; +import { getDb } from "@/lib/mongodb"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function publicOperation(operation) { + return { + idempotencyKey: operation.idempotencyKey, + operationType: operation.operationType, + state: operation.state, + stage: operation.stage || null, + transactionHash: operation.transactionHash || null, + ledgerSequence: operation.ledgerSequence ?? null, + retryCount: operation.retryCount || 0, + reconciliationFailureCount: operation.reconciliationFailureCount || 0, + terminal: operation.terminal === true, + nextAttemptAt: operation.nextAttemptAt || null, + createdAt: operation.createdAt, + updatedAt: operation.updatedAt, + }; +} + +export async function GET(request, { params }) { + return withApiHardening( + request, + { route: "escrow-operation-status", rateLimit: { limit: 60, windowMs: 60_000 } }, + async () => { + const user = await getUserFromCookie(request); + if (!user) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { idempotencyKey } = await params; + if (!idempotencyKey) { + return NextResponse.json({ error: "Missing idempotency key" }, { status: 400 }); + } + + const db = await getDb(); + const operation = await db + .collection(COLLECTIONS.escrowOperations) + .findOne({ idempotencyKey: decodeURIComponent(idempotencyKey) }); + + if (!operation) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const userWallets = [ + user.walletAddress, + user.walletAddressLower, + user.payoutWalletAddress, + user.payoutWalletAddressLower, + user.address, + user.id, + ] + .filter(Boolean) + .map((value) => String(value).toLowerCase()); + const actor = operation.actor ? String(operation.actor).toLowerCase() : null; + if (actor && !userWallets.includes(actor) && user.role !== "admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + return NextResponse.json(publicOperation(operation)); + }, + ); +} diff --git a/src/hooks/useEscrow.js b/src/hooks/useEscrow.js index 892e9ea..9ac1922 100644 --- a/src/hooks/useEscrow.js +++ b/src/hooks/useEscrow.js @@ -27,6 +27,31 @@ async function fetchEscrowMilestones(escrowId) { return res.json(); } +async function fetchEscrowOperation(idempotencyKey) { + if (!idempotencyKey) return null; + const res = await fetch(`/api/escrow-operations/${idempotencyKey}`); + if (!res.ok) { + if (res.status === 404) return null; + throw new Error("Failed to fetch escrow operation"); + } + return res.json(); +} + +export function getEscrowOperationStatus(operation) { + if (!operation) return { state: "unknown", recoverable: false, pending: false, failed: false }; + const failed = operation.state === "failed"; + const pending = ["pending", "submitted", "reconciling"].includes(operation.state); + return { + state: operation.state, + stage: operation.stage || null, + pending, + failed, + recoverable: failed && operation.terminal !== true, + retryCount: operation.retryCount || 0, + reconciliationFailureCount: operation.reconciliationFailureCount || 0, + }; +} + export function useEscrow(escrowId, { enabled = true, refetchInterval = false } = {}) { return useQuery({ queryKey: ["escrow", escrowId], @@ -36,6 +61,19 @@ export function useEscrow(escrowId, { enabled = true, refetchInterval = false } }); } +export function useEscrowOperation(idempotencyKey, { enabled = true, refetchInterval = false } = {}) { + return useQuery({ + queryKey: ["escrow-operation", idempotencyKey], + queryFn: () => fetchEscrowOperation(idempotencyKey), + enabled: enabled && !!idempotencyKey, + refetchInterval, + select: (operation) => ({ + operation, + status: getEscrowOperationStatus(operation), + }), + }); +} + export function useUserEscrows(walletAddress, { enabled = true } = {}) { return useQuery({ queryKey: ["escrows", "user", walletAddress], diff --git a/src/lib/backend/schemaContracts.js b/src/lib/backend/schemaContracts.js index ded5508..25cce66 100644 --- a/src/lib/backend/schemaContracts.js +++ b/src/lib/backend/schemaContracts.js @@ -33,6 +33,13 @@ export const COLLECTIONS = Object.freeze({ files: "files", fileCleanupOutbox: "file_cleanup_outbox", uploadQuarantine: "upload_quarantine", + + // Escrow / Trustless Work. + escrows: "escrows", + milestones: "milestones", + payouts: "payouts", + escrowOperations: "escrow_operations", + escrowOperationAudit: "escrow_operation_audit", }); // File lifecycle states (#98). A file object moves: @@ -62,10 +69,6 @@ export const FILE_PURPOSES = Object.freeze({ MILESTONE_EVIDENCE: { purpose: "milestone_evidence", visibility: FILE_VISIBILITY.PRIVATE, maxBytes: 25 * 1024 * 1024 }, PAYOUT_DOCUMENT: { purpose: "payout_document", visibility: FILE_VISIBILITY.PRIVATE, maxBytes: 10 * 1024 * 1024 }, FEEDBACK_ATTACHMENT: { purpose: "feedback_attachment", visibility: FILE_VISIBILITY.PRIVATE, maxBytes: 10 * 1024 * 1024 }, - // Escrow / Trustless Work. - escrows: "escrows", - milestones: "milestones", - payouts: "payouts", }); export const REQUIRED_INDEXES = Object.freeze({ @@ -574,6 +577,10 @@ export const REQUIRED_INDEXES = Object.freeze({ { name: "quarantine_status_lease", keys: { status: 1, leaseUntil: 1 }, + options: {}, + }, + ], + escrows: [ { name: "escrows_escrow_id_unique", @@ -625,6 +632,39 @@ export const REQUIRED_INDEXES = Object.freeze({ options: {}, }, ], + + escrow_operations: [ + { + name: "escrow_operations_idempotency_key_unique", + keys: { idempotencyKey: 1 }, + options: { unique: true }, + }, + { + name: "escrow_operations_state_next_attempt", + keys: { state: 1, nextAttemptAt: 1 }, + options: {}, + }, + { + name: "escrow_operations_transaction_hash", + keys: { transactionHash: 1 }, + options: { + partialFilterExpression: { transactionHash: { $type: "string" } }, + }, + }, + ], + + escrow_operation_audit: [ + { + name: "escrow_operation_audit_operation_created_at", + keys: { operationId: 1, createdAt: -1 }, + options: {}, + }, + { + name: "escrow_operation_audit_actor_created_at", + keys: { actor: 1, createdAt: -1 }, + options: {}, + }, + ], }); // ── Material field contracts ─────────────────────────────────────────────── @@ -1073,6 +1113,51 @@ export const COLLECTION_VALIDATORS = Object.freeze({ }, }, }, + + escrow_operations: { + $jsonSchema: { + bsonType: "object", + required: [ + "idempotencyKey", + "operationType", + "payloadHash", + "actor", + "state", + "createdAt", + "updatedAt", + ], + properties: { + idempotencyKey: { bsonType: "string", minLength: 1 }, + operationType: { bsonType: "string", minLength: 1 }, + payloadHash: { bsonType: "string", minLength: 64 }, + actor: { bsonType: ["string", "null"] }, + state: { enum: ["pending", "submitted", "confirmed", "failed", "reconciling"] }, + transactionHash: { bsonType: ["string", "null"] }, + ledgerSequence: { bsonType: ["int", "long", "null"] }, + retryCount: { bsonType: ["int", "long"] }, + reconciliationFailureCount: { bsonType: ["int", "long"] }, + terminal: { bsonType: "bool" }, + nextAttemptAt: { bsonType: ["date", "null"] }, + createdAt: { bsonType: "date" }, + updatedAt: { bsonType: "date" }, + }, + }, + }, + + escrow_operation_audit: { + $jsonSchema: { + bsonType: "object", + required: ["operationId", "action", "actor", "createdAt"], + properties: { + operationId: { bsonType: "string", minLength: 1 }, + action: { bsonType: "string", minLength: 1 }, + actor: { bsonType: ["string", "null"] }, + before: { bsonType: ["object", "null"] }, + after: { bsonType: ["object", "null"] }, + createdAt: { bsonType: "date" }, + }, + }, + }, }); export const EDITABLE_MATERIAL_FIELDS = Object.freeze([ diff --git a/src/lib/escrow/escrowOperations.js b/src/lib/escrow/escrowOperations.js new file mode 100644 index 0000000..efd54b3 --- /dev/null +++ b/src/lib/escrow/escrowOperations.js @@ -0,0 +1,435 @@ +import { createHash } from "node:crypto"; + +import { COLLECTIONS } from "../backend/schemaContracts.js"; +import { incrementCounter, setGauge } from "../telemetry/metrics.js"; + +export const ESCROW_OPERATION_STATE = Object.freeze({ + PENDING: "pending", + SUBMITTED: "submitted", + CONFIRMED: "confirmed", + FAILED: "failed", + RECONCILING: "reconciling", +}); + +export const ESCROW_OPERATION_STAGE = Object.freeze({ + SUBMISSION: "submission", + CONFIRMATION: "confirmation", + PROJECTION: "projection", + DONE: "done", +}); + +export const DEFAULT_ESCROW_OPERATION_RETRY_POLICY = Object.freeze({ + maxRetries: 5, + baseDelayMs: 2_000, + maxDelayMs: 60_000, +}); + +function duplicateKey(error) { + return error?.code === 11000; +} + +function ordered(value) { + if (!value || typeof value !== "object" || value instanceof Date) return value; + if (Array.isArray(value)) return value.map(ordered); + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, ordered(value[key])])); +} + +export function hashEscrowPayload(payload) { + return createHash("sha256").update(JSON.stringify(ordered(payload ?? {}))).digest("hex"); +} + +export function calculateEscrowBackoffMs( + retryCount, + { + baseDelayMs = DEFAULT_ESCROW_OPERATION_RETRY_POLICY.baseDelayMs, + maxDelayMs = DEFAULT_ESCROW_OPERATION_RETRY_POLICY.maxDelayMs, + } = {}, +) { + return Math.min(maxDelayMs, baseDelayMs * 2 ** Math.max(0, retryCount - 1)); +} + +async function insertAudit(db, { operationId, action, actor, before = null, after = null, now = new Date() }) { + await db.collection(COLLECTIONS.escrowOperationAudit).insertOne({ + _id: `${operationId}:${action}:${now.getTime()}:${Math.random().toString(16).slice(2)}`, + operationId, + action, + actor: actor ?? null, + before, + after, + createdAt: now, + }); +} + +function summarizeForMetrics(operation, now) { + const ageMs = operation?.createdAt instanceof Date ? now.getTime() - operation.createdAt.getTime() : 0; + setGauge("escrow_operation_age_ms", { state: operation?.state || "unknown" }, Math.max(0, ageMs)); + setGauge("escrow_operation_retry_count", { state: operation?.state || "unknown" }, operation?.retryCount || 0); + setGauge( + "escrow_operation_reconciliation_failure_count", + { state: operation?.state || "unknown" }, + operation?.reconciliationFailureCount || 0, + ); +} + +export async function createEscrowOperation( + db, + { idempotencyKey, operationType, payload, actor = null, now = new Date() }, +) { + if (!idempotencyKey) throw new Error("idempotencyKey is required"); + if (!operationType) throw new Error("operationType is required"); + + const payloadHash = hashEscrowPayload(payload); + const operation = { + _id: idempotencyKey, + idempotencyKey, + operationType, + payloadHash, + payload, + actor, + state: ESCROW_OPERATION_STATE.PENDING, + stage: ESCROW_OPERATION_STAGE.SUBMISSION, + transactionHash: null, + ledgerSequence: null, + retryCount: 0, + reconciliationFailureCount: 0, + terminal: false, + nextAttemptAt: now, + createdAt: now, + updatedAt: now, + }; + + try { + await db.collection(COLLECTIONS.escrowOperations).insertOne(operation); + await insertAudit(db, { + operationId: idempotencyKey, + action: "created", + actor, + after: operation, + now, + }); + incrementCounter("escrow_operations_created_total", { operationType }); + summarizeForMetrics(operation, now); + return { operation, created: true }; + } catch (error) { + if (!duplicateKey(error)) throw error; + } + + const existing = await db.collection(COLLECTIONS.escrowOperations).findOne({ idempotencyKey }); + if (!existing) throw new Error("idempotency key exists but operation record could not be loaded"); + if (existing.operationType !== operationType || existing.payloadHash !== payloadHash) { + throw new Error("idempotency key was already used for a different escrow operation"); + } + summarizeForMetrics(existing, now); + return { operation: existing, created: false }; +} + +async function loadOperation(db, idempotencyKey) { + const operation = await db.collection(COLLECTIONS.escrowOperations).findOne({ idempotencyKey }); + if (!operation) throw new Error(`escrow operation not found: ${idempotencyKey}`); + return operation; +} + +async function markFailure(db, operation, { stage, error, actor, retryPolicy, now, reconciliation = false }) { + const retryCount = (operation.retryCount || 0) + 1; + const terminal = retryCount >= retryPolicy.maxRetries; + const state = terminal ? ESCROW_OPERATION_STATE.FAILED : operation.state; + const backoffMs = calculateEscrowBackoffMs(retryCount, retryPolicy); + const nextAttemptAt = terminal ? null : new Date(now.getTime() + backoffMs); + const before = { ...operation }; + const patch = { + state, + stage, + retryCount, + terminal, + nextAttemptAt, + lastError: String(error?.message || error), + updatedAt: now, + }; + if (reconciliation) { + patch.reconciliationFailureCount = (operation.reconciliationFailureCount || 0) + 1; + } + + await db.collection(COLLECTIONS.escrowOperations).updateOne( + { idempotencyKey: operation.idempotencyKey }, + { $set: patch }, + ); + const after = { ...operation, ...patch }; + await insertAudit(db, { + operationId: operation.idempotencyKey, + action: terminal ? "failed_terminal" : "retry_scheduled", + actor, + before, + after, + now, + }); + incrementCounter("escrow_operation_stage_failures_total", { + stage, + terminal: String(terminal), + }); + summarizeForMetrics(after, now); + return after; +} + +async function persistTransition(db, operation, patch, { action, actor, now }) { + const before = { ...operation }; + const after = { ...operation, ...patch, updatedAt: now }; + await db.collection(COLLECTIONS.escrowOperations).updateOne( + { idempotencyKey: operation.idempotencyKey }, + { $set: { ...patch, updatedAt: now } }, + ); + await insertAudit(db, { + operationId: operation.idempotencyKey, + action, + actor, + before, + after, + now, + }); + summarizeForMetrics(after, now); + return after; +} + +export async function processEscrowOperation( + db, + idempotencyKey, + { + submit, + confirm, + project, + actor = "system", + now = new Date(), + retryPolicy = DEFAULT_ESCROW_OPERATION_RETRY_POLICY, + } = {}, +) { + let operation = await loadOperation(db, idempotencyKey); + if (operation.terminal || operation.stage === ESCROW_OPERATION_STAGE.DONE) { + summarizeForMetrics(operation, now); + return { operation, attemptedSubmission: false }; + } + if (operation.nextAttemptAt && operation.nextAttemptAt > now) { + summarizeForMetrics(operation, now); + return { operation, deferred: true, attemptedSubmission: false }; + } + + let attemptedSubmission = false; + + if (operation.stage === ESCROW_OPERATION_STAGE.SUBMISSION || operation.state === ESCROW_OPERATION_STATE.PENDING) { + if (typeof submit !== "function") throw new Error("submit handler is required"); + try { + attemptedSubmission = true; + const submitted = await submit(operation); + operation = await persistTransition( + db, + operation, + { + state: ESCROW_OPERATION_STATE.SUBMITTED, + stage: ESCROW_OPERATION_STAGE.CONFIRMATION, + transactionHash: submitted?.transactionHash || submitted?.txHash || operation.transactionHash || null, + ledgerSequence: submitted?.ledgerSequence ?? submitted?.ledger ?? operation.ledgerSequence ?? null, + submittedAt: now, + nextAttemptAt: now, + lastError: null, + }, + { action: "submitted", actor, now }, + ); + } catch (error) { + return { + operation: await markFailure(db, operation, { + stage: ESCROW_OPERATION_STAGE.SUBMISSION, + error, + actor, + retryPolicy, + now, + }), + attemptedSubmission, + }; + } + } + + if (operation.stage === ESCROW_OPERATION_STAGE.CONFIRMATION || operation.state === ESCROW_OPERATION_STATE.SUBMITTED) { + if (typeof confirm !== "function") throw new Error("confirm handler is required"); + try { + const confirmation = await confirm(operation); + if (confirmation?.confirmed === false) { + throw new Error(confirmation.reason || "escrow operation has not confirmed"); + } + operation = await persistTransition( + db, + operation, + { + state: ESCROW_OPERATION_STATE.CONFIRMED, + stage: ESCROW_OPERATION_STAGE.PROJECTION, + transactionHash: confirmation?.transactionHash || confirmation?.txHash || operation.transactionHash || null, + ledgerSequence: confirmation?.ledgerSequence ?? confirmation?.ledger ?? operation.ledgerSequence ?? null, + onChainState: confirmation?.onChainState || confirmation || null, + confirmedAt: now, + nextAttemptAt: now, + lastError: null, + }, + { action: "confirmed", actor, now }, + ); + } catch (error) { + return { + operation: await markFailure(db, operation, { + stage: ESCROW_OPERATION_STAGE.CONFIRMATION, + error, + actor, + retryPolicy, + now, + }), + attemptedSubmission, + }; + } + } + + if (operation.stage === ESCROW_OPERATION_STAGE.PROJECTION || operation.state === ESCROW_OPERATION_STATE.CONFIRMED) { + if (typeof project !== "function") throw new Error("project handler is required"); + try { + await project(operation); + operation = await persistTransition( + db, + operation, + { + state: ESCROW_OPERATION_STATE.CONFIRMED, + stage: ESCROW_OPERATION_STAGE.DONE, + projectedAt: now, + terminal: false, + nextAttemptAt: null, + lastError: null, + }, + { action: "projected", actor, now }, + ); + incrementCounter("escrow_operations_completed_total", { operationType: operation.operationType }); + } catch (error) { + return { + operation: await markFailure(db, operation, { + stage: ESCROW_OPERATION_STAGE.PROJECTION, + error, + actor, + retryPolicy, + now, + }), + attemptedSubmission, + }; + } + } + + return { operation, attemptedSubmission }; +} + +export async function executeEscrowCommand( + db, + command, + handlers, + options = {}, +) { + const { operation, created } = await createEscrowOperation(db, { + idempotencyKey: command.idempotencyKey, + operationType: command.operationType, + payload: command.payload, + actor: command.actor ?? null, + now: options.now || new Date(), + }); + + const result = await processEscrowOperation(db, operation.idempotencyKey, { + ...handlers, + actor: command.actor ?? options.actor ?? "system", + now: options.now || new Date(), + retryPolicy: options.retryPolicy || DEFAULT_ESCROW_OPERATION_RETRY_POLICY, + }); + + return { ...result, created }; +} + +async function collectDueOperations(db, { now, limit }) { + const filter = { + state: { + $in: [ + ESCROW_OPERATION_STATE.PENDING, + ESCROW_OPERATION_STATE.SUBMITTED, + ESCROW_OPERATION_STATE.RECONCILING, + ], + }, + terminal: { $ne: true }, + $or: [{ nextAttemptAt: null }, { nextAttemptAt: { $lte: now } }, { nextAttemptAt: { $exists: false } }], + }; + const cursor = db.collection(COLLECTIONS.escrowOperations).find(filter).limit(limit); + const operations = []; + for await (const operation of cursor) operations.push(operation); + return operations; +} + +export async function reconcileEscrowOperations( + db, + { + queryChainState, + project, + actor = "system", + now = new Date(), + limit = 100, + retryPolicy = DEFAULT_ESCROW_OPERATION_RETRY_POLICY, + } = {}, +) { + if (typeof queryChainState !== "function") throw new Error("queryChainState handler is required"); + + const operations = await collectDueOperations(db, { now, limit }); + const reconciled = []; + const failed = []; + + for (const candidate of operations) { + let operation = await persistTransition( + db, + candidate, + { state: ESCROW_OPERATION_STATE.RECONCILING, previousState: candidate.state, nextAttemptAt: now }, + { action: "reconcile_started", actor, now }, + ); + + try { + const chain = await queryChainState(operation); + if (!chain || chain.found === false || chain.confirmed === false) { + throw new Error(chain?.reason || "escrow operation not found confirmed on-chain"); + } + + operation = await persistTransition( + db, + operation, + { + state: ESCROW_OPERATION_STATE.CONFIRMED, + stage: ESCROW_OPERATION_STAGE.PROJECTION, + transactionHash: chain.transactionHash || chain.txHash || operation.transactionHash || null, + ledgerSequence: chain.ledgerSequence ?? chain.ledger ?? operation.ledgerSequence ?? null, + onChainState: chain, + confirmedAt: operation.confirmedAt || now, + nextAttemptAt: now, + lastError: null, + }, + { action: "reconciled_from_chain", actor, now }, + ); + + const projected = await processEscrowOperation(db, operation.idempotencyKey, { + submit: async () => { + throw new Error("submission is not allowed during projection-only reconciliation"); + }, + confirm: async () => chain, + project, + actor, + now, + retryPolicy, + }); + reconciled.push({ idempotencyKey: operation.idempotencyKey, state: projected.operation.state }); + } catch (error) { + const next = await markFailure(db, operation, { + stage: operation.stage || ESCROW_OPERATION_STAGE.CONFIRMATION, + error, + actor, + retryPolicy, + now, + reconciliation: true, + }); + failed.push({ idempotencyKey: next.idempotencyKey, state: next.state, error: next.lastError }); + } + } + + incrementCounter("escrow_operation_reconcile_runs_total", { outcome: "completed" }); + return { scanned: operations.length, reconciled, failed }; +} diff --git a/src/lib/indexer/escrowIndexer.js b/src/lib/indexer/escrowIndexer.js index 3d4d1a5..c7defe7 100644 --- a/src/lib/indexer/escrowIndexer.js +++ b/src/lib/indexer/escrowIndexer.js @@ -1,7 +1,7 @@ import { COLLECTIONS } from "../backend/schemaContracts.js"; import { incrementCounter } from "../telemetry/metrics.js"; import { logger } from "../logger.js"; -import { eventId, deadLetterId } from "./stellarIndexer.js"; +import { eventId } from "./eventIdentity.js"; function duplicateKey(error) { return error?.code === 11000; diff --git a/src/lib/indexer/eventIdentity.js b/src/lib/indexer/eventIdentity.js new file mode 100644 index 0000000..13bc555 --- /dev/null +++ b/src/lib/indexer/eventIdentity.js @@ -0,0 +1,30 @@ +import { createHash } from "node:crypto"; + +export function eventId(event) { + if (event.id || event.eventId) return String(event.id || event.eventId); + + const identity = [ + event.network || event.source || "stellar", + event.contractId || event.contract || "unknown-contract", + event.ledger ?? event.ledgerSequence, + event.transactionHash || event.txHash, + event.index ?? event.eventIndex ?? event.position, + ]; + + return identity.some((part) => part === undefined || part === null || part === "") + ? "" + : identity.map(String).join(":"); +} + +export function deadLetterId(event, source = "stellar") { + const identity = eventId(event); + if (identity) return identity; + + let serialized; + try { + serialized = JSON.stringify(event); + } catch { + serialized = String(event); + } + return `${source}:unidentified:${createHash("sha256").update(serialized).digest("hex").slice(0, 32)}`; +} diff --git a/src/lib/indexer/stellarIndexer.js b/src/lib/indexer/stellarIndexer.js index 4b44e3a..bc360cc 100644 --- a/src/lib/indexer/stellarIndexer.js +++ b/src/lib/indexer/stellarIndexer.js @@ -1,5 +1,3 @@ -import { createHash } from "node:crypto"; - import { COLLECTIONS } from "../backend/schemaContracts.js"; import { incrementCounter, setGauge } from "../telemetry/metrics.js"; import { logger } from "../logger.js"; @@ -7,6 +5,7 @@ import { auditLog } from "../api/audit.js"; import { runWithContext } from "../telemetry/context.js"; import { withSpan } from "../telemetry/tracing.js"; import { decodeContractEvent } from "./eventDecoder.js"; +import { deadLetterId, eventId } from "./eventIdentity.js"; function duplicateKey(error) { return error?.code === 11000; @@ -29,34 +28,7 @@ let transactionSupport = "unknown"; * stable identity (malformed payloads) still need a deterministic key, so we * hash the payload instead of generating a random one — a random key would * write a fresh row per attempt and never reach the retry ceiling. */ -export function deadLetterId(event, source = "stellar") { - const identity = eventId(event); - if (identity) return identity; - - let serialized; - try { - serialized = JSON.stringify(event); - } catch { - serialized = String(event); - } - return `${source}:unidentified:${createHash("sha256").update(serialized).digest("hex").slice(0, 32)}`; -} - -export function eventId(event) { - if (event.id || event.eventId) return String(event.id || event.eventId); - - const identity = [ - event.network || event.source || "stellar", - event.contractId || event.contract || "unknown-contract", - event.ledger ?? event.ledgerSequence, - event.transactionHash || event.txHash, - event.index ?? event.eventIndex ?? event.position, - ]; - - return identity.some((part) => part === undefined || part === null || part === "") - ? "" - : identity.map(String).join(":"); -} +export { deadLetterId, eventId }; function writeOptions(session, extra = {}) { return session ? { ...extra, session } : extra; diff --git a/tests/backend/escrowOperations.test.mjs b/tests/backend/escrowOperations.test.mjs new file mode 100644 index 0000000..4290ba2 --- /dev/null +++ b/tests/backend/escrowOperations.test.mjs @@ -0,0 +1,265 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { COLLECTIONS } from "../../src/lib/backend/schemaContracts.js"; +import { + calculateEscrowBackoffMs, + createEscrowOperation, + executeEscrowCommand, + processEscrowOperation, + reconcileEscrowOperations, +} from "../../src/lib/escrow/escrowOperations.js"; +import { createFakeDb } from "./helpers/fakeMongo.mjs"; + +const retryPolicy = Object.freeze({ maxRetries: 3, baseDelayMs: 1_000, maxDelayMs: 5_000 }); + +function command(idempotencyKey = "op-1") { + return { + idempotencyKey, + operationType: "release", + actor: "GACTOR", + payload: { escrowId: "escrow-1", recipient: "GRECIPIENT", amount: "100" }, + }; +} + +test("idempotency key creates one operation and one submission attempt", async () => { + const db = createFakeDb(); + let submissions = 0; + const handlers = { + async submit() { + submissions += 1; + return { transactionHash: "tx-1", ledgerSequence: 100 }; + }, + async confirm() { + return { confirmed: false, reason: "not indexed yet" }; + }, + async project() {}, + }; + + const first = await executeEscrowCommand(db, command(), handlers, { + now: new Date("2026-01-01T00:00:00Z"), + retryPolicy, + }); + const second = await executeEscrowCommand(db, command(), handlers, { + now: new Date("2026-01-01T00:00:00Z"), + retryPolicy, + }); + + assert.equal(first.created, true); + assert.equal(second.created, false); + assert.equal(submissions, 1); + assert.equal(db.dump(COLLECTIONS.escrowOperations).length, 1); + assert.equal(db.dump(COLLECTIONS.escrowOperations)[0].transactionHash, "tx-1"); +}); + +test("same idempotency key with different payload is rejected", async () => { + const db = createFakeDb(); + await createEscrowOperation(db, { + ...command("same-key"), + now: new Date("2026-01-01T00:00:00Z"), + }); + + await assert.rejects( + () => + createEscrowOperation(db, { + ...command("same-key"), + payload: { escrowId: "other" }, + now: new Date("2026-01-01T00:00:00Z"), + }), + /different escrow operation/, + ); +}); + +test("state machine reaches pending to submitted to confirmed", async () => { + const db = createFakeDb(); + const result = await executeEscrowCommand( + db, + command("happy-path"), + { + async submit() { + return { transactionHash: "tx-happy", ledgerSequence: 200 }; + }, + async confirm() { + return { confirmed: true, transactionHash: "tx-happy", ledgerSequence: 201 }; + }, + async project(operation) { + await db.collection(COLLECTIONS.payouts).updateOne( + { payoutId: `${operation.payload.escrowId}-${operation.payload.recipient}` }, + { + $set: { + payoutId: `${operation.payload.escrowId}-${operation.payload.recipient}`, + escrowId: operation.payload.escrowId, + recipient: operation.payload.recipient, + amount: operation.payload.amount, + status: "claimed", + chainTxHash: operation.transactionHash, + createdAt: new Date("2026-01-01T00:00:00Z"), + updatedAt: new Date("2026-01-01T00:00:00Z"), + }, + }, + { upsert: true }, + ); + }, + }, + { now: new Date("2026-01-01T00:00:00Z"), retryPolicy }, + ); + + assert.equal(result.operation.state, "confirmed"); + assert.equal(result.operation.stage, "done"); + assert.equal(result.operation.transactionHash, "tx-happy"); + assert.equal(db.dump(COLLECTIONS.payouts)[0].status, "claimed"); +}); + +test("confirmation failure reaches terminal failed instead of retrying forever", async () => { + const db = createFakeDb(); + await executeEscrowCommand( + db, + command("timeout-path"), + { + async submit() { + return { transactionHash: "tx-timeout" }; + }, + async confirm() { + return { confirmed: false, reason: "timeout" }; + }, + async project() {}, + }, + { now: new Date("2026-01-01T00:00:00Z"), retryPolicy }, + ); + + for (const now of [ + new Date("2026-01-01T00:00:02Z"), + new Date("2026-01-01T00:00:05Z"), + ]) { + await processEscrowOperation(db, "timeout-path", { + confirm: async () => ({ confirmed: false, reason: "timeout" }), + project: async () => {}, + now, + retryPolicy, + }); + } + + const operation = db.dump(COLLECTIONS.escrowOperations)[0]; + assert.equal(operation.state, "failed"); + assert.equal(operation.terminal, true); + assert.equal(operation.retryCount, 3); +}); + +test("retry backoff is bounded exponential", () => { + assert.equal(calculateEscrowBackoffMs(1, retryPolicy), 1_000); + assert.equal(calculateEscrowBackoffMs(2, retryPolicy), 2_000); + assert.equal(calculateEscrowBackoffMs(4, retryPolicy), 5_000); +}); + +test("timeout after submit is resolved by reconciliation from chain state", async () => { + const db = createFakeDb(); + await executeEscrowCommand( + db, + command("reconcile-success"), + { + async submit() { + return { transactionHash: "tx-reconcile" }; + }, + async confirm() { + return { confirmed: false, reason: "client timed out" }; + }, + async project() {}, + }, + { now: new Date("2026-01-01T00:00:00Z"), retryPolicy }, + ); + + const result = await reconcileEscrowOperations(db, { + now: new Date("2026-01-01T00:00:02Z"), + retryPolicy, + queryChainState: async () => ({ found: true, confirmed: true, transactionHash: "tx-reconcile", ledgerSequence: 300 }), + project: async (operation) => { + await db.collection(COLLECTIONS.escrows).updateOne( + { escrowId: operation.payload.escrowId }, + { + $set: { + escrowId: operation.payload.escrowId, + status: "released", + chainTxHash: operation.transactionHash, + createdAt: new Date("2026-01-01T00:00:02Z"), + updatedAt: new Date("2026-01-01T00:00:02Z"), + }, + }, + { upsert: true }, + ); + }, + }); + + assert.equal(result.reconciled.length, 1); + assert.equal(db.dump(COLLECTIONS.escrowOperations)[0].state, "confirmed"); + assert.equal(db.dump(COLLECTIONS.escrows)[0].chainTxHash, "tx-reconcile"); +}); + +test("confirmation never found reaches terminal failed through reconciliation", async () => { + const db = createFakeDb(); + await executeEscrowCommand( + db, + command("reconcile-failure"), + { + async submit() { + return { transactionHash: "tx-missing" }; + }, + async confirm() { + return { confirmed: false, reason: "missing" }; + }, + async project() {}, + }, + { now: new Date("2026-01-01T00:00:00Z"), retryPolicy }, + ); + + for (const now of [ + new Date("2026-01-01T00:00:02Z"), + new Date("2026-01-01T00:00:05Z"), + ]) { + await reconcileEscrowOperations(db, { + now, + retryPolicy, + queryChainState: async () => ({ found: false, reason: "not found" }), + project: async () => {}, + }); + } + + const operation = db.dump(COLLECTIONS.escrowOperations)[0]; + assert.equal(operation.state, "failed"); + assert.equal(operation.terminal, true); + assert.equal(operation.reconciliationFailureCount, 2); +}); + +test("crash recovery resumes after submission and completes without another submit", async () => { + const db = createFakeDb(); + await createEscrowOperation(db, { + ...command("crash-recovery"), + now: new Date("2026-01-01T00:00:00Z"), + }); + await db.collection(COLLECTIONS.escrowOperations).updateOne( + { idempotencyKey: "crash-recovery" }, + { + $set: { + state: "submitted", + stage: "confirmation", + transactionHash: "tx-before-crash", + nextAttemptAt: new Date("2026-01-01T00:00:00Z"), + }, + }, + ); + + let submissions = 0; + const result = await processEscrowOperation(db, "crash-recovery", { + submit: async () => { + submissions += 1; + return { transactionHash: "should-not-submit" }; + }, + confirm: async () => ({ confirmed: true, transactionHash: "tx-before-crash", ledgerSequence: 400 }), + project: async () => {}, + now: new Date("2026-01-01T00:00:01Z"), + retryPolicy, + }); + + assert.equal(submissions, 0); + assert.equal(result.operation.state, "confirmed"); + assert.equal(result.operation.transactionHash, "tx-before-crash"); +});