fix(transactions): prevent TTL index from deleting confirmed purchases and donations (#94)

docs/transaction-lifecycle.md

@@ -0,0 +1,86 @@
+# Transaction Expiry Lifecycle and TTL Invariant
+
+This document describes the `expiresAt` / TTL behavior of the `Transaction`
+collection and the invariant that protects confirmed purchases and donations
+from being deleted.
+
+## The problem this invariant solves
+
+`Transaction` rows are created when a buyer starts a checkout. To garbage-collect
+abandoned checkouts, the collection used a blanket TTL index on `expiresAt`
+(`expireAfterSeconds: 0`) with a schema default of `now + 30 minutes` applied to
+**every** row — including rows that later became `confirmed`. A confirmed on-chain
+purchase or donation was therefore permanently deleted ~30 minutes after it was
+created: the proof that the buyer paid and the educator earned silently vanished.
+
+## The invariant
+
+> **`expiresAt` is only ever set on `pending` transactions. Every terminal state
+> MUST clear it, the schema enforces this on save, and the TTL index is scoped
+> strictly to `status: "pending"` so the reaper cannot match anything else.**
+
+### Status → `expiresAt` mapping
+
+| Status      | `expiresAt`      | Rationale                                                          |
+|-------------|------------------|--------------------------------------------------------------------|
+| `pending`   | `Date` (now + 30m) | Abandoned checkout awaiting wallet signature / submission. Eligible for TTL reaping. |
+| `submitted` | retained          | In-flight on the Stellar network. Transient, non-terminal.         |
+| `retrying`  | retained          | In-flight async on-chain verification. Transient, non-terminal.    |
+| `confirmed` | unset             | Settled on-chain — item access granted / donation recorded. **Must never be reaped.** |
+| `failed`    | unset             | Permanent failure. Kept for audit and reconciliation.              |
+| `expired`   | unset             | Cancelled by the user or explicitly timed out. Kept for audit.     |
+| `refunded`  | unset             | Refund executed on-chain. Kept for audit.                          |
+| `disputed`  | unset             | Under administrator review. Kept for audit.                        |
+
+## Defense in depth
+
+The guarantee is enforced at three independent layers, so no single future code
+path can regress confirmed rows back into the reaper's window:
+
+1. **Partial TTL index (structural).** `src/models/Transaction.js` declares
+   `transactionSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0, partialFilterExpression: { status: "pending" } })`.
+   MongoDB's TTL monitor only considers documents that match the partial filter,
+   so a non-`pending` document is never a deletion candidate even if it somehow
+   still carries an `expiresAt`.
+
+2. **Conditional schema default.** The `expiresAt` default only produces a
+   timestamp when the document's status is `pending` (or unset at creation).
+   Records created directly in a terminal state — e.g. worker-created confirmed
+   donations/purchases from the reconciliation service — are never born with an
+   expiry.
+
+3. **Pre-save hook (runtime).** A `pre("save")` middleware clears `expiresAt`
+   whenever a document is saved in a terminal status (`confirmed`, `failed`,
+   `expired`, `refunded`, `disputed`). Even if a controller forgets to unset it
+   explicitly, the model enforces the invariant.
+
+On top of the schema layers, every current transition to a terminal state also
+unsets `expiresAt` explicitly for clarity:
+
+- `submitPayment` / `submitDonation` — validation failure, Stellar error,
+  verification failure, and confirmation paths.
+- `cancelTransaction` — `$unset: { expiresAt: 1 }` alongside `status: "expired"`.
+- `submitRefund` / `escalateDispute` — `$unset: { expiresAt: 1 }` alongside the
+  terminal status update.
+- `promoteTransaction` (reconciliation) and the `verifyPaymentOnChain` job —
+  cleared before saving the confirmed/failed row.
+
+## Migrations
+
+Databases created before this invariant may still hold confirmed rows with a
+30-minute `expiresAt` and the old blanket TTL index. Run the idempotent
+migration to rescue those rows and swap the index:
+
+```bash
+node src/migrations/fixTtlTransactionExpiry.js
+```
+
+It (1) `$unset`s `expiresAt` on all non-`pending` rows and (2) drops the blanket
+`{ expiresAt: 1 }` index and recreates it with the `partialFilterExpression`.
+Running it again is a no-op.
+
+## Out of scope
+
+The `Session` and `Refund` collections have their own TTL indexes on
+`expiresAt`; those are intentional and correct (revoked/abandoned sessions and
+expired refund windows should be reaped) and are not affected by this invariant.

src/controllers/stellar/donationController.js

@@ -180,6 +180,7 @@ export const submitDonation = async (req, res) => {
       );
     } catch (validationError) {
       donation.status = "failed";
+      donation.expiresAt = undefined;
       donation.failureReason = `validation_failed: ${validationError.message}`;
       await donation.save({ session });
       await session.commitTransaction();
@@ -206,6 +207,7 @@ export const submitDonation = async (req, res) => {
       result = await submitTransaction(signedXdr);
     } catch (stellarError) {
       donation.status = "failed";
+      donation.expiresAt = undefined;
       donation.failureReason = stellarError.message;
       await donation.save({ session });
       await session.commitTransaction();
@@ -250,6 +252,7 @@ export const submitDonation = async (req, res) => {
         });
       }
       donation.status = "failed";
+      donation.expiresAt = undefined;
       donation.failureReason = `On-chain verification failed: ${verification.reason}`;
       await donation.save({ session });
       await session.commitTransaction();
@@ -271,6 +274,7 @@ export const submitDonation = async (req, res) => {
     donation.stellarLedger = result.ledger;
     donation.status = "confirmed";
     donation.confirmedAt = new Date();
+    donation.expiresAt = undefined; // terminal state — never TTL-reapable
     await donation.save({ session });
     await enqueue(
       "generateReceipt",

src/controllers/stellar/paymentController.js

@@ -642,6 +642,7 @@ export const submitPayment = async (req, res) => {
       );
     } catch (validationError) {
       transaction.status = "failed";
+      transaction.expiresAt = undefined;
       transaction.failureReason = `validation_failed: ${validationError.message}`;
       await transaction.save({ session });
       await session.commitTransaction();
@@ -667,6 +668,7 @@ export const submitPayment = async (req, res) => {
       result = await submitTransaction(signedXdr);
     } catch (stellarError) {
       transaction.status = "failed";
+      transaction.expiresAt = undefined;
       transaction.failureReason = stellarError.message;
       await transaction.save({ session });
       await session.commitTransaction();
@@ -727,6 +729,7 @@ export const submitPayment = async (req, res) => {
         });
       }
       transaction.status = "failed";
+      transaction.expiresAt = undefined;
       transaction.failureReason = `On-chain verification failed: ${verification.reason}`;
       await transaction.save({ session });
       await session.commitTransaction();
@@ -761,6 +764,7 @@ export const submitPayment = async (req, res) => {
     transaction.stellarLedger = result.ledger;
     transaction.status = "confirmed";
     transaction.confirmedAt = new Date();
+    transaction.expiresAt = undefined; // terminal state — never TTL-reapable
     await transaction.save({ session });
     paymentsConfirmed.inc({ type: "purchase" });
 
@@ -962,8 +966,13 @@ export const cancelTransaction = async (req, res) => {
         status: "pending",
       },
       {
-        status: "expired",
-        failureReason: "Cancelled by user",
+        $set: {
+          status: "expired",
+          failureReason: "Cancelled by user",
+        },
+        $unset: {
+          expiresAt: 1,
+        },
       },
       { new: true }
     );

src/controllers/stellar/refundController.js

@@ -321,7 +321,10 @@ export const submitRefund = async (req, res) => {
 
       await Transaction.findByIdAndUpdate(
         refund.originalTransaction,
-        { status: "refunded", refund: refund._id }
+        {
+          $set: { status: "refunded", refund: refund._id },
+          $unset: { expiresAt: 1 }, // terminal state — never TTL-reapable
+        }
       );
 
       logger.info(`Refund confirmed and access revoked atomically for refund ${refund._id}`);
@@ -432,7 +435,8 @@ export const escalateDispute = async (req, res) => {
     await refund.save();
 
     await Transaction.findByIdAndUpdate(refund.originalTransaction, {
-      status: "disputed",
+      $set: { status: "disputed" },
+      $unset: { expiresAt: 1 }, // terminal state — never TTL-reapable
     });
 
     logger.info(`Refund ${refund._id} escalated to dispute by buyer ${buyerId}`);

src/jobs/handlers.js

@@ -48,13 +48,15 @@ registerJob("verifyPaymentOnChain", async ({ transactionId }, context) => {
       throw new Error(verification.reason);
     }
     transaction.status = "failed";
+    transaction.expiresAt = undefined; // terminal state — never TTL-reapable
     transaction.failureReason = `On-chain verification failed: ${verification.reason}`;
     await transaction.save();
     return;
   }
 
   transaction.status = "confirmed";
   transaction.confirmedAt = new Date();
+  transaction.expiresAt = undefined; // terminal state — never TTL-reapable
   transaction.failureReason = undefined;
   await transaction.save();
 

src/migrations/fixTtlTransactionExpiry.js

@@ -0,0 +1,82 @@
+import dotenv from "dotenv";
+import mongoose from "mongoose";
+import Transaction from "../models/Transaction.js";
+import logger from "../config/logger.js";
+
+dotenv.config();
+
+/**
+ * Migration: fixTtlTransactionExpiry
+ *
+ * The `Transaction` collection previously had a blanket TTL index on
+ * `expiresAt` ({ expireAfterSeconds: 0 }) and a schema default that stamped a
+ * 30-minute expiry on every row regardless of status. Because confirm paths
+ * never cleared `expiresAt`, confirmed purchases/donations were reaped ~30
+ * minutes after creation — deleting the proof of payment and leaving orphaned
+ * earnings behind.
+ *
+ * This migration:
+ *   1. `$unset`s `expiresAt` on every existing non-`pending` transaction so
+ *      the TTL reaper can never touch already-confirmed (or otherwise
+ *      terminal) rows before the index swap completes.
+ *   2. Drops the blanket `{ expiresAt: 1 }` TTL index (if present) and
+ *      recreates it as a partial index scoped strictly to
+ *      `{ status: "pending" }`, so reaping is structurally impossible for
+ *      non-pending rows even if a future code path forgets step 1.
+ *
+ * Idempotent: running it again is a no-op for data (no non-pending rows carry
+ * `expiresAt`) and for the index (the partial index already matches).
+ */
+export const fixTtlTransactionExpiry = async () => {
+  const collection = Transaction.collection;
+
+  // 1. Rescue legacy terminal rows from the TTL reaper before touching indexes.
+  const updateResult = await Transaction.updateMany(
+    {
+      status: { $ne: "pending" },
+      expiresAt: { $exists: true, $ne: null },
+    },
+    { $unset: { expiresAt: 1 } }
+  );
+
+  const modifiedCount = updateResult.modifiedCount ?? updateResult.nModified ?? 0;
+  logger.info(`Unset expiresAt on ${modifiedCount} non-pending transaction(s).`);
+
+  // 2. Replace the blanket TTL index with the partial-filter version. A schema
+  //    `.index()` edit does NOT alter an already-built index, so this must be
+  //    done explicitly.
+  const indexes = await collection.indexes();
+  const ttlIndex = indexes.find((idx) => idx.key && idx.key.expiresAt === 1);
+
+  const hasPendingPartialFilter =
+    ttlIndex?.partialFilterExpression?.status === "pending";
+
+  if (ttlIndex && !hasPendingPartialFilter) {
+    logger.info(`Dropping blanket TTL index "${ttlIndex.name}"...`);
+    await collection.dropIndex(ttlIndex.name);
+  }
+
+  // If the correct partial index already exists this is a no-op.
+  await collection.createIndex(
+    { expiresAt: 1 },
+    { expireAfterSeconds: 0, partialFilterExpression: { status: "pending" } }
+  );
+  logger.info("Ensured partial TTL index scoped to status: pending.");
+
+  return { modifiedCount };
+};
+
+// Standalone CLI execution
+if (process.argv[1] && process.argv[1].endsWith("fixTtlTransactionExpiry.js")) {
+  if (!process.env.MONGO_URI) {
+    throw new Error("MONGO_URI must be set to run TTL transaction expiry migration");
+  }
+
+  try {
+    await mongoose.connect(process.env.MONGO_URI);
+    const result = await fixTtlTransactionExpiry();
+    console.log(`Migration complete: ${result.modifiedCount} documents updated.`);
+  } finally {
+    await mongoose.disconnect();
+  }
+}

src/models/Transaction.js

@@ -147,15 +147,43 @@ const transactionSchema = new mongoose.Schema(
     confirmedAt: Date,
     expiresAt: {
       type: Date,
-      default: () => new Date(Date.now() + 30 * 60 * 1000), // 30 minutes
+      // Only abandoned `pending` checkouts get a reaping deadline. Records
+      // created directly in a terminal state (e.g. worker-created confirmed
+      // donations/purchases) must never be born with an expiry.
+      default: function () {
+        return this.status === "pending" || !this.status
+          ? new Date(Date.now() + 30 * 60 * 1000) // 30 minutes
+          : undefined;
+      },
     },
   },
   { timestamps: true }
 );
+
+// Terminal statuses are permanent records (paid purchases, donations, refunds,
+// disputes, failures) that must never be reaped by the TTL monitor.
+const TERMINAL_STATUSES = ["confirmed", "failed", "expired", "refunded", "disputed"];
+
+// TTL invariant: `expiresAt` is only meaningful for abandoned `pending`
+// checkouts. Enforce it at the schema level so a future code path that forgets
+// to clear `expiresAt` cannot regress confirmed/terminal rows back into the
+// TTL reaper's window — defense in depth on top of the partial index below.
+transactionSchema.pre("save", function (next) {
+  if (TERMINAL_STATUSES.includes(this.status)) {
+    this.expiresAt = undefined;
+  }
+  next();
+});
+
 // Indexes for efficient queries
 transactionSchema.index({ buyer: 1, status: 1 });
 transactionSchema.index({ creator: 1, status: 1 });
 transactionSchema.index({ itemType: 1, itemId: 1 });
 transactionSchema.index({ type: 1, status: 1, createdAt: -1 }); // Donation stats
-transactionSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 }); // TTL for expired pending
+// TTL for expired pending checkouts only — a blanket index would also reap
+// confirmed purchases/donations once their original 30-minute expiry passes.
+transactionSchema.index(
+  { expiresAt: 1 },
+  { expireAfterSeconds: 0, partialFilterExpression: { status: "pending" } }
+);
 export default mongoose.model("Transaction", transactionSchema);

src/services/stellar/reconciliationService.js

@@ -55,6 +55,7 @@ const promoteTransaction = async (transaction, paymentRecord) => {
   transaction.stellarLedger = paymentRecord.ledger || undefined;
   transaction.status = "confirmed";
   transaction.confirmedAt = new Date();
+  transaction.expiresAt = undefined; // terminal state — never TTL-reapable
   await transaction.save();
 
   await recordSaleEarnings(transaction);
@@ -86,6 +87,10 @@ const createConfirmedDonation = async ({ sourceAccount, amount, hash, memo }) =>
     status: "confirmed",
     stellarTxHash: hash,
     confirmedAt: new Date(),
+    // Terminal state: the conditional schema default omits expiresAt, and the
+    // pre-save hook enforces it — an already-confirmed row must never carry a
+    // TTL deadline.
+    expiresAt: undefined,
   });
 
   await donation.save();
@@ -130,6 +135,10 @@ const createConfirmedPurchase = async ({ sourceAccount, amount, hash, memo, item
     status: "confirmed",
     stellarTxHash: hash,
     confirmedAt: new Date(),
+    // Terminal state: the conditional schema default omits expiresAt, and the
+    // pre-save hook enforces it — an already-confirmed row must never carry a
+    // TTL deadline.
+    expiresAt: undefined,
   });
 
   await purchase.save();

test/reconciliation.test.js

@@ -236,6 +236,9 @@ describe("Payment Reconciliation Service", () => {
       const updated = await Transaction.findById(tx._id);
       expect(updated.status).toBe("confirmed");
       expect(updated.confirmedAt).toBeDefined();
+      // Terminal state — the reconciliation confirm path must leave the row
+      // without an expiry so the TTL reaper can never delete it.
+      expect(updated.expiresAt).toBeUndefined();
       expect(mockRecordSaleEarnings).toHaveBeenCalled();
     });
 

test/stellarPaymentController.test.js

@@ -334,6 +334,9 @@ describe("Stellar payment controller", () => {
       { session }
     );
     expect(tx.status).toBe("confirmed");
+    // Terminal state — the submit confirm path must leave the row without an
+    // expiry so the TTL reaper can never delete it.
+    expect(tx.expiresAt).toBeUndefined();
     expect(session.commitTransaction).toHaveBeenCalledTimes(1);
     expect(session.abortTransaction).not.toHaveBeenCalled();
   });