feat: Add background retry for pending claim disbursements

app/backend/prisma/migrations/20260529000000_add_disbursing_status/migration.sql

@@ -0,0 +1,36 @@
+-- For SQLite, enum changes require recreating the table
+-- This migration adds the 'disbursing' status to ClaimStatus enum
+
+CREATE TABLE "Claim_new" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "deletedAt" DATETIME,
+    "status" TEXT NOT NULL DEFAULT 'requested',
+    "campaignId" TEXT NOT NULL,
+    "amount" REAL NOT NULL,
+    "recipientRef" TEXT NOT NULL,
+    "evidenceRef" TEXT,
+    "expiresAt" DATETIME,
+    "cancelledAt" DATETIME,
+    "cancelledBy" TEXT,
+    "cancelReason" TEXT,
+    "reissuedFromId" TEXT,
+    FOREIGN KEY ("campaignId") REFERENCES "Campaign"("id") ON DELETE CASCADE ON UPDATE CASCADE,
+    FOREIGN KEY ("reissuedFromId") REFERENCES "Claim"("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+INSERT INTO "Claim_new" ("id", "createdAt", "updatedAt", "deletedAt", "status", "campaignId", "amount", "recipientRef", "evidenceRef", "expiresAt", "cancelledAt", "cancelledBy", "cancelReason", "reissuedFromId")
+SELECT "id", "createdAt", "updatedAt", "deletedAt", "status", "campaignId", "amount", "recipientRef", "evidenceRef", "expiresAt", "cancelledAt", "cancelledBy", "cancelReason", "reissuedFromId"
+FROM "Claim";
+
+DROP TABLE "Claim";
+
+ALTER TABLE "Claim_new" RENAME TO "Claim";
+
+CREATE INDEX "Claim_status_idx" ON "Claim"("status");
+CREATE INDEX "Claim_campaignId_idx" ON "Claim"("campaignId");
+CREATE INDEX "Claim_createdAt_idx" ON "Claim"("createdAt");
+CREATE INDEX "Claim_deletedAt_idx" ON "Claim"("deletedAt");
+CREATE INDEX "Claim_reissuedFromId_idx" ON "Claim"("reissuedFromId");
+CREATE INDEX "Claim_expiresAt_idx" ON "Claim"("expiresAt");

app/backend/prisma/schema.prisma

@@ -19,6 +19,7 @@ enum ClaimStatus {
   requested
   verified
   approved
+  disbursing
   disbursed
   archived
   cancelled

app/backend/src/claims/claim-retry.service.ts

@@ -0,0 +1,217 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { Cron, CronExpression } from '@nestjs/schedule';
+import { PrismaService } from '../prisma/prisma.service';
+import { InjectQueue } from '@nestjs/bullmq';
+import { Queue } from 'bullmq';
+import { ClaimStatus } from '@prisma/client';
+import { ConfigService } from '@nestjs/config';
+import { createHash } from 'crypto';
+
+/**
+ * Service to handle background retry logic for stuck or failed claim disbursements
+ */
+@Injectable()
+export class ClaimRetryService {
+  private readonly logger = new Logger(ClaimRetryService.name);
+  private readonly maxDisbursingDuration: number;
+  private readonly maxRetryAttempts: number;
+
+  constructor(
+    private readonly prisma: PrismaService,
+    @InjectQueue('onchain') private readonly onchainQueue: Queue,
+    private readonly configService: ConfigService,
+  ) {
+    this.maxDisbursingDuration = 
+      this.configService.get<number>('CLAIM_MAX_DISBURSING_DURATION', 30 * 60 * 1000);
+    this.maxRetryAttempts = 
+      this.configService.get<number>('CLAIM_MAX_RETRY_ATTEMPTS', 5);
+  }
+
+  @Cron(CronExpression.EVERY_5_MINUTES)
+  async handleStuckDisbursements(): Promise<void> {
+    try {
+      this.logger.log('Checking for stuck disbursements...');
+      const result = await this.checkAndRetryStuckDisbursements();
+
+      if (result.retriedCount > 0) {
+        this.logger.log(`Retried ${result.retriedCount} stuck disbursement(s)`);
+      }
+
+      if (result.revertedCount > 0) {
+        this.logger.warn(`Reverted ${result.revertedCount} disbursement(s) to approved status`);
+      }
+
+      if (result.alertCount > 0) {
+        this.logger.error(`Generated ${result.alertCount} alert(s) for failed disbursements`);
+      }
+    } catch (error) {
+      this.logger.error(
+        'Failed to check for stuck disbursements',
+        error instanceof Error ? error.stack : undefined,
+      );
+    }
+  }
+
+  async checkAndRetryStuckDisbursements(): Promise<{
+    retriedCount: number;
+    revertedCount: number;
+    alertCount: number;
+  }> {
+    const stuckThreshold = new Date(Date.now() - this.maxDisbursingDuration);
+
+    const stuckClaims = await this.prisma.claim.findMany({
+      where: {
+        status: ClaimStatus.disbursing,
+        updatedAt: { lt: stuckThreshold },
+        deletedAt: null,
+      },
+      include: { campaign: true },
+    });
+
+    if (stuckClaims.length === 0) {
+      return { retriedCount: 0, revertedCount: 0, alertCount: 0 };
+    }
+
+    this.logger.log(`Found ${stuckClaims.length} claim(s) stuck in disbursing status`);
+
+    let retriedCount = 0;
+    let revertedCount = 0;
+    let alertCount = 0;
+
+    for (const claim of stuckClaims) {
+      try {
+        const jobs = await this.onchainQueue.getJobs(['waiting', 'active', 'delayed'], 0, 100);
+        const existingJob = jobs.find(job => 
+          job.data.type === 'disburse' && 
+          job.data.params.claimId === claim.id
+        );
+
+        if (existingJob) {
+          if (existingJob.attemptsMade >= this.maxRetryAttempts) {
+            await this.revertClaimToApproved(claim.id);
+            revertedCount++;
+            this.generateAlert(claim, 'max_retries_reached');
+            alertCount++;
+          }
+        } else {
+          await this.retryDisbursement(claim);
+          retriedCount++;
+        }
+      } catch (error) {
+        this.logger.error(
+          `Failed to process stuck claim ${claim.id}: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        );
+        this.generateAlert(claim, 'processing_failed');
+        alertCount++;
+      }
+    }
+
+    return { retriedCount, revertedCount, alertCount };
+  }
+
+  private async retryDisbursement(claim: any): Promise<void> {
+    const packageId = this.generateMockPackageId(claim.id);
+    const tokenAddress = this.getTokenAddressForClaim(claim);
+
+    await this.onchainQueue.add(
+      'disburse',
+      {
+        type: 'disburse',
+        params: {
+          claimId: claim.id,
+          packageId,
+          recipientAddress: claim.recipientRef,
+          amount: claim.amount.toString(),
+          tokenAddress,
+        },
+        timestamp: Date.now(),
+      },
+      {
+        attempts: this.maxRetryAttempts,
+        backoff: { type: 'exponential', delay: 2000 },
+        removeOnComplete: { count: 10 },
+        removeOnFail: { count: 50 },
+      },
+    );
+
+    this.logger.log(`Re-enqueued disbursement job for claim ${claim.id}`);
+  }
+
+  private async revertClaimToApproved(claimId: string): Promise<void> {
+    await this.prisma.claim.update({
+      where: { id: claimId },
+      data: { status: ClaimStatus.approved },
+    });
+
+    this.logger.log(`Reverted claim ${claimId} from disbursing to approved`);
+  }
+
+  private generateAlert(claim: any, reason: string): void {
+    const alertMessage = {
+      claimId: claim.id,
+      campaignId: claim.campaignId,
+      amount: claim.amount,
+      reason,
+      timestamp: new Date().toISOString(),
+    };
+
+    this.logger.error(`Claim disbursement alert: ${JSON.stringify(alertMessage)}`);
+  }
+
+  private generateMockPackageId(claimId: string): string {
+    const hash = createHash('sha256')
+      .update(`package-${claimId}`)
+      .digest('hex');
+    return BigInt('0x' + hash.substring(0, 16)).toString();
+  }
+
+  private getTokenAddressForClaim(claim: any): string {
+    const defaultTokenAddress =
+      'GATEMHCCKCY67ZUCKTROYN24ZYT5GK4EQZ5LKG3FZTSZ3NYNEJBBENSN';
+
+    const claimMetadata = claim.metadata as Record<string, unknown> | undefined;
+    if (claimMetadata?.tokenAddress) {
+      return claimMetadata.tokenAddress as string;
+    }
+
+    const campaignMetadata = claim.campaign?.metadata as
+      | Record<string, unknown>
+      | undefined;
+    if (campaignMetadata?.tokenAddress) {
+      return campaignMetadata.tokenAddress as string;
+    }
+
+    return defaultTokenAddress;
+  }
+
+  async getDisbursementStats(): Promise<{
+    disbursing: number;
+    stuck: number;
+    recentlyFailed: number;
+  }> {
+    const stuckThreshold = new Date(Date.now() - this.maxDisbursingDuration);
+    const recentFailureThreshold = new Date(Date.now() - 24 * 60 * 60 * 1000);
+
+    const [disbursing, stuck, recentlyFailed] = await Promise.all([
+      this.prisma.claim.count({
+        where: { status: ClaimStatus.disbursing, deletedAt: null },
+      }),
+      this.prisma.claim.count({
+        where: {
+          status: ClaimStatus.disbursing,
+          updatedAt: { lt: stuckThreshold },
+          deletedAt: null,
+        },
+      }),
+      this.prisma.claim.count({
+        where: {
+          status: ClaimStatus.approved,
+          updatedAt: { gte: recentFailureThreshold },
+          deletedAt: null,
+        },
+      }),
+    ]);
+
+    return { disbursing, stuck, recentlyFailed };
+  }
+}

app/backend/src/claims/claims.controller.ts

@@ -9,6 +9,7 @@ import {
   Request,
   Res,
   Version,
+  UseGuards,
 } from '@nestjs/common';
 import type { Response } from 'express';
 import { Request as ExpressRequest } from 'express';
@@ -40,6 +41,8 @@ import { AppRole } from 'src/auth/app-role.enum';
 import { InternalNotesService } from 'src/common/services/internal-notes.service';
 import { CreateInternalNoteDto } from 'src/common/dto/create-internal-note.dto';
 import { InternalNoteResponseDto } from 'src/common/dto/internal-note-response.dto';
+import { CostAwareRateLimitGuard } from 'src/common/guards/cost-aware-rate-limit.guard';
+import { RateLimit } from 'src/common/decorators/rate-limit.decorator';
 
 @ApiTags('Onchain Proxy')
 @ApiBearerAuth('JWT-auth')
@@ -52,6 +55,8 @@ export class ClaimsController {
   ) {}
 
   @Post()
+  @UseGuards(CostAwareRateLimitGuard)
+  @RateLimit({ limit: 50, window: 60, cost: 5 }) // Write operation
   @ApiOperation({
     summary: 'Create a claim',
     description: 'Initializes a new claim for a specific campaign.',
@@ -144,6 +149,8 @@ export class ClaimsController {
 
   @Post(':id/disburse')
   @Roles(AppRole.admin)
+  @UseGuards(CostAwareRateLimitGuard)
+  @RateLimit({ limit: 10, window: 60, cost: 20 }) // Expensive on-chain operation
   @ApiOperation({
     summary: 'Disburse funds for a claim',
     description:

app/backend/src/claims/claims.module.ts

@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
 import { ClaimsService } from './claims.service';
 import { ClaimsController } from './claims.controller';
 import { CancelAndReissueService } from './cancel-and-reissue.service';
+import { ClaimRetryService } from './claim-retry.service';
 import { PrismaModule } from '../prisma/prisma.module';
 import { OnchainModule } from '../onchain/onchain.module';
 import { MetricsModule } from '../observability/metrics/metrics.module';
@@ -22,7 +23,7 @@ import { CommonServicesModule } from '../common/services/common-services.module'
     CommonServicesModule,
   ],
   controllers: [ClaimsController],
-  providers: [ClaimsService, CancelAndReissueService, BudgetService],
+  providers: [ClaimsService, CancelAndReissueService, BudgetService, ClaimRetryService],
   exports: [CancelAndReissueService],
 })
 export class ClaimsModule {}