feat(backend): transactional escrow orchestration service (#21)

backend-api/README.md

@@ -308,6 +308,23 @@ docker compose up -d
 CI runs the same suite against a Postgres service container on every push/PR
 to `development` (`.github/workflows/test.yml`).
 
+**Background `@Scheduled` pollers default to off in tests.** `pom.xml`'s
+`maven-surefire-plugin` sets `chain.events.poll-delay-ms`,
+`escrow.orchestration.*-poll-delay-ms`, and
+`escrow.reconciliation.poll-delay-ms` to 1 hour via `systemPropertyVariables`
+for every test JVM. Without this, a `@SpringBootTest` class that doesn't
+explicitly disable scheduling leaves its poller running against the shared
+test database for the rest of the test JVM's life (Spring caches
+`ApplicationContext`s), racing with whatever test class runs next and
+processing its rows out from under it — this was an actual source of
+intermittent CI failures before the fix. Tests that specifically want a
+poller running (e.g. `ChainEventServiceIntegrationTest`,
+`EscrowOrchestrationIntegrationTest`) override the relevant property via
+their own `@SpringBootTest(properties = …)`, which takes precedence over
+the surefire-level system properties. Don't remove that
+`systemPropertyVariables` block without replacing it with an equivalent
+per-test opt-out.
+
 Existing tests cover `MailServiceTest`, `ClientServiceTest`,
 `SkilledWorkerServiceTest`, `AppointmentServiceTest`, `ReviewServiceTest`, and
 `SkillServiceTest`. All 14 tests pass as of this writing.

backend-api/pom.xml

@@ -164,6 +164,35 @@
 					</excludes>
 				</configuration>
 			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<configuration>
+					<!--
+					    Background @Scheduled pollers (chain-event ingestion, escrow
+					    orchestration) default to a 1s delay. Every @SpringBootTest spins
+					    up its own ApplicationContext, and Spring caches those contexts,
+					    including their live scheduler threads, for the rest of this test
+					    JVM's life. A @SpringBootTest class that doesn't explicitly
+					    disable scheduling leaves a poller running against the shared
+					    test database indefinitely, racing with whatever test class runs
+					    next and stealing/mutating its rows.
+
+					    Defaulting the delays to effectively "off" here means only tests
+					    that explicitly want a poller running opt into it, via their own
+					    @SpringBootTest(properties = ...). Inline test properties take
+					    precedence over these system properties, so
+					    ChainEventServiceIntegrationTest and
+					    EscrowOrchestrationIntegrationTest are unaffected.
+					-->
+					<systemPropertyVariables>
+						<chain.events.poll-delay-ms>3600000</chain.events.poll-delay-ms>
+						<escrow.orchestration.submit-poll-delay-ms>3600000</escrow.orchestration.submit-poll-delay-ms>
+						<escrow.orchestration.confirm-poll-delay-ms>3600000</escrow.orchestration.confirm-poll-delay-ms>
+						<escrow.reconciliation.poll-delay-ms>3600000</escrow.reconciliation.poll-delay-ms>
+					</systemPropertyVariables>
+				</configuration>
+			</plugin>
 		</plugins>
 	</build>
 

backend-api/src/main/java/com/guildworkman/api/chain/repository/OnChainEventRepository.java

@@ -18,4 +18,20 @@ public interface OnChainEventRepository extends JpaRepository<OnChainEvent, Long
     List<OnChainEvent> claimNext(@Param("statuses") Set<ChainEventStatus> statuses, @Param("now") Instant now, Pageable pageable);
     List<OnChainEvent> findByLedgerBetweenOrderByContractIdAscLedgerAscEventIndexAsc(long fromLedger, long toLedger);
     long countByStatus(ChainEventStatus status);
+    /**
+     * Used by {@code EscrowReconciliationService} to find events tagged with
+     * a given operation ref. {@code topics} is a JSON-encoded string column
+     * (see {@code ChainEventInserter}), not a normalized list, so this is a
+     * {@code contract_id = ? AND topics LIKE '%' || ? || '%'} scan — the
+     * {@code idx_chain_event_stream_order}/{@code idx_chain_event_status}
+     * indexes narrow by {@code contract_id} but can't help with the
+     * {@code LIKE} itself, so it's a sequential scan over every event for
+     * that contract. Fine at today's volumes; if a single contract's event
+     * count grows large enough for this to show up in slow-query logs, the
+     * fix is a normalized child table (one row per {@code (event_id, topic)}
+     * pair, indexed on {@code topic}) or a Postgres {@code jsonb} column with
+     * a GIN index — not a bigger {@code LIKE} index, which Postgres can't use
+     * for a leading-wildcard search anyway.
+     */
+    List<OnChainEvent> findByContractIdAndTopicsContaining(String contractId, String topicFragment);
 }

backend-api/src/main/java/com/guildworkman/api/escrow/api/EscrowOrchestrationController.java

@@ -0,0 +1,63 @@
+package com.guildworkman.api.escrow.api;
+
+import com.guildworkman.api.escrow.service.EscrowOrchestrationService;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import jakarta.validation.Valid;
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.*;
+
+/**
+ * See {@code docs/ESCROW_ORCHESTRATION.md} for the full HTTP-behavior write-up.
+ * In short: {@code POST} always returns {@code 202 Accepted} — for a request
+ * whose {@code idempotencyKey} already exists, the body is the pre-existing
+ * request's current state (not a fresh copy), signalled by the
+ * {@code X-Idempotent-Replay} response header rather than a different status
+ * code, since a replay is not an error condition.
+ */
+@RestController
+@RequestMapping("/api/v1/escrow/orchestrations")
+@RequiredArgsConstructor
+@SecurityRequirement(name = "bearerAuth")
+public class EscrowOrchestrationController {
+
+    /** Set to {@code true}/{@code false} on the submit response; see class Javadoc. */
+    public static final String IDEMPOTENT_REPLAY_HEADER = "X-Idempotent-Replay";
+
+    private final EscrowOrchestrationService service;
+
+    @PostMapping
+    @ResponseStatus(HttpStatus.ACCEPTED)
+    @Operation(summary = "Submit a signed escrow-contract transaction for orchestration",
+            description = "Idempotent on idempotencyKey: resubmitting the same key returns the "
+                    + "original request (see the X-Idempotent-Replay response header) rather than "
+                    + "creating a second one. Always 202 Accepted, whether newly created or replayed.")
+    public ResponseEntity<EscrowOrchestrationResponse> submit(@Valid @RequestBody SubmitOrchestrationRequest request) {
+        var outcome = service.submit(request);
+        return ResponseEntity.status(HttpStatus.ACCEPTED)
+                .header(IDEMPOTENT_REPLAY_HEADER, String.valueOf(outcome.replayed()))
+                .body(EscrowOrchestrationResponse.from(outcome.request()));
+    }
+
+    @GetMapping("/{id}")
+    @Operation(summary = "Fetch a request's current status",
+            description = "Returns the request's current-state timestamps (submittedAt/confirmedAt/reconciledAt) "
+                    + "and attempt count. It is not a per-attempt audit log — individual submit/poll attempts "
+                    + "are only recorded in application logs, correlated by orchestration request id.")
+    public EscrowOrchestrationResponse get(@PathVariable Long id) {
+        return EscrowOrchestrationResponse.from(service.get(id));
+    }
+
+    @PostMapping("/{id}/requeue-reconciliation")
+    @PreAuthorize("hasRole('ADMIN')")
+    @Operation(summary = "Requeue a MISMATCHED request for reconciliation",
+            description = "ADMIN only. Resets reconciliationStatus back to PENDING so the next reconciliation "
+                    + "sweep reconsiders it -- e.g. after confirming the on-chain event the sweep was missing "
+                    + "has since been ingested. 409 if the request isn't currently MISMATCHED.")
+    public EscrowOrchestrationResponse requeueReconciliation(@PathVariable Long id) {
+        return EscrowOrchestrationResponse.from(service.requeueReconciliation(id));
+    }
+}

backend-api/src/main/java/com/guildworkman/api/escrow/api/EscrowOrchestrationResponse.java

@@ -0,0 +1,38 @@
+package com.guildworkman.api.escrow.api;
+
+import com.guildworkman.api.escrow.model.EscrowOrchestrationRequest;
+
+import java.time.Instant;
+
+public record EscrowOrchestrationResponse(
+        Long id,
+        String idempotencyKey,
+        String operationType,
+        String contractId,
+        String operationRef,
+        String status,
+        String sorobanTxHash,
+        String reconciliationStatus,
+        int attempts,
+        String lastError,
+        Instant submittedAt,
+        Instant confirmedAt,
+        Instant reconciledAt) {
+
+    public static EscrowOrchestrationResponse from(EscrowOrchestrationRequest r) {
+        return new EscrowOrchestrationResponse(
+                r.getId(),
+                r.getIdempotencyKey(),
+                r.getOperationType().name(),
+                r.getContractId(),
+                r.getOperationRef(),
+                r.getStatus().name(),
+                r.getSorobanTxHash(),
+                r.getReconciliationStatus().name(),
+                r.getAttempts(),
+                r.getLastError(),
+                r.getSubmittedAt(),
+                r.getConfirmedAt(),
+                r.getReconciledAt());
+    }
+}

backend-api/src/main/java/com/guildworkman/api/escrow/api/SubmitOrchestrationRequest.java

@@ -0,0 +1,33 @@
+package com.guildworkman.api.escrow.api;
+
+import com.guildworkman.api.escrow.model.EscrowOperationType;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
+import jakarta.validation.constraints.Pattern;
+import jakarta.validation.constraints.Size;
+
+/**
+ * Submits one escrow-contract operation for orchestration.
+ *
+ * @param idempotencyKey        caller-generated key; resubmitting the same key returns the original request
+ * @param operationType         which escrow contract entrypoint this transaction invokes
+ * @param contractId             Soroban contract id (strkey, e.g. {@code C...})
+ * @param operationRef           appointment id or milestone-escrow id this operation acts on, as a string
+ * @param signedTransactionXdr  base64 {@code TransactionEnvelope} XDR, already signed by the caller
+ */
+public record SubmitOrchestrationRequest(
+        @NotBlank @Size(max = 128) String idempotencyKey,
+        @NotNull EscrowOperationType operationType,
+        @NotBlank @Size(max = 128) String contractId,
+        // No '"' allowed: EscrowReconciliationService matches this value as a
+        // literal JSON-quoted substring against ingested event topics, so a
+        // stray quote could make it match (or fail to match) unrelated events.
+        @NotBlank @Size(max = 128) @Pattern(regexp = "[^\"]*", message = "must not contain '\"'") String operationRef,
+        // 8192 chars comfortably covers a realistic single-invoke-host-function
+        // envelope (typically well under 2KB base64-encoded) with headroom for
+        // multi-operation/fee-bump transactions, while still bounding request
+        // size against an oversized/malicious payload.
+        @NotBlank @Size(max = 8192)
+        @Pattern(regexp = "^[A-Za-z0-9+/]+={0,2}$", message = "must be base64-encoded")
+        String signedTransactionXdr) {
+}

backend-api/src/main/java/com/guildworkman/api/escrow/model/EscrowOperationType.java

@@ -0,0 +1,16 @@
+package com.guildworkman.api.escrow.model;
+
+/**
+ * The escrow contract entrypoints the orchestration service can submit.
+ * Mirrors the subset of {@code EscrowContract} functions (see
+ * {@code soroban-contracts/contracts/escrow}) that move funds or change
+ * escrow/appointment status on-chain.
+ */
+public enum EscrowOperationType {
+    CREATE_APPOINTMENT,
+    CONFIRM_COMPLETION,
+    CANCEL_APPOINTMENT,
+    RAISE_DISPUTE,
+    RESOLVE_DISPUTE,
+    RELEASE_MILESTONE_FUNDS
+}

backend-api/src/main/java/com/guildworkman/api/escrow/model/EscrowOrchestrationRequest.java

@@ -0,0 +1,96 @@
+package com.guildworkman.api.escrow.model;
+
+import jakarta.persistence.*;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.time.Instant;
+
+/**
+ * A single request to submit and confirm one escrow-contract operation over
+ * Soroban RPC.
+ *
+ * <p>The caller supplies an already-signed transaction envelope
+ * ({@link #signedTransactionXdr}) — this service does not build or sign
+ * transactions itself, it only relays them to Soroban RPC and tracks their
+ * outcome. That keeps idempotent submission, retry/backoff, and status
+ * polling entirely inside the JVM, without needing to hand-roll Soroban's
+ * XDR wire format.
+ *
+ * <p>{@link #idempotencyKey} is unique so retried client requests (e.g. a
+ * network retry on the REST call) never create a second row for the same
+ * logical operation; see {@code EscrowOrchestrationInserter}.
+ */
+@Entity
+@Table(name = "escrow_orchestration_requests",
+        uniqueConstraints = @UniqueConstraint(name = "uk_escrow_orch_idempotency_key", columnNames = "idempotency_key"),
+        indexes = {
+                @Index(name = "idx_escrow_orch_status", columnList = "status,next_attempt_at"),
+                @Index(name = "idx_escrow_orch_reconciliation", columnList = "reconciliation_status"),
+                @Index(name = "idx_escrow_orch_contract_ref", columnList = "contract_id,operation_ref")
+        })
+@Getter
+@Setter
+@NoArgsConstructor
+public class EscrowOrchestrationRequest {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(name = "idempotency_key", nullable = false, updatable = false, length = 128)
+    private String idempotencyKey;
+
+    @Enumerated(EnumType.STRING)
+    @Column(name = "operation_type", nullable = false, length = 32, updatable = false)
+    private EscrowOperationType operationType;
+
+    @Column(name = "contract_id", nullable = false, length = 128, updatable = false)
+    private String contractId;
+
+    /**
+     * Domain identifier the operation acts on (appointment id or milestone
+     * escrow id, as a string). Used to correlate this request with ingested
+     * on-chain events during reconciliation.
+     */
+    @Column(name = "operation_ref", nullable = false, length = 128, updatable = false)
+    private String operationRef;
+
+    /** Base64-encoded, already-signed {@code TransactionEnvelope} XDR. */
+    @Lob
+    @Column(name = "signed_transaction_xdr", nullable = false, updatable = false)
+    private String signedTransactionXdr;
+
+    @Enumerated(EnumType.STRING)
+    @Column(nullable = false, length = 20)
+    private OrchestrationStatus status = OrchestrationStatus.PENDING;
+
+    @Column(name = "soroban_tx_hash", length = 64)
+    private String sorobanTxHash;
+
+    @Enumerated(EnumType.STRING)
+    @Column(name = "reconciliation_status", nullable = false, length = 20)
+    private ReconciliationStatus reconciliationStatus = ReconciliationStatus.PENDING;
+
+    @Column(nullable = false)
+    private int attempts;
+
+    @Column(name = "next_attempt_at", nullable = false)
+    private Instant nextAttemptAt = Instant.now();
+
+    @Column(name = "last_error", length = 1000)
+    private String lastError;
+
+    @Column(name = "created_at", nullable = false, updatable = false)
+    private Instant createdAt = Instant.now();
+
+    private Instant submittedAt;
+
+    private Instant confirmedAt;
+
+    private Instant reconciledAt;
+
+    @Version
+    private long version;
+}

backend-api/src/main/java/com/guildworkman/api/escrow/model/OrchestrationStatus.java

@@ -0,0 +1,19 @@
+package com.guildworkman.api.escrow.model;
+
+/**
+ * Lifecycle of a single {@link EscrowOrchestrationRequest}.
+ *
+ * <pre>
+ * PENDING --submit--> SUBMITTED --poll(SUCCESS)--> CONFIRMED
+ *    |                    |
+ *    | (RPC errors,       | poll(FAILED on-chain)
+ *    |  retries exhausted) v
+ *    +----------------> FAILED
+ *    |
+ *    v (retries exhausted before ever reaching SUBMITTED)
+ * DEAD_LETTER
+ * </pre>
+ */
+public enum OrchestrationStatus {
+    PENDING, SUBMITTED, CONFIRMED, FAILED, DEAD_LETTER
+}

backend-api/src/main/java/com/guildworkman/api/escrow/model/ReconciliationStatus.java

@@ -0,0 +1,14 @@
+package com.guildworkman.api.escrow.model;
+
+/**
+ * Result of comparing a {@link OrchestrationStatus#CONFIRMED} request against
+ * the ingested on-chain event stream (see {@code com.guildworkman.api.chain}).
+ */
+public enum ReconciliationStatus {
+    /** Not yet due for reconciliation, or awaiting the indexer to catch up. */
+    PENDING,
+    /** A corresponding processed on-chain event was found. */
+    MATCHED,
+    /** No corresponding on-chain event appeared within the reconciliation window. */
+    MISMATCHED
+}