Add deterministic local indexing smoke flow

apps/s03-indexer/.gitignore

@@ -58,6 +58,12 @@ Thumbs.db
 .data
 .eslintcache
 
+# Local smoke run reports
+.smoke/
+
+# Local contract manifest generated by sync:contracts:local
+config/contracts.local.json
+
 # ENV local files
 .env.local
 .env.develop.local

apps/s03-indexer/README.md

@@ -127,6 +127,89 @@ fails fast on malformed contract IDs and required missing values. Missing
 `MARKET_TOKEN_*` values are reported as warnings because they are expected before
 market bootstrap.
 
+## Local Smoke Flow
+
+`smoke:local` is the one documented command that proves the whole stack works
+together: it drives real protocol actions through the deployed contracts and then
+asserts the indexer turned those on-chain events into GraphQL entities.
+
+```bash
+bun run --cwd apps/s03-indexer build      # verify the indexer compiles
+bun run --cwd apps/s03-indexer smoke:local
+```
+
+### What it does
+
+The runner executes layered, timed steps and records each one in a JSON report:
+
+1. **preflight** — checks `stellar` (and `make`/`docker` for fresh deploys) are on
+   PATH, locates the contracts repo, and ensures local-only signing keys exist
+   (generating and friendbot-funding them when needed). It never uses keys outside
+   the local/test keystore.
+2. **services** — confirms the Soroban RPC, Horizon, and SubQuery GraphQL endpoints
+   are reachable before any transaction is sent.
+3. **contracts-deploy** — validates a pre-existing local deployment discovered from
+   `.deployed/local.env`, or, on a fresh network, deploys test tokens + faucet,
+   deploys the protocol contracts, and bootstraps a market via the contract repo
+   `make` targets.
+4. **frontend-config** — runs `sync:contracts:local` so the indexer manifest matches
+   the freshly deployed contract IDs.
+5. **indexer-runtime** — rebuilds the indexer for the local config and starts the
+   Docker stack (skippable when it is already running).
+6. **contract-action** — submits fixed oracle prices, claims faucet tokens, then
+   creates and executes a deposit, a `MarketIncrease` that opens a long position, a
+   `MarketDecrease` that closes it, and (optionally) registers and sets a referral
+   code.
+7. **graphql-query** — waits for the indexer to catch up to the latest ledger and
+   asserts the expected `market`, `deposit`, `order`, `position`, and `transfer`
+   entity counts.
+
+### Run modes
+
+| Mode | Behavior |
+| --- | --- |
+| `--mode auto` (default) | Reuse a complete local deployment if present, otherwise deploy + bootstrap. |
+| `--mode fresh` | Always deploy test tokens, contracts, and bootstrap a market. |
+| `--mode existing` | Require a complete `.deployed/local.env`; fail fast if missing. |
+
+### Common options
+
+All options also have `SMOKE_*` environment-variable equivalents.
+
+```bash
+bun run --cwd apps/s03-indexer smoke:local \
+  --contracts-repo /path/to/contracts \   # SO4_CONTRACTS_REPO
+  --source so4-local --keeper so4-local \  # local stellar CLI keys
+  --long-code TWBTC --short-code TUSDC \
+  --soroban-endpoint http://localhost:8000/soroban/rpc \
+  --graphql-endpoint http://localhost:3000 \
+  --report .smoke/report.json
+```
+
+Useful flags: `--skip-referral` (skip the optional referral step),
+`--skip-indexer-restart` (assume the stack is already running with fresh config),
+and `--skip-indexer-check` (contracts-only run, no GraphQL assertions).
+
+### The report
+
+A JSON report is written to `.smoke/report.json` (override with `--report`). It
+contains the run mode, service reachability, deployment artifacts (market token,
+core contract IDs, faucet), each action's ledger/transaction-hash/key data, the
+GraphQL entity counts with pass/fail assertions, and — when something breaks — a
+top-level `failure` block naming the broken layer (`contracts-deploy`,
+`contract-action`, `indexer-runtime`, `graphql-query`, or `frontend-config`) so
+you know exactly where to look.
+
+### Rerunning cleanly
+
+The indexer persists entities in Postgres, so rerun against a clean database:
+
+```bash
+bun run --cwd apps/s03-indexer smoke:clean        # stop stack + drop DB + clear reports
+bun run --cwd apps/s03-indexer smoke:clean --keep-db
+bun run --cwd apps/s03-indexer smoke:clean --reports   # only clear .smoke reports
+```
+
 ## Generated Artifacts
 
 `project.ts`, `schema.graphql`, and the TypeScript mapping sources are the

apps/s03-indexer/package.json

@@ -8,6 +8,8 @@
     "codegen": "subql codegen",
     "sync:contracts:testnet": "bun run scripts/sync-contracts.ts --network testnet",
     "sync:contracts:local": "bun run scripts/sync-contracts.ts --network local",
+    "smoke:local": "bun run scripts/local-smoke.ts",
+    "smoke:clean": "bun run scripts/clean-local.ts",
     "start": "docker compose pull && docker compose up --remove-orphans",
     "start:docker": "bun run start",
     "dev": "bun run codegen && bun run build && bun run start",

apps/s03-indexer/project.ts

@@ -90,8 +90,10 @@ const project: StellarProject = {
   dataSources: [
     {
       kind: StellarDatasourceKind.Runtime,
-      /* Set this as a logical start block, it might be block 1 (genesis) or when your contract was deployed */
-      startBlock: 228206,
+      /* Set this as a logical start block, it might be block 1 (genesis) or when your
+         contract was deployed. Override with INDEXER_START_LEDGER for local runs, where
+         a fresh standalone network must backfill from an early ledger. */
+      startBlock: Number(process.env.INDEXER_START_LEDGER ?? 228206),
       mapping: {
         file: "./dist/index.js",
         handlers: indexedContractIds.map((contractId) => ({

apps/s03-indexer/scripts/clean-local.ts

@@ -0,0 +1,76 @@
+// Reset local indexer state so the smoke flow can be rerun cleanly.
+//
+// SubQuery persists indexed entities in the Postgres volume started by
+// docker-compose. Re-running the smoke flow against a stale database double-counts
+// entities and makes assertions ambiguous, so this script tears the stack down
+// (optionally removing the volume) and clears the local report.
+//
+// Usage:
+//   bun run scripts/clean-local.ts            # stop stack + drop DB volume + clear report
+//   bun run scripts/clean-local.ts --keep-db  # stop stack only, keep indexed data
+//   bun run scripts/clean-local.ts --reports  # only delete the .smoke report dir
+
+import { existsSync, rmSync } from "node:fs";
+import path from "node:path";
+
+import { parseArgs } from "./lib/config";
+import { hasBinary, run } from "./lib/process";
+
+const packageRoot = path.resolve(import.meta.dir, "..");
+const { bools } = parseArgs(process.argv.slice(2));
+
+const reportsOnly = bools.has("reports");
+const keepDb = bools.has("keepDb");
+
+function clearReports(): void {
+  const smokeDir = path.join(packageRoot, ".smoke");
+  if (existsSync(smokeDir)) {
+    rmSync(smokeDir, { recursive: true, force: true });
+    process.stdout.write(`Removed ${smokeDir}\n`);
+  } else {
+    process.stdout.write("No .smoke report directory to remove.\n");
+  }
+}
+
+function stopStack(): void {
+  if (!hasBinary("docker")) {
+    process.stdout.write("docker not found — skipping stack teardown.\n");
+    return;
+  }
+
+  const args = ["compose", "down", "--remove-orphans"];
+  if (!keepDb) {
+    args.push("--volumes");
+  }
+  run("docker", args, { cwd: packageRoot, stream: true });
+
+  // The Postgres volume is also bind-mounted at .data/postgres in this repo.
+  // The container writes those files as root, so a plain rm may be denied — fall
+  // back to removing them from inside a throwaway container.
+  if (!keepDb) {
+    const dataDir = path.join(packageRoot, ".data");
+    if (existsSync(dataDir)) {
+      try {
+        rmSync(dataDir, { recursive: true, force: true });
+        process.stdout.write(`Removed ${dataDir}\n`);
+      } catch {
+        run("docker", ["run", "--rm", "-v", `${packageRoot}:/work`, "alpine", "rm", "-rf", "/work/.data"], {
+          stream: true,
+        });
+        process.stdout.write(`Removed ${dataDir} (via container)\n`);
+      }
+    }
+  }
+}
+
+if (reportsOnly) {
+  clearReports();
+} else {
+  stopStack();
+  clearReports();
+  process.stdout.write(
+    keepDb
+      ? "Indexer stack stopped (database preserved).\n"
+      : "Indexer stack stopped and local database cleared. Safe to rerun smoke:local.\n",
+  );
+}

apps/s03-indexer/scripts/lib/config.ts

@@ -0,0 +1,138 @@
+// Resolve smoke-run configuration from CLI flags and environment variables.
+//
+// Precedence: CLI flag > environment variable > sensible local default. Every
+// value has a local-friendly default so a contributor can run the smoke flow with
+// zero flags against a standard stellar/quickstart standalone network.
+
+import { existsSync } from "node:fs";
+import path from "node:path";
+
+import type { SmokeConfig, SmokeMode } from "./types";
+
+export interface ParsedArgs {
+  flags: Record<string, string>;
+  bools: Set<string>;
+  positionals: string[];
+}
+
+export function parseArgs(argv: string[]): ParsedArgs {
+  const flags: Record<string, string> = {};
+  const bools = new Set<string>();
+  const positionals: string[] = [];
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (!arg.startsWith("--")) {
+      positionals.push(arg);
+      continue;
+    }
+
+    const [rawKey, inlineValue] = arg.slice(2).split("=", 2);
+    const key = rawKey.replace(/-([a-z])/g, (_, c: string) => c.toUpperCase());
+
+    if (inlineValue !== undefined) {
+      flags[key] = inlineValue;
+      continue;
+    }
+
+    const next = argv[i + 1];
+    if (next === undefined || next.startsWith("--")) {
+      bools.add(key);
+    } else {
+      flags[key] = next;
+      i += 1;
+    }
+  }
+
+  return { flags, bools, positionals };
+}
+
+const packageRoot = path.resolve(import.meta.dir, "..", "..");
+const repoRoot = path.resolve(packageRoot, "..", "..");
+
+function resolveContractsRepo(explicit?: string): string {
+  const configured =
+    explicit ??
+    process.env.SO4_CONTRACTS_REPO ??
+    process.env.CONTRACTS_REPO_PATH ??
+    process.env.CONTRACTS_REPO;
+
+  if (configured) {
+    return path.resolve(configured);
+  }
+
+  const candidates = [
+    path.resolve(repoRoot, "..", "contracts"),
+    path.resolve(packageRoot, "..", "contracts"),
+    "/home/sunny/zero/so4-market-project/contracts",
+  ];
+
+  return candidates.find((candidate) => existsSync(path.join(candidate, "scripts"))) ?? candidates[0];
+}
+
+function resolveMode(value: string | undefined): SmokeMode {
+  switch ((value ?? "auto").toLowerCase()) {
+    case "fresh":
+      return "fresh";
+    case "existing":
+      return "existing";
+    default:
+      return "auto";
+  }
+}
+
+export function buildConfig(argv: string[]): SmokeConfig {
+  const { flags, bools } = parseArgs(argv);
+
+  const network = (flags.network ?? process.env.SMOKE_NETWORK ?? "local").toLowerCase();
+  const source = flags.source ?? process.env.SMOKE_SOURCE ?? "so4-local";
+  const keeper = flags.keeper ?? process.env.SMOKE_KEEPER ?? source;
+
+  const sorobanRpcEndpoint =
+    flags.sorobanEndpoint ??
+    process.env.SMOKE_SOROBAN_ENDPOINT ??
+    process.env.SOROBAN_ENDPOINT ??
+    "http://localhost:8000/soroban/rpc";
+
+  const horizonEndpoint =
+    flags.horizonEndpoint ??
+    process.env.SMOKE_HORIZON_ENDPOINT ??
+    process.env.ENDPOINT ??
+    "http://localhost:8000";
+
+  const friendbotUrl =
+    flags.friendbotUrl ?? process.env.SMOKE_FRIENDBOT_URL ?? "http://localhost:8000/friendbot";
+
+  const graphqlEndpoint =
+    flags.graphqlEndpoint ??
+    process.env.SMOKE_GRAPHQL_ENDPOINT ??
+    process.env.INDEXER_GRAPHQL_URL ??
+    "http://localhost:3000";
+
+  const reportPath = path.resolve(
+    flags.report ?? process.env.SMOKE_REPORT ?? path.join(packageRoot, ".smoke", "report.json"),
+  );
+
+  return {
+    network,
+    contractsRepo: resolveContractsRepo(flags.contractsRepo),
+    source,
+    keeper,
+    longCode: flags.longCode ?? process.env.SMOKE_LONG_CODE ?? "TWBTC",
+    shortCode: flags.shortCode ?? process.env.SMOKE_SHORT_CODE ?? "TUSDC",
+    mode: resolveMode(flags.mode),
+    horizonEndpoint,
+    sorobanRpcEndpoint,
+    friendbotUrl,
+    graphqlEndpoint,
+    indexerLagTolerance: Number(flags.lagTolerance ?? process.env.SMOKE_LAG_TOLERANCE ?? 2),
+    waitTimeoutMs: Number(flags.waitTimeout ?? process.env.SMOKE_WAIT_TIMEOUT_MS ?? 180_000),
+    skipReferral: bools.has("skipReferral") || process.env.SMOKE_SKIP_REFERRAL === "1",
+    skipIndexerRestart:
+      bools.has("skipIndexerRestart") || process.env.SMOKE_SKIP_INDEXER_RESTART === "1",
+    skipIndexerCheck: bools.has("skipIndexerCheck") || process.env.SMOKE_SKIP_INDEXER_CHECK === "1",
+    reportPath,
+  };
+}
+
+export const SMOKE_PATHS = { packageRoot, repoRoot };