chore: Make BinanceAPI more robust

src/finalizer/utils/binance.ts

@@ -70,10 +70,12 @@ export async function binanceFinalizer(
   ]);
   const fromTimestamp = _fromTimestamp * 1_000;
 
-  const [_binanceDeposits, accountCoins] = await Promise.all([
+  const [_binanceDeposits, _accountCoins] = await Promise.all([
     getBinanceDeposits(binanceApi, fromTimestamp),
-    getAccountCoins(binanceApi),
+    getAccountCoins(binanceApi, logger),
   ]);
+  // Normalize null (API failure) to [] so the rest of the code can safely iterate; no coin will match.
+  const accountCoins = _accountCoins ?? [];
   // Remove any _binanceDeposits that are marked as related to a swap. The reason why we check "!== SWAP" instead of
   // "=== BRIDGE" is because we want this code to be backwards compatible with the existing inventory client logic which
   // does not yet tag deposits with this BRIDGE type.
@@ -114,7 +116,7 @@ export async function binanceFinalizer(
         });
         continue;
       }
-      let coinBalance = Number(coin.balance);
+      let coinBalance = coin ? Number(coin.balance) : 0;
       const l1Token = TOKEN_SYMBOLS_MAP[symbol].addresses[hubChainId];
       const { decimals: l1Decimals } = getTokenInfo(EvmAddress.from(l1Token), hubChainId);
       const _withdrawals = await getBinanceWithdrawals(binanceApi, symbol, fromTimestamp);
@@ -144,7 +146,7 @@ export async function binanceFinalizer(
       // @dev There are only two possible withdraw networks for the finalizer, Ethereum L1 or Binance Smart Chain "L2." Withdrawals to Ethereum can originate from any L2 but
       // must be finalized on L1. Withdrawals to Binance Smart Chain must originate from Ethereum L1.
       for (const withdrawNetwork of [BINANCE_NETWORKS[l2ChainId], BINANCE_NETWORKS[hubChainId]]) {
-        const networkLimits = coin.networkList.find((network) => network.name === withdrawNetwork);
+        const networkLimits = coin?.networkList.find((network) => network.name === withdrawNetwork);
         // Get both the amount deposited and ready to be finalized and the amount already withdrawn on L2.
         const finalizingOnL2 = withdrawNetwork === BINANCE_NETWORKS[l2ChainId];
         const depositAmounts = depositsInScope
@@ -178,7 +180,7 @@ export async function binanceFinalizer(
         });
         // Additionally, binance imposes a minimum amount to withdraw. If the amount we want to finalize is less than the minimum, then
         // do not attempt to withdraw anything. Likewise, if the amount we want to withdraw is greater than the maximum, then warn and withdraw the maximum amount.
-        if (amountToFinalize >= Number(networkLimits.withdrawMax)) {
+        if (networkLimits && amountToFinalize >= Number(networkLimits.withdrawMax)) {
           logger.warn({
             at: "BinanceFinalizer",
             message: `(X -> ${withdrawNetwork}) Cannot withdraw total amount ${amountToFinalize} ${symbol} since it is above the network limit ${networkLimits.withdrawMax}. Withdrawing the maximum amount instead.`,
@@ -202,7 +204,7 @@ export async function binanceFinalizer(
           Number((coinBalance - creditedDepositAmount).toFixed(l1Decimals)),
           amountToFinalize
         );
-        if (amountToFinalize >= Number(networkLimits.withdrawMin)) {
+        if (networkLimits && amountToFinalize >= Number(networkLimits.withdrawMin)) {
           // Lastly, we need to truncate the amount to withdraw to 6 decimal places.
           amountToFinalize = Math.floor(amountToFinalize * DECIMAL_PRECISION) / DECIMAL_PRECISION;
           // Balance from Binance is in 8 decimal places, so we need to truncate to 8 decimal places.

src/rebalancer/adapters/binance.ts

@@ -43,6 +43,7 @@ interface SPOT_MARKET_META {
   isBuy: boolean;
 }
 
+const HIGH_COST = toBNWei(1e6, 18);
 export class BinanceStablecoinSwapAdapter extends BaseAdapter {
   private binanceApiClient: Binance;
 
@@ -91,21 +92,30 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
 
     this.binanceApiClient = await getBinanceApiClient(process.env.BINANCE_API_BASE);
 
+    const routesWithCoins: RebalanceRoute[] = [];
     await forEachAsync(this.availableRoutes, async (route) => {
       const { sourceToken, destinationToken, sourceChain, destinationChain } = route;
       const [sourceCoin, destinationCoin] = await Promise.all([
         this._getAccountCoins(sourceToken),
         this._getAccountCoins(destinationToken),
       ]);
-      assert(sourceCoin, `Source token ${sourceToken} not found in account coins`);
-      assert(destinationCoin, `Destination token ${destinationToken} not found in account coins`);
+      if (!sourceCoin || !destinationCoin) {
+        this.logger.warn({
+          at: "BinanceStablecoinSwapAdapter.initialize",
+          message: "Skipping Binance route (account coins unavailable; API may be down).",
+          sourceToken,
+          destinationToken,
+          sourceChain,
+          destinationChain,
+        });
+        return;
+      }
       const [sourceEntrypointNetwork, destinationEntrypointNetwork] = await Promise.all([
         this._getEntrypointNetwork(sourceChain, sourceToken),
         this._getEntrypointNetwork(destinationChain, destinationToken),
       ]);
       const getIntermediateAdapter = (token: string) => (token === "USDT" ? this.oftAdapter : this.cctpAdapter);
       const getIntermediateAdapterName = (token: string) => (token === "USDT" ? "oft" : "cctp");
-      // Validate that route can be supported using intermediate bridges to get to/from Arbitrum to access Binance.
       if (destinationEntrypointNetwork !== destinationChain) {
         const intermediateRoute = {
           ...route,
@@ -150,7 +160,16 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
           BINANCE_NETWORKS[destinationEntrypointNetwork]
         }", available networks: ${destinationCoin.networkList.map((network) => network.name).join(", ")}`
       );
+      routesWithCoins.push(route);
     });
+    this.availableRoutes = routesWithCoins;
+    if (this.availableRoutes.length === 0) {
+      this.logger.warn({
+        at: "BinanceStablecoinSwapAdapter.initialize",
+        message: "No Binance routes available (account coins empty; API/proxy may be down).",
+      });
+    }
+    this.initialized = true;
   }
 
   async updateRebalanceStatuses(): Promise<void> {
@@ -535,10 +554,27 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
     const { sourceChain, sourceToken, destinationToken, destinationChain } = rebalanceRoute;
 
     const destinationCoin = await this._getAccountCoins(destinationToken);
+    if (!destinationCoin) {
+      this.logger.warn({
+        at: "BinanceStablecoinSwapAdapter.initializeRebalance",
+        message: "Destination coin unavailable (Binance API may be down); skipping.",
+        destinationToken,
+      });
+      return bnZero;
+    }
     const destinationEntrypointNetwork = await this._getEntrypointNetwork(destinationChain, destinationToken);
     const destinationBinanceNetwork = destinationCoin.networkList.find(
       (network) => network.name === BINANCE_NETWORKS[destinationEntrypointNetwork]
     );
+    if (!destinationBinanceNetwork) {
+      this.logger.debug({
+        at: "BinanceStablecoinSwapAdapter.initializeRebalance",
+        message: "No Binance network for destination; skipping.",
+        destinationToken,
+        destinationEntrypointNetwork,
+      });
+      return bnZero;
+    }
     const { withdrawMin, withdrawMax } = destinationBinanceNetwork;
 
     // Make sure that the amount to transfer will be larger than the minimum withdrawal size after expected fees.
@@ -645,13 +681,24 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
     ).takerCommission;
     const tradeFee = toBNWei(tradeFeePct, 18).mul(amountToTransfer).div(toBNWei(100, 18));
     const destinationCoin = await this._getAccountCoins(destinationToken);
+    if (!destinationCoin) {
+      this.logger.warn({
+        at: "BinanceStablecoinSwapAdapter.getEstimatedCost",
+        message: "Destination coin unavailable (Binance API may be down); returning high cost.",
+        destinationToken,
+      });
+      // API unavailable: return high cost so this route is not selected.
+      return HIGH_COST;
+    }
     const destinationEntrypointNetwork = await this._getEntrypointNetwork(destinationChain, destinationToken);
-    const destinationTokenInfo = this._getTokenInfo(destinationToken, destinationEntrypointNetwork);
-    const withdrawFee = toBNWei(
-      destinationCoin.networkList.find((network) => network.name === BINANCE_NETWORKS[destinationEntrypointNetwork])
-        .withdrawFee,
-      destinationTokenInfo.decimals
+    const destinationNetwork = destinationCoin.networkList.find(
+      (network) => network.name === BINANCE_NETWORKS[destinationEntrypointNetwork]
     );
+    if (!destinationNetwork) {
+      return HIGH_COST;
+    }
+    const destinationTokenInfo = this._getTokenInfo(destinationToken, destinationEntrypointNetwork);
+    const withdrawFee = toBNWei(destinationNetwork.withdrawFee, destinationTokenInfo.decimals);
     const amountConverter = this._getAmountConverter(
       destinationEntrypointNetwork,
       this._getTokenInfo(destinationToken, destinationEntrypointNetwork).address,
@@ -773,7 +820,7 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
   // PRIVATE BINANCE HELPER METHODS
   // ////////////////////////////////////////////////////////////
 
-  private async _getAccountCoins(symbol: string, skipCache = false): Promise<Coin> {
+  private async _getAccountCoins(symbol: string, skipCache = false): Promise<Coin | undefined> {
     const cacheKey = "binance-account-coins";
 
     type ParsedAccountCoins = Awaited<ReturnType<typeof getAccountCoins>>;
@@ -785,10 +832,16 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
       }
     }
     if (!accountCoins) {
-      accountCoins = await getAccountCoins(this.binanceApiClient);
-      // Reset cache if we've fetched a new API response.
-      await this.redisCache.set(cacheKey, JSON.stringify(accountCoins)); // Use default TTL which is a long time as
-      // the entry for this coin is not expected to change frequently.
+      accountCoins = await getAccountCoins(this.binanceApiClient, this.logger);
+      // Cache only on success and non-empty so we don't cache "API down" (null) or empty.
+      if (accountCoins?.length > 0) {
+        await this.redisCache.set(cacheKey, JSON.stringify(accountCoins)); // Use default TTL which is a long time as
+        // the entry for this coin is not expected to change frequently.
+      }
+    }
+    // API failed (null) or no matching coin: treat as unavailable.
+    if (!accountCoins) {
+      return undefined;
     }
 
     const coin = accountCoins.find((coin) => coin.symbol === symbol);
@@ -804,6 +857,9 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
       return defaultBinanceNetwork;
     }
     const coin = await this._getAccountCoins(token);
+    if (!coin?.networkList?.length) {
+      return defaultBinanceNetwork;
+    }
     const coinHasNetwork = coin.networkList.find((network) => network.name === BINANCE_NETWORKS[chainId]);
     return coinHasNetwork ? chainId : defaultBinanceNetwork;
   }
@@ -841,7 +897,7 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter {
 
   private async _getBinanceBalance(token: string): Promise<number> {
     const coin = await this._getAccountCoins(token, true); // Skip cache so we load the balance fresh each time.
-    return Number(coin.balance);
+    return coin ? Number(coin.balance) : 0;
   }
 
   private async _getSymbol(sourceToken: string, destinationToken: string) {

src/utils/BinanceUtils.ts

@@ -5,7 +5,7 @@ import Binance, {
   type Binance as BinanceApi,
 } from "binance-api-node";
 import minimist from "minimist";
-import { getGckmsConfig, retrieveGckmsKeys, isDefined, assert, delay, CHAIN_IDs, getRedisCache } from "./";
+import { getGckmsConfig, retrieveGckmsKeys, isDefined, assert, delay, CHAIN_IDs, getRedisCache, winston } from "./";
 
 // Store global promises on Gckms key retrieval actions so that we don't retrieve the same key multiple times.
 let binanceSecretKeyPromise = undefined;
@@ -17,6 +17,36 @@ const KNOWN_BINANCE_ERROR_REASONS = [
   "TypeError: fetch failed",
 ];
 
+// Empty, non-JSON, or proxy/HTTP errors from API. After retries, treat as "no data" so the bot continues instead of crashing.
+const BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS = [
+  "Unexpected ''",
+  "Unexpected end of JSON input",
+  "Unexpected token",
+  "502 Bad Gateway",
+  "503 Service Unavailable",
+  "504 Gateway Timeout",
+];
+
+/** Serialize errors so they log as readable text instead of "[object Object]". */
+export function stringifyBinanceError(err: unknown): string {
+  if (err instanceof Error) {
+    return err.message;
+  }
+  if (typeof err === "object" && err !== null) {
+    const o = err as Record<string, unknown>;
+    if (typeof o.msg === "string") {
+      return o.msg;
+    }
+    if (typeof o.message === "string") {
+      return o.message;
+    }
+    if (typeof o.code !== "undefined" || Object.keys(o).length > 0) {
+      return JSON.stringify(err);
+    }
+  }
+  return String(err);
+}
+
 type WithdrawalQuota = {
   wdQuota: number;
   usedWdQuota: number;
@@ -241,13 +271,20 @@ export async function getBinanceDeposits(
   try {
     depositHistory = await binanceApi.depositHistory({ startTime });
   } catch (_err) {
-    const err = _err.toString();
-    if (KNOWN_BINANCE_ERROR_REASONS.some((errorReason) => err.includes(errorReason)) && nRetries < maxRetries) {
+    const err = stringifyBinanceError(_err);
+    const stringError = _err.toString();
+
+    const isRetriable = KNOWN_BINANCE_ERROR_REASONS.some((r) => err.includes(r) || stringError.includes(r));
+    if (isRetriable && nRetries < maxRetries) {
       const delaySeconds = 2 ** nRetries + Math.random();
       await delay(delaySeconds);
       return getBinanceDeposits(binanceApi, startTime, ++nRetries, maxRetries);
     }
-    throw err;
+    // Empty/non-JSON response from API or proxy: treat as no deposits so relayer continues with zero pending.
+    if (BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS.some((r) => err.includes(r) || stringError.includes(r))) {
+      return [];
+    }
+    throw new Error(err);
   }
   return Object.values(depositHistory).map((deposit) => {
     return {
@@ -275,13 +312,19 @@ export async function getBinanceWithdrawals(
   try {
     withdrawHistory = await binanceApi.withdrawHistory({ coin, startTime });
   } catch (_err) {
-    const err = _err.toString();
-    if (KNOWN_BINANCE_ERROR_REASONS.some((errorReason) => err.includes(errorReason)) && nRetries < maxRetries) {
+    const err = stringifyBinanceError(_err);
+    const stringError = _err.toString();
+    const isRetriable = KNOWN_BINANCE_ERROR_REASONS.some((r) => err.includes(r) || stringError.includes(r));
+    if (isRetriable && nRetries < maxRetries) {
       const delaySeconds = 2 ** nRetries + Math.random();
       await delay(delaySeconds);
       return getBinanceWithdrawals(binanceApi, coin, startTime, ++nRetries, maxRetries);
     }
-    throw err;
+    // Empty/non-JSON response from API or proxy: treat as no withdrawals so relayer continues with zero pending.
+    if (BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS.some((r) => err.includes(r) || stringError.includes(r))) {
+      return [];
+    }
+    throw new Error(err);
   }
   return Object.values(withdrawHistory).map((withdrawal) => {
     return {
@@ -301,25 +344,47 @@ export async function getBinanceWithdrawals(
 /**
  * The call to accountCoins returns an opaque `unknown` object with extraneous information. This function
  * parses the unknown into a readable object to be used by the finalizers.
- * @returns A typed `AccountCoins` response.
+ * When the Binance API (or proxy) fails (e.g. 502), returns null so callers can treat as "no data"
+ * (e.g. default to zero balances) without confusing empty success [] with failure.
+ * @param binanceApi Binance API client.
+ * @param logger Optional logger; when provided, API failures are logged here instead of console.
+ * @returns A typed `AccountCoins` response, or null on any API error.
  */
-export async function getAccountCoins(binanceApi: BinanceApi): Promise<ParsedAccountCoins> {
-  const coins = Object.values(await binanceApi["accountCoins"]());
-  return coins.map((coin) => {
-    const networkList = coin["networkList"]?.map((network) => {
+export async function getAccountCoins(
+  binanceApi: BinanceApi,
+  logger: winston.Logger
+): Promise<ParsedAccountCoins | null> {
+  try {
+    const coins = Object.values(await binanceApi["accountCoins"]());
+    return coins.map((coin) => {
+      const networkList = coin["networkList"]?.map((network) => {
+        return {
+          name: network["network"],
+          coin: network["coin"],
+          withdrawMin: network["withdrawMin"],
+          withdrawMax: network["withdrawMax"],
+          withdrawFee: network["withdrawFee"],
+          contractAddress: network["contractAddress"],
+        } as Network;
+      });
       return {
-        name: network["network"],
-        coin: network["coin"],
-        withdrawMin: network["withdrawMin"],
-        withdrawMax: network["withdrawMax"],
-        withdrawFee: network["withdrawFee"],
-        contractAddress: network["contractAddress"],
-      } as Network;
+        symbol: coin["coin"],
+        balance: coin["free"],
+        networkList,
+      } as Coin;
     });
-    return {
-      symbol: coin["coin"],
-      balance: coin["free"],
-      networkList,
-    } as Coin;
-  });
+  } catch (_err) {
+    const err = stringifyBinanceError(_err);
+    const stringError = _err.toString();
+    const logMeta = {
+      at: "BinanceUtils#getAccountCoins",
+      message: "Binance accountCoins API failed; returning null.",
+      errorMessage: err,
+    };
+    logger.warn(logMeta);
+    if (BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS.some((r) => err.includes(r) || stringError.includes(r))) {
+      return null;
+    }
+    throw new Error(err);
+  }
 }