diff --git a/src/finalizer/utils/binance.ts b/src/finalizer/utils/binance.ts index 2a16cb5c75..256d39bcae 100644 --- a/src/finalizer/utils/binance.ts +++ b/src/finalizer/utils/binance.ts @@ -70,10 +70,12 @@ export async function binanceFinalizer( ]); const fromTimestamp = _fromTimestamp * 1_000; - const [_binanceDeposits, accountCoins] = await Promise.all([ + const [_binanceDeposits, _accountCoins] = await Promise.all([ getBinanceDeposits(binanceApi, fromTimestamp), - getAccountCoins(binanceApi), + getAccountCoins(binanceApi, logger), ]); + // Normalize null (API failure) to [] so the rest of the code can safely iterate; no coin will match. + const accountCoins = _accountCoins ?? []; // Remove any _binanceDeposits that are marked as related to a swap. The reason why we check "!== SWAP" instead of // "=== BRIDGE" is because we want this code to be backwards compatible with the existing inventory client logic which // does not yet tag deposits with this BRIDGE type. @@ -114,7 +116,7 @@ export async function binanceFinalizer( }); continue; } - let coinBalance = Number(coin.balance); + let coinBalance = coin ? Number(coin.balance) : 0; const l1Token = TOKEN_SYMBOLS_MAP[symbol].addresses[hubChainId]; const { decimals: l1Decimals } = getTokenInfo(EvmAddress.from(l1Token), hubChainId); const _withdrawals = await getBinanceWithdrawals(binanceApi, symbol, fromTimestamp); @@ -144,7 +146,7 @@ export async function binanceFinalizer( // @dev There are only two possible withdraw networks for the finalizer, Ethereum L1 or Binance Smart Chain "L2." Withdrawals to Ethereum can originate from any L2 but // must be finalized on L1. Withdrawals to Binance Smart Chain must originate from Ethereum L1. for (const withdrawNetwork of [BINANCE_NETWORKS[l2ChainId], BINANCE_NETWORKS[hubChainId]]) { - const networkLimits = coin.networkList.find((network) => network.name === withdrawNetwork); + const networkLimits = coin?.networkList.find((network) => network.name === withdrawNetwork); // Get both the amount deposited and ready to be finalized and the amount already withdrawn on L2. const finalizingOnL2 = withdrawNetwork === BINANCE_NETWORKS[l2ChainId]; const depositAmounts = depositsInScope @@ -178,7 +180,7 @@ export async function binanceFinalizer( }); // Additionally, binance imposes a minimum amount to withdraw. If the amount we want to finalize is less than the minimum, then // do not attempt to withdraw anything. Likewise, if the amount we want to withdraw is greater than the maximum, then warn and withdraw the maximum amount. - if (amountToFinalize >= Number(networkLimits.withdrawMax)) { + if (networkLimits && amountToFinalize >= Number(networkLimits.withdrawMax)) { logger.warn({ at: "BinanceFinalizer", message: `(X -> ${withdrawNetwork}) Cannot withdraw total amount ${amountToFinalize} ${symbol} since it is above the network limit ${networkLimits.withdrawMax}. Withdrawing the maximum amount instead.`, @@ -202,7 +204,7 @@ export async function binanceFinalizer( Number((coinBalance - creditedDepositAmount).toFixed(l1Decimals)), amountToFinalize ); - if (amountToFinalize >= Number(networkLimits.withdrawMin)) { + if (networkLimits && amountToFinalize >= Number(networkLimits.withdrawMin)) { // Lastly, we need to truncate the amount to withdraw to 6 decimal places. amountToFinalize = Math.floor(amountToFinalize * DECIMAL_PRECISION) / DECIMAL_PRECISION; // Balance from Binance is in 8 decimal places, so we need to truncate to 8 decimal places. diff --git a/src/rebalancer/adapters/binance.ts b/src/rebalancer/adapters/binance.ts index 6263438826..14bc169b36 100644 --- a/src/rebalancer/adapters/binance.ts +++ b/src/rebalancer/adapters/binance.ts @@ -43,6 +43,7 @@ interface SPOT_MARKET_META { isBuy: boolean; } +const HIGH_COST = toBNWei(1e6, 18); export class BinanceStablecoinSwapAdapter extends BaseAdapter { private binanceApiClient: Binance; @@ -91,21 +92,30 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { this.binanceApiClient = await getBinanceApiClient(process.env.BINANCE_API_BASE); + const routesWithCoins: RebalanceRoute[] = []; await forEachAsync(this.availableRoutes, async (route) => { const { sourceToken, destinationToken, sourceChain, destinationChain } = route; const [sourceCoin, destinationCoin] = await Promise.all([ this._getAccountCoins(sourceToken), this._getAccountCoins(destinationToken), ]); - assert(sourceCoin, `Source token ${sourceToken} not found in account coins`); - assert(destinationCoin, `Destination token ${destinationToken} not found in account coins`); + if (!sourceCoin || !destinationCoin) { + this.logger.warn({ + at: "BinanceStablecoinSwapAdapter.initialize", + message: "Skipping Binance route (account coins unavailable; API may be down).", + sourceToken, + destinationToken, + sourceChain, + destinationChain, + }); + return; + } const [sourceEntrypointNetwork, destinationEntrypointNetwork] = await Promise.all([ this._getEntrypointNetwork(sourceChain, sourceToken), this._getEntrypointNetwork(destinationChain, destinationToken), ]); const getIntermediateAdapter = (token: string) => (token === "USDT" ? this.oftAdapter : this.cctpAdapter); const getIntermediateAdapterName = (token: string) => (token === "USDT" ? "oft" : "cctp"); - // Validate that route can be supported using intermediate bridges to get to/from Arbitrum to access Binance. if (destinationEntrypointNetwork !== destinationChain) { const intermediateRoute = { ...route, @@ -150,7 +160,16 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { BINANCE_NETWORKS[destinationEntrypointNetwork] }", available networks: ${destinationCoin.networkList.map((network) => network.name).join(", ")}` ); + routesWithCoins.push(route); }); + this.availableRoutes = routesWithCoins; + if (this.availableRoutes.length === 0) { + this.logger.warn({ + at: "BinanceStablecoinSwapAdapter.initialize", + message: "No Binance routes available (account coins empty; API/proxy may be down).", + }); + } + this.initialized = true; } async updateRebalanceStatuses(): Promise { @@ -535,10 +554,27 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { const { sourceChain, sourceToken, destinationToken, destinationChain } = rebalanceRoute; const destinationCoin = await this._getAccountCoins(destinationToken); + if (!destinationCoin) { + this.logger.warn({ + at: "BinanceStablecoinSwapAdapter.initializeRebalance", + message: "Destination coin unavailable (Binance API may be down); skipping.", + destinationToken, + }); + return bnZero; + } const destinationEntrypointNetwork = await this._getEntrypointNetwork(destinationChain, destinationToken); const destinationBinanceNetwork = destinationCoin.networkList.find( (network) => network.name === BINANCE_NETWORKS[destinationEntrypointNetwork] ); + if (!destinationBinanceNetwork) { + this.logger.debug({ + at: "BinanceStablecoinSwapAdapter.initializeRebalance", + message: "No Binance network for destination; skipping.", + destinationToken, + destinationEntrypointNetwork, + }); + return bnZero; + } const { withdrawMin, withdrawMax } = destinationBinanceNetwork; // Make sure that the amount to transfer will be larger than the minimum withdrawal size after expected fees. @@ -645,13 +681,24 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { ).takerCommission; const tradeFee = toBNWei(tradeFeePct, 18).mul(amountToTransfer).div(toBNWei(100, 18)); const destinationCoin = await this._getAccountCoins(destinationToken); + if (!destinationCoin) { + this.logger.warn({ + at: "BinanceStablecoinSwapAdapter.getEstimatedCost", + message: "Destination coin unavailable (Binance API may be down); returning high cost.", + destinationToken, + }); + // API unavailable: return high cost so this route is not selected. + return HIGH_COST; + } const destinationEntrypointNetwork = await this._getEntrypointNetwork(destinationChain, destinationToken); - const destinationTokenInfo = this._getTokenInfo(destinationToken, destinationEntrypointNetwork); - const withdrawFee = toBNWei( - destinationCoin.networkList.find((network) => network.name === BINANCE_NETWORKS[destinationEntrypointNetwork]) - .withdrawFee, - destinationTokenInfo.decimals + const destinationNetwork = destinationCoin.networkList.find( + (network) => network.name === BINANCE_NETWORKS[destinationEntrypointNetwork] ); + if (!destinationNetwork) { + return HIGH_COST; + } + const destinationTokenInfo = this._getTokenInfo(destinationToken, destinationEntrypointNetwork); + const withdrawFee = toBNWei(destinationNetwork.withdrawFee, destinationTokenInfo.decimals); const amountConverter = this._getAmountConverter( destinationEntrypointNetwork, this._getTokenInfo(destinationToken, destinationEntrypointNetwork).address, @@ -773,7 +820,7 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { // PRIVATE BINANCE HELPER METHODS // //////////////////////////////////////////////////////////// - private async _getAccountCoins(symbol: string, skipCache = false): Promise { + private async _getAccountCoins(symbol: string, skipCache = false): Promise { const cacheKey = "binance-account-coins"; type ParsedAccountCoins = Awaited>; @@ -785,10 +832,16 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { } } if (!accountCoins) { - accountCoins = await getAccountCoins(this.binanceApiClient); - // Reset cache if we've fetched a new API response. - await this.redisCache.set(cacheKey, JSON.stringify(accountCoins)); // Use default TTL which is a long time as - // the entry for this coin is not expected to change frequently. + accountCoins = await getAccountCoins(this.binanceApiClient, this.logger); + // Cache only on success and non-empty so we don't cache "API down" (null) or empty. + if (accountCoins?.length > 0) { + await this.redisCache.set(cacheKey, JSON.stringify(accountCoins)); // Use default TTL which is a long time as + // the entry for this coin is not expected to change frequently. + } + } + // API failed (null) or no matching coin: treat as unavailable. + if (!accountCoins) { + return undefined; } const coin = accountCoins.find((coin) => coin.symbol === symbol); @@ -804,6 +857,9 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { return defaultBinanceNetwork; } const coin = await this._getAccountCoins(token); + if (!coin?.networkList?.length) { + return defaultBinanceNetwork; + } const coinHasNetwork = coin.networkList.find((network) => network.name === BINANCE_NETWORKS[chainId]); return coinHasNetwork ? chainId : defaultBinanceNetwork; } @@ -841,7 +897,7 @@ export class BinanceStablecoinSwapAdapter extends BaseAdapter { private async _getBinanceBalance(token: string): Promise { const coin = await this._getAccountCoins(token, true); // Skip cache so we load the balance fresh each time. - return Number(coin.balance); + return coin ? Number(coin.balance) : 0; } private async _getSymbol(sourceToken: string, destinationToken: string) { diff --git a/src/utils/BinanceUtils.ts b/src/utils/BinanceUtils.ts index 2958a08e63..ad0487ed2c 100644 --- a/src/utils/BinanceUtils.ts +++ b/src/utils/BinanceUtils.ts @@ -5,7 +5,7 @@ import Binance, { type Binance as BinanceApi, } from "binance-api-node"; import minimist from "minimist"; -import { getGckmsConfig, retrieveGckmsKeys, isDefined, assert, delay, CHAIN_IDs, getRedisCache } from "./"; +import { getGckmsConfig, retrieveGckmsKeys, isDefined, assert, delay, CHAIN_IDs, getRedisCache, winston } from "./"; // Store global promises on Gckms key retrieval actions so that we don't retrieve the same key multiple times. let binanceSecretKeyPromise = undefined; @@ -17,6 +17,36 @@ const KNOWN_BINANCE_ERROR_REASONS = [ "TypeError: fetch failed", ]; +// Empty, non-JSON, or proxy/HTTP errors from API. After retries, treat as "no data" so the bot continues instead of crashing. +const BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS = [ + "Unexpected ''", + "Unexpected end of JSON input", + "Unexpected token", + "502 Bad Gateway", + "503 Service Unavailable", + "504 Gateway Timeout", +]; + +/** Serialize errors so they log as readable text instead of "[object Object]". */ +export function stringifyBinanceError(err: unknown): string { + if (err instanceof Error) { + return err.message; + } + if (typeof err === "object" && err !== null) { + const o = err as Record; + if (typeof o.msg === "string") { + return o.msg; + } + if (typeof o.message === "string") { + return o.message; + } + if (typeof o.code !== "undefined" || Object.keys(o).length > 0) { + return JSON.stringify(err); + } + } + return String(err); +} + type WithdrawalQuota = { wdQuota: number; usedWdQuota: number; @@ -241,13 +271,20 @@ export async function getBinanceDeposits( try { depositHistory = await binanceApi.depositHistory({ startTime }); } catch (_err) { - const err = _err.toString(); - if (KNOWN_BINANCE_ERROR_REASONS.some((errorReason) => err.includes(errorReason)) && nRetries < maxRetries) { + const err = stringifyBinanceError(_err); + const stringError = _err.toString(); + + const isRetriable = KNOWN_BINANCE_ERROR_REASONS.some((r) => err.includes(r) || stringError.includes(r)); + if (isRetriable && nRetries < maxRetries) { const delaySeconds = 2 ** nRetries + Math.random(); await delay(delaySeconds); return getBinanceDeposits(binanceApi, startTime, ++nRetries, maxRetries); } - throw err; + // Empty/non-JSON response from API or proxy: treat as no deposits so relayer continues with zero pending. + if (BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS.some((r) => err.includes(r) || stringError.includes(r))) { + return []; + } + throw new Error(err); } return Object.values(depositHistory).map((deposit) => { return { @@ -275,13 +312,19 @@ export async function getBinanceWithdrawals( try { withdrawHistory = await binanceApi.withdrawHistory({ coin, startTime }); } catch (_err) { - const err = _err.toString(); - if (KNOWN_BINANCE_ERROR_REASONS.some((errorReason) => err.includes(errorReason)) && nRetries < maxRetries) { + const err = stringifyBinanceError(_err); + const stringError = _err.toString(); + const isRetriable = KNOWN_BINANCE_ERROR_REASONS.some((r) => err.includes(r) || stringError.includes(r)); + if (isRetriable && nRetries < maxRetries) { const delaySeconds = 2 ** nRetries + Math.random(); await delay(delaySeconds); return getBinanceWithdrawals(binanceApi, coin, startTime, ++nRetries, maxRetries); } - throw err; + // Empty/non-JSON response from API or proxy: treat as no withdrawals so relayer continues with zero pending. + if (BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS.some((r) => err.includes(r) || stringError.includes(r))) { + return []; + } + throw new Error(err); } return Object.values(withdrawHistory).map((withdrawal) => { return { @@ -301,25 +344,47 @@ export async function getBinanceWithdrawals( /** * The call to accountCoins returns an opaque `unknown` object with extraneous information. This function * parses the unknown into a readable object to be used by the finalizers. - * @returns A typed `AccountCoins` response. + * When the Binance API (or proxy) fails (e.g. 502), returns null so callers can treat as "no data" + * (e.g. default to zero balances) without confusing empty success [] with failure. + * @param binanceApi Binance API client. + * @param logger Optional logger; when provided, API failures are logged here instead of console. + * @returns A typed `AccountCoins` response, or null on any API error. */ -export async function getAccountCoins(binanceApi: BinanceApi): Promise { - const coins = Object.values(await binanceApi["accountCoins"]()); - return coins.map((coin) => { - const networkList = coin["networkList"]?.map((network) => { +export async function getAccountCoins( + binanceApi: BinanceApi, + logger: winston.Logger +): Promise { + try { + const coins = Object.values(await binanceApi["accountCoins"]()); + return coins.map((coin) => { + const networkList = coin["networkList"]?.map((network) => { + return { + name: network["network"], + coin: network["coin"], + withdrawMin: network["withdrawMin"], + withdrawMax: network["withdrawMax"], + withdrawFee: network["withdrawFee"], + contractAddress: network["contractAddress"], + } as Network; + }); return { - name: network["network"], - coin: network["coin"], - withdrawMin: network["withdrawMin"], - withdrawMax: network["withdrawMax"], - withdrawFee: network["withdrawFee"], - contractAddress: network["contractAddress"], - } as Network; + symbol: coin["coin"], + balance: coin["free"], + networkList, + } as Coin; }); - return { - symbol: coin["coin"], - balance: coin["free"], - networkList, - } as Coin; - }); + } catch (_err) { + const err = stringifyBinanceError(_err); + const stringError = _err.toString(); + const logMeta = { + at: "BinanceUtils#getAccountCoins", + message: "Binance accountCoins API failed; returning null.", + errorMessage: err, + }; + logger.warn(logMeta); + if (BINANCE_EMPTY_RESPONSE_ERROR_PATTERNS.some((r) => err.includes(r) || stringError.includes(r))) { + return null; + } + throw new Error(err); + } }