feat: collapse payments csv

constants/index.ts

@@ -47,7 +47,12 @@ export const RESPONSE_MESSAGES = {
   FAILED_TO_FETCH_PRICE_FROM_API_500: (day: string, ticker: string) => { return { statusCode: 500, message: `Failed to fetch ${ticker} price for day ${day}` } },
   MISSING_WS_AUTH_KEY_400: { statusCode: 400, message: 'Missing WS_AUTH_KEY environment variable' },
   MISSING_PRICE_FOR_TRANSACTION_400: { statusCode: 400, message: 'Missing price for transaction.' },
-  INVALID_PRICES_AMOUNT_FOR_TX_ON_CSV_CREATION_500: (pricesLenght: number) => { return { statusCode: 500, message: `Wrong number of prices for transactions group in CSV creation. Expected 1, got ${pricesLenght}.` } },
+  INVALID_PRICES_AMOUNT_FOR_TX_ON_CSV_CREATION_500: (pricesLenght: number) => {
+    return {
+      statusCode: 500,
+      message: `Got wrong number of prices for transactions group in CSV creation. Expected 1, got ${pricesLenght}.`
+    }
+  },
   INVALID_PRICE_STATE_400: { statusCode: 400, message: 'Missing expected quote price for transaction.' },
   COULD_NOT_GET_BLOCK_INFO_500: { statusCode: 500, message: "Couldn't get block info." },
   NETWORK_SLUG_NOT_PROVIDED_400: { statusCode: 400, message: "'networkSlug' not provided." },

package.json

@@ -18,7 +18,8 @@
     "docker": "./scripts/docker-exec-shortcuts.sh",
     "ci:integration:test": "yarn pretest && dotenv -e .env.test -- ts-node -O '{\"module\":\"commonjs\"}' node_modules/jest/bin/jest.js tests/integration-tests --forceExit",
     "tarDebug": "tar cf debug.tar logs/ paybutton-config.json .env*",
-    "updateAllPrices": "./scripts/update-all-prices.sh"
+    "updateAllPrices": "./scripts/update-all-prices.sh",
+    "updateAllPriceConnections": "./scripts/update-all-price-connections.sh"
   },
   "dependencies": {
     "@emotion/react": "^11.8.2",

pages/api/paybutton/download/transactions/[paybuttonId].ts

@@ -25,9 +25,9 @@ export default async (req: any, res: any): Promise<void> => {
     const userId = req.session.userId
     const user = await fetchUserProfileFromId(userId)
     const paybuttonId = req.query.paybuttonId as string
-    const networkTickerReq = (req.query.network as string).toUpperCase()
+    const networkTickerReq = req.query.network as string
 
-    const networkTicker = (networkTickerReq !== '' && isNetworkValid(networkTickerReq as NetworkTickersType)) ? networkTickerReq as NetworkTickersType : undefined
+    const networkTicker = (networkTickerReq !== '' && isNetworkValid(networkTickerReq as NetworkTickersType)) ? networkTickerReq.toUpperCase() as NetworkTickersType : undefined
     let quoteId: number
     if (req.query.currency === undefined || req.query.currency === '' || Number.isNaN(req.query.currency)) {
       quoteId = user.preferredCurrencyId

pages/api/payments/download/index.ts

@@ -31,9 +31,9 @@ export default async (req: any, res: any): Promise<void> => {
     const userReqTimezone = req.headers.timezone as string
     const userPreferredTimezone = user?.preferredTimezone
     const timezone = userPreferredTimezone !== '' ? userPreferredTimezone : userReqTimezone
-    const networkTickerReq = (req.query.network as string).toUpperCase()
+    const networkTickerReq = req.query.network as string
 
-    const networkTicker = (networkTickerReq !== '' && isNetworkValid(networkTickerReq as NetworkTickersType)) ? networkTickerReq as NetworkTickersType : undefined
+    const networkTicker = (networkTickerReq !== '' && isNetworkValid(networkTickerReq as NetworkTickersType)) ? networkTickerReq.toUpperCase() as NetworkTickersType : undefined
     res.setHeader('Content-Type', 'text/csv')
     let networkIdArray = Object.values(NETWORK_IDS)
     if (networkTicker !== undefined) {

scripts/update-all-price-connections.sh

@@ -0,0 +1,9 @@
+set -e
+
+echo This will affect the database. Are you sure of what you\'re doing? [y/N]
+read ans 
+if [[ "$ans" = "Y" || "$ans" = "y" ]]; then
+  JOBS_ENV=true dotenv -e .env -c -- ts-node -O '{"module":"commonjs"}' -r tsconfig-paths/register ./scripts/updateAllPriceConnections.ts
+else 
+  echo Exited.
+fi

scripts/updateAllPriceConnections.ts

@@ -0,0 +1,103 @@
+import { flattenTimestamp } from '../services/priceService'
+import prisma from 'prisma/clientInstance'
+import { connectTransactionsListToPrices } from 'services/transactionService'
+import { Transaction } from '@prisma/client'
+import moment from 'moment'
+import { exit } from 'process'
+
+async function misconnectTxs (txsIds: string[]): Promise<void> {
+  if (process.env.ENVIRONMENT === 'production') {
+    return
+  }
+  if (txsIds.length === 0) {
+    return
+  }
+  console.log('Misconnecting', txsIds.length, 'for testing purposes')
+  for (const txId of txsIds) {
+    const tx = await prisma.transaction.findUniqueOrThrow({
+      where: {
+        id: txId
+      }
+    })
+    const txDayTimestamp = flattenTimestamp(tx.timestamp)
+    let signal = 1
+    if (txDayTimestamp === flattenTimestamp(moment.utc().unix())) {
+      signal = -1
+    }
+    await prisma.transaction.update({
+      where: {
+        id: txId
+      },
+      data: {
+        timestamp: tx.timestamp + (signal * 86400)
+      }
+    })
+  }
+  console.log('Finished misconnecting txs')
+}
+
+async function fixMisconnectedTxs (): Promise<void> {
+  const total = await prisma.transaction.count()
+  const pageSize = 1000
+  let page = 0
+
+  await misconnectTxs([
+    // ADD TXS IDS HERE TO TEST IT
+  ])
+
+  console.log('Fixing misconnected txs...')
+  while (true) {
+    const txsToFix: Transaction[] = []
+    // Get txs page
+    const txs = await prisma.transaction.findMany({
+      orderBy: {
+        timestamp: 'asc'
+      },
+      include: {
+        prices: {
+          select: {
+            price: {
+              select: {
+                timestamp: true
+              }
+            }
+          }
+        }
+      },
+      skip: page * pageSize,
+      take: pageSize
+    })
+
+    const viewedCount = page * pageSize + txs.length
+
+    // Finish if empty
+    if (txs.length === 0) {
+      break
+    }
+
+    // Find txs with misaligned timestamps
+    txs.forEach(tx => {
+      const txFlattenedTimestamp = flattenTimestamp(tx.timestamp)
+      const txPriceTimestamps = tx.prices.map(p => p.price.timestamp)
+      if (txPriceTimestamps.filter(t => t !== txFlattenedTimestamp).length > 0) {
+        txsToFix.push(tx)
+      }
+    })
+    if (txsToFix.length !== 0) {
+      console.log(`[${viewedCount}/${total}] Fixing ${txsToFix.length} txs...`)
+      console.log('Tx ids:\n', txsToFix.map(t => t.id).join('\n'))
+      await connectTransactionsListToPrices(txsToFix)
+      console.log(`[${viewedCount}/${total}] Finished fixing ${txsToFix.length} txs.`)
+    }
+    console.log(`[${viewedCount}/${total}]`)
+    page++
+  }
+  console.log('FINISHED')
+  exit()
+}
+
+async function run (): Promise<void> {
+  await fixMisconnectedTxs()
+}
+
+void run()

services/chronikService.ts

@@ -9,7 +9,7 @@ import {
   createManyTransactions,
   deleteTransactions,
   fetchUnconfirmedTransactions,
-  createTransaction,
+  upsertTransaction,
   getSimplifiedTransactions,
   getSimplifiedTrasaction,
   connectAllTransactionsToPrices
@@ -417,7 +417,7 @@ export class ChronikBlockchainClient {
         const addressesWithTransactions = await this.getAddressesForTransaction(transaction)
         const inputAddresses = this.getSortedInputAddresses(transaction)
         for (const addressWithTransaction of addressesWithTransactions) {
-          const { created, tx } = await createTransaction(addressWithTransaction.transaction)
+          const { created, tx } = await upsertTransaction(addressWithTransaction.transaction)
           if (tx !== undefined) {
             const broadcastTxData = this.broadcastIncomingTx(addressWithTransaction.address.address, tx, inputAddresses)
             if (created) { // only execute trigger for newly added txs
@@ -470,7 +470,7 @@ export class ChronikBlockchainClient {
       const inputAddresses = this.getSortedInputAddresses(transaction)
 
       for (const addressWithTransaction of addressesWithTransactions) {
-        const { created, tx } = await createTransaction(addressWithTransaction.transaction)
+        const { created, tx } = await upsertTransaction(addressWithTransaction.transaction)
         if (tx !== undefined) {
           const broadcastTxData = this.broadcastIncomingTx(addressWithTransaction.address.address, tx, inputAddresses)
           if (created) { // only execute trigger for newly added txs

services/priceService.ts

@@ -7,7 +7,7 @@ import { validatePriceAPIUrlAndToken, validateNetworkTicker } from 'utils/valida
 import moment from 'moment'
 
 export function flattenTimestamp (timestamp: number): number {
-  const date = moment((timestamp * 1000))
+  const date = moment.utc((timestamp * 1000))
   const dateStart = date.startOf('day')
   return dateStart.unix()
 }

services/transactionService.ts

@@ -295,7 +295,7 @@ interface CreateTransactionResult {
   created: boolean
 }
 
-export async function createTransaction (
+export async function upsertTransaction (
   transactionData: Prisma.TransactionUncheckedCreateInput
 ): Promise<CreateTransactionResult> {
   if (transactionData.amount === new Prisma.Decimal(0)) { // out transactions

tests/unittests/transactionService.test.ts

@@ -71,7 +71,7 @@ describe('Create services', () => {
       ...mockedTransaction,
       addressId: mockedBCHAddress.id
     }
-    const result = await transactionService.createTransaction(
+    const result = await transactionService.upsertTransaction(
       argsTransaction
     )
     expect(result).toEqual({

utils/files.ts

@@ -139,16 +139,31 @@ export const collapseSmallPayments = (
     }
     const totalAmount = tempTxGroup.reduce((sum, p) => sum + Number(p.amount), 0)
     const totalValue = tempTxGroup.reduce((sum, p) => sum + Number(getTransactionValue(p)[currency]), 0)
-    const uniquePrices = new Set()
+    const uniquePrices: Set<number> = new Set()
+    const quoteId = QUOTE_IDS[currency.toUpperCase()]
     tempTxGroup
       .forEach(tx => {
-        const price = tx.prices.find(p => p.price.quoteId === QUOTE_IDS[currency.toUpperCase()])!.price.value
+        const price = tx.prices.find(p => p.price.quoteId === quoteId)!.price.value
         uniquePrices.add(Number(price))
       })
     if (uniquePrices.size !== 1) {
-      throw new Error(RESPONSE_MESSAGES.INVALID_PRICES_AMOUNT_FOR_TX_ON_CSV_CREATION_500(uniquePrices.size).message)
+      if (uniquePrices.size > 1) {
+        const nonUniquePrices = [...uniquePrices]
+        const txsForPrice: Record<number, string[]> = {}
+        nonUniquePrices.forEach(nonUniquePrice => {
+          txsForPrice[nonUniquePrice] = tempTxGroup.filter(tx => nonUniquePrice === tx.prices.find(p => p.price.quoteId === quoteId)!.price.value.toNumber()).map(tx => tx.id)
+        })
+        console.error('ERROR WHEN TRYING TO COLLAPSE TXS INTO DIFFERENT PRICES:', { txsForPrice, nonUniquePrices })
+      } else {
+        console.error('ERROR WHEN TRYING TO COLLAPSE TXS INTO DIFFERENT PRICES, NO PRICES FOR GROUP KEY', { groupKey })
+      }
+
+      throw new Error(
+        RESPONSE_MESSAGES
+          .INVALID_PRICES_AMOUNT_FOR_TX_ON_CSV_CREATION_500(tempTxGroup.length).message
+      )
     }
-    const rate = uniquePrices.values().next().value
+    const rate = new Prisma.Decimal(uniquePrices.values().next().value as number)
     const buttonName = tempTxGroup[0].address.paybuttons[0].paybutton.name
     const notes = `${buttonName} - ${tempTxGroup.length.toString()} transactions`