WIP: Code Refactoring

.env.example

@@ -20,8 +20,10 @@ LATEST_NFT_LIST_URL=https://t2crnfts.eth.link
 
 PROVIDER_URL=https://mainnet.infura.io/v3/<api-key>
 IPFS_GATEWAY=https://ipfs.kleros.io
+ESTUARY_GATEWAY=https://api.estuary.tech
+ESTUARY_BASE_URL=https://shuttle-7.estuary.tech/
 WALLET_KEY=
-
+ESTUARY_API_KEY=
 # Optional. If not provided, 300 seconds (5 minutes will be used).
 POLL_PERIOD_SECONDS=
 

package.json

@@ -17,7 +17,7 @@
   },
   "jest": {
     "transform": {
-      ".(ts|tsx)": "<rootDir>/node_modules/ts-jest/preprocessor.js"
+      ".(ts|tsx)": "ts-jest"
     },
     "testRegex": "(/__tests__/.*|\\.(test|spec))\\.(ts|tsx|js)$",
     "moduleFileExtensions": [
@@ -42,43 +42,44 @@
     "@commitlint/config-conventional": "^9.0.1",
     "@secretlint/secretlint-rule-preset-recommend": "^2.1.0",
     "@secretlint/secretlint-rule-secp256k1-privatekey": "^2.1.0",
-    "@types/jest": "^26.0.4",
-    "@typescript-eslint/eslint-plugin": "^3.6.0",
-    "@typescript-eslint/parser": "^3.6.0",
+    "@types/jest": "^29.2.2",
+    "@typescript-eslint/eslint-plugin": "^5.41.0",
+    "@typescript-eslint/parser": "^5.41.0",
     "concurrently": "^5.2.0",
     "eslint": "^7.4.0",
     "eslint-config-prettier": "^6.11.0",
     "eslint-plugin-prettier": "^3.1.4",
     "husky": "^4.2.5",
-    "jest": "^26.1.0",
+    "jest": "^29.3.1",
     "nodemon": "^2.0.4",
     "prettier": "^2.0.5",
     "rimraf": "^3.0.2",
     "secretlint": "^2.1.0",
     "standard-version": "^8.0.1",
-    "ts-jest": "^26.1.2",
-    "ts-node": "^9.1.1",
+    "ts-jest": "^29.0.3",
+    "ts-node": "^10.9.1",
     "ts-node-dev": "^1.1.1",
-    "typescript": "^3.9.6"
+    "typescript": "^4.8.4"
   },
   "dependencies": {
     "@0xsequence/collectible-lists": "^1.1.0",
     "@ensdomains/resolver": "^0.2.4",
     "@kleros/gtcr-sdk": "^1.8.0",
     "@pinata/sdk": "^1.1.10",
     "@types/lodash": "^4.14.157",
-    "@types/node": "^14.0.23",
-    "@types/node-fetch": "^2.5.7",
+    "@types/node": "^18.11.9",
     "@types/sharp": "^0.25.0",
-    "@uniswap/token-lists": "^1.0.0-beta.17",
-    "ajv": "^6.12.3",
+    "@uniswap/token-lists": "^1.0.0-beta.30",
+    "ajv": "^8.11.0",
+    "ajv-formats": "^2.1.1",
+    "axios": "^1.1.3",
     "content-hash": "^2.5.2",
     "dotenv-safe": "^8.2.0",
     "eth-ens-namehash": "^2.0.8",
     "ethers": "^5.0.26",
+    "form-data": "^4.0.0",
     "ipfs-only-hash": "^2.0.1",
     "lodash": "^4.17.19",
-    "node-fetch": "^2.6.0",
     "sharp": "^0.25.4"
   },
   "volta": {

src/api/estuary-api.ts

@@ -0,0 +1,74 @@
+import dotenv from 'dotenv'
+dotenv.config()
+import axios, { AxiosRequestConfig } from 'axios'
+import FormData from 'form-data'
+import {
+  AxiosConfigCustom,
+  AxiosErrorCustom,
+  EstuaryUploadResponse,
+} from './interfaces'
+
+const upload = axios.create({
+  baseURL: `${process.env.ESTUARY_BASE_URL}`,
+})
+
+upload.interceptors.request.use(
+  (config: AxiosRequestConfig) => {
+    config.headers = config.headers ?? {}
+    config.headers['Authorization'] = `Bearer ${process.env.ESTUARY_API_KEY}`
+    return config
+  },
+  (error) => {
+    return Promise.reject(error)
+  },
+)
+
+upload.interceptors.response.use(undefined, async (error: AxiosErrorCustom) => {
+  const { config, message } = error
+
+  if (!config || !config.retryCount) {
+    return Promise.reject(error).then(() =>
+      console.error('Maximum retry count for request has been reached'),
+    )
+  }
+
+  if (!(message.includes('timeout') || message.includes('Network error'))) {
+    return Promise.reject(error)
+  }
+
+  config.retryCount -= 1
+  const delayRetryRequest = new Promise<void>((resolve) => {
+    setTimeout(() => {
+      console.warn('Api call failed. Retrying the request to', config.url)
+      console.log(`${config.retryCount} retry count left`)
+      resolve()
+    }, config.retryDelay || 1000)
+  })
+  await delayRetryRequest
+  return await upload(config)
+})
+
+const uploadFile = async (
+  fileName: string,
+  data: ArrayBuffer,
+  config?: AxiosConfigCustom,
+): Promise<EstuaryUploadResponse> => {
+  const formData = new FormData()
+  formData.append('data', data, fileName)
+  let response: any
+  try {
+    response = await upload.post(`content/add`, formData, <AxiosConfigCustom>{
+      headers: formData.getHeaders(),
+      ...config,
+    })
+  } catch (error) {
+    if (axios.isAxiosError(error) && error.response) {
+      console.log(error.response.data)
+    } else {
+      console.log('Unexpected Error', error)
+    }
+  }
+  return response.data
+}
+
+export default { uploadFile }

src/api/interfaces.ts

@@ -0,0 +1,16 @@
+import { AxiosError, AxiosRequestConfig } from 'axios'
+
+export interface AxiosConfigCustom extends AxiosRequestConfig {
+  retryCount?: number
+  retryDelay?: number
+}
+
+export interface AxiosErrorCustom extends AxiosError {
+  config: AxiosConfigCustom
+}
+export interface EstuaryUploadResponse {
+  cid: string
+  estuaryId: number
+  retrieval_url: string
+  provideres: string[]
+}

src/erc20.ts

@@ -1,47 +1,19 @@
 import { schema, TokenInfo, TokenList, Version } from '@uniswap/token-lists'
 import { isEqual } from 'lodash'
 import { ethers } from 'ethers'
-import Ajv from 'ajv'
-import namehash from 'eth-ens-namehash'
-import { encode } from 'content-hash'
-import fetch from 'node-fetch'
-import { TextEncoder } from 'util'
-import { abi as resolverABI } from '@ensdomains/resolver/build/contracts/Resolver.json'
 
-import { ipfsPublish } from './utils'
-import { getNewErc20ListVersion } from './versioning'
+import { getListVersion } from './versioning'
+import { tokenListUtils } from './utils'
 
-const ajv = new Ajv({
-  allErrors: true,
-  format: 'full',
-  $data: true,
-  verbose: true,
-})
-
-const validator = ajv.compile(schema)
-
-export default async function checkPublishErc20(
+export default async function updateAndValidateErc20List(
   latestTokens: TokenInfo[],
-  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any
-  pinata: any,
-  provider: ethers.providers.JsonRpcProvider,
   listURL = '',
-  ensListName = '',
   listName: string,
-  fileName: string,
-): Promise<void> {
+): Promise<TokenList> {
   const timestamp = new Date().toISOString()
   console.info(`Pulling latest list from ${listURL}`)
 
-  let previousList: TokenList = await (
-    await fetch(listURL, {
-      method: 'GET',
-      headers: {
-        pragma: 'no-cache',
-        'cache-control': 'no-cache',
-      },
-    })
-  ).json()
+  let previousList: TokenList = await tokenListUtils.fetch(listURL)
   console.info('Done.')
 
   // Ensure addresses of the fetched lists are normalized.
@@ -62,120 +34,54 @@ export default async function checkPublishErc20(
   )
   const invalidTokens: TokenInfo[] = []
   const validatedTokens = latestTokens
-    .filter((t) => {
-      if (!nameRe.test(t.name)) {
-        console.warn(` ${t.name} failed name regex test, dropping it.`)
-        invalidTokens.push(t)
+    .filter((token) => {
+      if (!nameRe.test(token.name)) {
+        console.warn(` ${token.name} failed name regex test, dropping it.`)
+        invalidTokens.push(token)
         return false
       }
-      if (t.name.length > 40) {
-        console.warn(` ${t.name} longer than 40 chars, dropping it.`)
-        console.warn(` Address: ${t.address}`)
-        invalidTokens.push(t)
+      if (token.name.length > 40) {
+        console.warn(` ${token.name} longer than 40 chars, dropping it.`)
+        console.warn(` Address: ${token.address}`)
+        invalidTokens.push(token)
         return false
       }
       return true
     })
-    .filter((t) => {
-      if (!tickerRe.test(t.symbol)) {
-        console.warn(` ${t.symbol} failed ticker regex test, dropping it.`)
-        invalidTokens.push(t)
+    .filter((token) => {
+      if (!tickerRe.test(token.symbol)) {
+        console.warn(` ${token.symbol} failed ticker regex test, dropping it.`)
+        invalidTokens.push(token)
         return false
       }
       return true
     })
 
-  const version: Version = getNewErc20ListVersion(
+  const version: Version = getListVersion(
     previousList,
     latestTokens,
     invalidTokens,
   )
-
   if (isEqual(previousList.version, version)) {
     // List did not change. Stop here.
     console.info('List did not change.')
     console.info(
       'Latest list can be found at',
       process.env.LATEST_TOKEN_LIST_URL,
     )
-    return
+    return previousList
   } else {
     console.info('List changed.')
   }
 
   // Build the JSON object.
-  const tokenList: TokenList = {
-    name: `Kleros ${listName}`,
-    logoURI: 'ipfs://QmRYXpD8X4sQZwA1E4SJvEjVZpEK1WtSrTqzTWvGpZVDwa',
-    keywords: ['t2cr', 'kleros', 'list'],
+  const tokenList: TokenList = tokenListUtils.generate(
+    listName,
     timestamp,
     version,
-    tags: {
-      erc20: {
-        name: 'ERC20',
-        description: `This token is verified to be ERC20 thus there should not be incompatibility issues with the Uniswap protocol.`,
-      },
-      stablecoin: {
-        name: 'Stablecoin',
-        description: `This token is verified to maintain peg against a target.`,
-      },
-      trueCrypto: {
-        name: 'TrueCrypto',
-        description: `TrueCryptosystem verifies the token is a necessary element of a self sustaining public utility.`,
-      },
-      dutchX: {
-        name: 'DutchX',
-        description: `This token is verified to comply with the DutchX exchange listing criteria.`,
-      },
-    },
-    tokens: validatedTokens,
-  }
-
-  if (!validator(tokenList)) {
-    console.error('Validation errors encountered.')
-    if (validator.errors)
-      validator.errors.map((err: unknown) => {
-        console.error(err)
-      })
-    throw new Error(`Could not validate generated list ${tokenList}`)
-  }
-
-  console.info('Uploading to IPFS...')
-  const data = new TextEncoder().encode(JSON.stringify(tokenList, null, 2))
-  const ipfsResponse = await ipfsPublish(fileName, data)
-  const contentHash = ipfsResponse[0].hash
-  console.info(`Done. ${process.env.IPFS_GATEWAY}/ipfs/${contentHash}`)
-
-  if (pinata) {
-    console.info('Pinning list in pinata.cloud...')
-    await pinata.pinByHash(contentHash)
-    console.info('Done.')
-  }
-
-  // As of v5.0.5, Ethers ENS API doesn't include managing ENS names, so we
-  // can't use it directly. Neither does the ethjs API.
-  // Web3js supports it via web3.eth.ens but it can't sign transactions
-  // locally and send them via eth_sendRawTransaction, which means it can't
-  // be used with Ethereum endpoints that don't support
-  // eth_sendTransaction (e.g. Infura).
-  //
-  // We'll have to interact with the contracts directly.
-  const signer = new ethers.Wallet(process.env.WALLET_KEY || '', provider)
-  const ensName = namehash.normalize(ensListName)
-  const ensNamehash = namehash.hash(ensName)
-
-  const resolver = new ethers.Contract(
-    await provider._getResolver(ensName),
-    resolverABI,
-    signer,
+    validatedTokens,
   )
 
-  const encodedContentHash = `0x${encode('ipfs-ns', contentHash)}`
-  console.info()
-  console.info('Updating ens entry...')
-  console.info(`Manager: ${await signer.getAddress()}`)
-  await resolver.setContenthash(ensNamehash, encodedContentHash)
-  console.info(
-    `Done. List available at ${process.env.IPFS_GATEWAY}/ipfs/${contentHash}`,
-  )
+  tokenListUtils.validate(schema, tokenList)
+  return tokenList
 }