Use new fountainhead-core (need check)

.env.example

@@ -11,5 +11,9 @@ zmq_incoming_host=127.0.0.1
 zmq_incoming_port=28332
 zmq_outgoing_host=127.0.0.1
 zmq_outgoing_port=28339
+zmq_outgoing_test_host=127.0.0.1
+zmq_outgoing_test_port=28369
 
 core_from=525470
+core_verbose=false
+core_enable_utxo_tracking=false

bit.js

@@ -1,7 +1,7 @@
-const bch = require('bitcore-lib-cash')
+const bch = require('fountainhead-core').bitcore
 const zmq = require('zeromq')
 const RpcClient = require('bitcoind-rpc')
-const TNA = require('fountainhead-tna')
+const TNA = require('fountainhead-core').tna
 const pLimit = require('p-limit')
 const pQueue = require('p-queue')
 const Config = require('./config.js')
@@ -146,6 +146,9 @@ const handle_zmq_block = async function() {
       console.time('DB Insert ' + index)
 
       await Db.block.insert(content, index)
+      if (Config.core.utxo_tracking) {
+        await Db.utxo.apply_block(index)
+      }
 
       await Info.updateTip(index)
       console.timeEnd('DB Insert ' + index)

config.js

@@ -14,7 +14,7 @@ module.exports = {
     'index': {
       'confirmed': {
         'keys': [
-          'tx.h', 'blk.i', 'blk.t', 'blk.h',
+          'blk.i', 'blk.t', 'blk.h',
           'in.e.a', 'in.e.h', 'in.e.i', 'in.i',
           'out.e.a', 'out.e.i', 'out.e.v', 'out.i',
           'in.b0', 'in.b1', 'in.b2', 'in.b3', 'in.b4', 'in.b5', 'in.b6', 'in.b7', 'in.b8', 'in.b9', 'in.b10', 'in.b11', 'in.b12', 'in.b13', 'in.b14', 'in.b15',
@@ -24,8 +24,19 @@ module.exports = {
         'fulltext': ['out.s0', 'out.s1', 'out.s2', 'out.s3', 'out.s4', 'out.s5', 'out.s6', 'out.s7', 'out.s8', 'out.s9', 'out.s10', 'out.s11', 'out.s12', 'out.s13', 'out.s14', 'out.s15']
       },
       'unconfirmed': {
+        'keys': [
+          'in.e.a', 'in.e.h', 'in.e.i', 'in.i',
+          'out.e.a', 'out.e.i', 'out.e.v', 'out.i',
+          'in.b0', 'in.b1', 'in.b2', 'in.b3', 'in.b4', 'in.b5', 'in.b6', 'in.b7', 'in.b8', 'in.b9', 'in.b10', 'in.b11', 'in.b12', 'in.b13', 'in.b14', 'in.b15',
+          'out.b0', 'out.b1', 'out.b2', 'out.b3', 'out.b4', 'out.b5', 'out.b6', 'out.b7', 'out.b8', 'out.b9', 'out.b10', 'out.b11', 'out.b12', 'out.b13', 'out.b14', 'out.b15',
+          'out.s0', 'out.s1', 'out.s2', 'out.s3', 'out.s4', 'out.s5', 'out.s6', 'out.s7', 'out.s8', 'out.s9', 'out.s10', 'out.s11', 'out.s12', 'out.s13', 'out.s14', 'out.s15'
+        ],
+        'fulltext': ['out.s0', 'out.s1', 'out.s2', 'out.s3', 'out.s4', 'out.s5', 'out.s6', 'out.s7', 'out.s8', 'out.s9', 'out.s10', 'out.s11', 'out.s12', 'out.s13', 'out.s14', 'out.s15']
+      },
+      'utxos': {
         'keys': [
           'tx.h',
+          'blk.i', 'blk.t', 'blk.h',
           'in.e.a', 'in.e.h', 'in.e.i', 'in.i',
           'out.e.a', 'out.e.i', 'out.e.v', 'out.i',
           'in.b0', 'in.b1', 'in.b2', 'in.b3', 'in.b4', 'in.b5', 'in.b6', 'in.b7', 'in.b8', 'in.b9', 'in.b10', 'in.b11', 'in.b12', 'in.b13', 'in.b14', 'in.b15',
@@ -44,11 +55,16 @@ module.exports = {
     'outgoing': {
       'host': process.env.zmq_outgoing_host ? process.env.zmq_outgoing_host : '0.0.0.0',
       'port': process.env.zmq_outgoing_port ? process.env.zmq_outgoing_port : '28339'
-    }
+    },
+    'outgoing_test': {
+      'host': process.env.zmq_outgoing_test_host ? process.env.zmq_outgoing_test_host : '0.0.0.0',
+      'port': process.env.zmq_outgoing_test_port ? process.env.zmq_outgoing_test_port : '28369'
+    },
   },
   'core': {
     'version': '0.2.0',
     'from': Number.parseInt(process.env.core_from ? process.env.core_from : 525470),
-    'verbose': process.env.core_verbose ? process.env.core_verbose : false
+    'verbose': process.env.core_verbose === "true" ? true : false,
+    'utxo_tracking': process.env.core_utxo_tracking === "true" ? true : false,
   }
 }

db.js

@@ -28,6 +28,7 @@ var mempool =  {
       console.log('## ERR ', err)
       process.exit()
     })
+    console.log('Reset unconfirmed')
   },
   sync: async function(items) {
     await db.collection('unconfirmed').deleteMany({}).catch(function(err) {
@@ -59,6 +60,8 @@ var block = {
       console.log('## ERR ', err)
       process.exit()
     })
+    block.dropindexes();
+    console.log('Reset confirmed')
   },
   replace: async function(items, block_index) {
     console.log('Deleting all blocks greater than or equal to', block_index)
@@ -121,27 +124,32 @@ var block = {
         let collectionName = collectionNames[j]
         let keys = config.index[collectionName].keys
         let fulltext = config.index[collectionName].fulltext
+
+        console.log('Indexing tx.h');
+        console.time('Unique Index: tx.h')
+        try {
+          await db.collection(collectionName).createIndex({'tx.h': 1}, { unique: true })
+        } catch (e) {
+          console.log(e)
+          process.exit()
+        }
+        console.timeEnd('Unique Index: tx.h')
+        console.log('* Created unique index for ', 'tx.h')
+
         if (keys) {
-          console.log('Indexing keys...')
-          for(let i=0; i<keys.length; i++) {
-            let o = {}
-            o[keys[i]] = 1
-            console.time('Index:' + keys[i])
-            try {
-              if (keys[i] === 'tx.h') {
-                await db.collection(collectionName).createIndex(o, { unique: true })
-                console.log('* Created unique index for ', keys[i])
-              } else {
-                await db.collection(collectionName).createIndex(o)
-                console.log('* Created index for ', keys[i])
-              }
-            } catch (e) {
-              console.log(e)
-              process.exit()
-            }
-            console.timeEnd('Index:' + keys[i])
+          console.log('Indexing keys... [' + keys.join(',') + ']')
+          console.time('Indexing');
+          try {
+            const keyPatterns = keys.map(k => ({ 'key': { [k]: 1 } }));
+            console.log(keyPatterns);
+            await db.collection(collectionName).createIndexes(keyPatterns);
+          } catch (e) {
+            console.log(e)
+            process.exit()
           }
+          console.timeEnd('Indexing');
         }
+
         if (fulltext) {
           console.log('Creating full text index...')
           let o = {}
@@ -172,8 +180,156 @@ var block = {
       console.log('* Error fetching index info ', e)
       process.exit()
     }
+  },
+  dropindexes: async function() {
+    console.log('* Dropping all MongoDB Indexes...')
+    console.time('TotalDropIndex')
+
+    try {
+      console.time('Drop Indexes: confirmed')
+      await db.collection('confirmed').dropIndexes();
+      console.timeEnd('Drop Indexes: confirmed')
+    } catch (e) {
+      console.log('* Error dropping indexes', e)
+    }
+
+    try {
+      console.time('Drop Indexes: unconfirmed')
+      await db.collection('unconfirmed').dropIndexes();
+      console.timeEnd('Drop Indexes: unconfirmed')
+    } catch (e) {
+      console.log('* Error dropping indexes', e)
+    }
+
+    console.log('* Finished dropping all MongoDB Indexes...')
+    console.timeEnd('TotalDropIndex')
+  }
+}
+
+var utxo = {
+  initial_index: async function() {
+    console.log('Indexing utxo');
+    console.time('Unique Index: utxo')
+    try {
+      await db.collection('utxos').createIndex({'uxto': 1}/*, { unique: false }*/)
+    } catch (e) {
+      console.log(e)
+      process.exit()
+    }
+    console.timeEnd('Unique Index: utxo')
+    console.log('* Created unique index for ', 'utxo')
+  },
+  sync: async function() {
+    utxo.initial_index();
+
+    // TODO get these from querying db
+    let start_height = 581000;
+    let end_height = 582250;
+
+    for (let height=start_height; height<end_height; ++height) {
+      await utxo.apply_block(height);
+    }
+  },
+  apply_block: async function(block_index) {
+    await db.collection('confirmed').find({
+      "blk.i": block_index
+    }).toArray().then(async (docs) => {
+      console.time('UTXO Update');
+
+      let input_tx_vout_pairs = [];
+      let output_map = new Map();
+
+      let blk_key = null;
+
+      for (const doc of docs) {
+        if (! blk_key) {
+          blk_key = doc.blk;
+        }
+
+        input_tx_vout_pairs.push(...doc.in.map(v => v.e.h+':'+v.e.i));
+
+        for (const o of doc.out) {
+          output_map.set(doc.tx.h+':'+o.e.i, {
+            utxo: doc.tx.h+':'+o.e.i,
+            tx: doc.tx,
+            in: doc.in,
+            out: o,
+            blk: blk_key
+          });
+        }
+      }
+
+      let spent_inside_block = 0;
+      for (let v of input_tx_vout_pairs) {
+        spent_inside_block += output_map.delete(v);
+      }
+      console.log('Spent inside block ' + spent_inside_block);
+
+      let outputs_list = [...output_map.values()];
+
+      console.time('INSERTING Utxos');
+      while (true) {
+        const chunk = outputs_list.splice(0, 1000)
+        if (chunk.length > 0) {
+          await db.collection('utxos')
+                .insertMany(chunk/*, { ordered: false }*/).catch(function(err) {
+            // duplicates are ok because they will be ignored
+            if (err.code !== 11000) {
+              console.log('## ERR ', err, chunk)
+              process.exit()
+            } else {
+              console.log(err)
+            }
+          })
+        } else {
+          break
+        }
+      }
+      console.timeEnd('INSERTING Utxos');
+
+      console.time('DELETING Utxos');
+      while (true) {
+        const chunk = input_tx_vout_pairs.splice(0, 1000)
+        if (chunk.length > 0) {
+          await db.collection('utxos').deleteMany({
+            "utxo": {
+                $in: chunk
+            }
+          }).catch(function(err) {
+            // duplicates are ok because they will be ignored
+            if (err.code !== 11000) {
+              console.log('## ERR ', err, items)
+              process.exit()
+            }
+          })
+        } else {
+          break
+        }
+      }
+      console.timeEnd('DELETING Utxos');
+
+      console.timeEnd('UTXO Update');
+    })
+  },
+  reset: async function() {
+    try {
+      console.time('Drop Indexes: utxos')
+      await db.collection('utxos').dropIndexes();
+      console.timeEnd('Drop Indexes: utxos')
+    } catch (e) {
+      console.log('* Error dropping indexes', e)
+      process.exit()
+    }
+
+    await db.collection('utxos').deleteMany({}).catch(function(err) {
+      console.log('## ERR ', err)
+      process.exit()
+    })
+    console.log('Reset utxos')
   }
 }
+
+
 module.exports = {
-  init: init, exit: exit, block: block, mempool: mempool
+  init: init, exit: exit, block: block, mempool: mempool, utxo: utxo
 }

index.js

@@ -3,6 +3,7 @@ const Config = require('./config.js')
 const Info = require('./info.js')
 const Bit = require('./bit.js')
 const Db = require('./db')
+const Stresstest = require('./stresstest')
 const ip = require('ip')
 console.log(ip.address())
 
@@ -15,13 +16,6 @@ const daemon = {
     // 2. Bootstrap actions depending on first time
     const lastSynchronized = await Info.checkpoint()
 
-    console.time('Indexing Keys')
-    if (lastSynchronized === Config.core.from) {
-      // First time. Try indexing
-      console.log('Indexing...', new Date())
-      await Db.block.index()
-    }
-    console.timeEnd('Indexing Keys')
 
     if (lastSynchronized !== Config.core.from) {
       // Resume
@@ -31,6 +25,10 @@ const daemon = {
       // and the block was not indexed completely.
       console.log('Resuming...')
       await util.fix(lastSynchronized-1)
+    } else {
+      if (Config.core.utxo_tracking) {
+        Db.utxo.initial_index();
+      }
     }
 
     // 3. Start synchronizing
@@ -39,6 +37,15 @@ const daemon = {
     await Bit.run()
     console.timeEnd('Initial Sync')
 
+
+    if (lastSynchronized === Config.core.from) {
+      console.time('Indexing Keys')
+      // First time. Try indexing
+      console.log('Indexing...', new Date())
+      await Db.block.index()
+      console.timeEnd('Indexing Keys')
+    }
+
     // 4. Start listening
     Bit.listen()
   }
@@ -59,11 +66,26 @@ const util = {
     } else if (cmd === 'reset') {
       await Db.block.reset()
       await Db.mempool.reset()
+      if (Config.core.utxo_tracking) {
+        await Db.utxo.reset()
+      }
       await Info.deleteTip()
       process.exit()
     } else if (cmd === 'index') {
       await Db.block.index()
       process.exit()
+    } else if (cmd === 'dropindexes') {
+      await Db.block.dropindexes()
+      process.exit()
+    } else if (cmd === 'utxo-sync') {
+      await Db.utxo.sync();
+      process.exit();
+    } else if (cmd === 'utxo-reset') { // TODO delete me
+      await Db.utxo.reset();
+      process.exit();
+    } else {
+      console.log('Unknown command')
+      process.exit()
     }
   },
   fix: async function(from) {