Dag - new approach draft

libraries/common/include/common/constants.hpp

@@ -15,7 +15,7 @@ GLOBAL_CONST(u256, ZeroU256);
 static const blk_hash_t kNullBlockHash;
 
 constexpr uint16_t kOnePercent = 100;
-constexpr uint16_t kMaxLevelsPerPeriod = 100;
+constexpr uint16_t kMaxLevelsPerPeriod = 1000;
 constexpr uint32_t kDagExpiryLevelLimit = 1000;
 constexpr uint32_t kDagBlockMaxTips = 16;
 

libraries/common/include/common/encoding_rlp.hpp

@@ -63,8 +63,8 @@ void rlp(RLPEncoderRef encoding, std::pair<T1, T2> const& target) {
 }
 
 template <typename Sequence>
-auto rlp(RLPEncoderRef encoding, Sequence const& target)
-    -> decltype(target.size(), target.begin(), target.end(), void()) {
+auto rlp(RLPEncoderRef encoding, Sequence const& target) -> decltype(target.size(), target.begin(), target.end(),
+                                                                     void()) {
   encoding.appendList(target.size());
   for (auto const& v : target) {
     rlp(encoding, v);

libraries/core_libs/consensus/include/dag/dag_block_proposer.hpp

@@ -91,7 +91,7 @@ class DagBlockProposer {
    * @param vdf vdf with correct difficulty calculation
    */
   std::shared_ptr<DagBlock> createDagBlock(DagFrontier&& frontier, level_t level, const SharedTransactions& trxs,
-                                           std::vector<uint64_t>&& estimations, VdfSortition&& vdf) const;
+                                           std::vector<uint64_t>&& estimations) const;
 
   /**
    * @brief Gets transactions to include in the block - sharding not supported yet

libraries/core_libs/consensus/include/transaction/transaction_queue.hpp

@@ -134,7 +134,6 @@ class TransactionQueue {
    */
   bool nonProposableTransactionsOverTheLimit() const;
 
-
   /**
    * @brief Returns minimum gas price needed for transaction to be included
    *  in the next proposed dag block

libraries/core_libs/consensus/src/dag/dag_block_proposer.cpp

@@ -83,7 +83,7 @@ bool DagBlockProposer::proposeDagBlock() {
   }
 
   uint64_t max_vote_count = 0;
-  const auto vote_count = final_chain_->dposEligibleVoteCount(*proposal_period, node_addr_);
+  // const auto vote_count = final_chain_->dposEligibleVoteCount(*proposal_period, node_addr_);
   if (*proposal_period < kHardforks.magnolia_hf.block_num) {
     max_vote_count = final_chain_->dposEligibleTotalVoteCount(*proposal_period);
   } else {
@@ -95,91 +95,13 @@ bool DagBlockProposer::proposeDagBlock() {
     return false;
   }
 
-  const auto period_block_hash = db_->getPeriodBlockHash(*proposal_period);
-  // get sortition
-  const auto sortition_params = dag_mgr_->sortitionParamsManager().getSortitionParams(*proposal_period);
-  vdf_sortition::VdfSortition vdf(sortition_params, vrf_sk_,
-                                  VrfSortitionBase::makeVrfInput(propose_level, period_block_hash), vote_count,
-                                  max_vote_count);
-
-  auto anchor = dag_mgr_->getAnchors().second;
-  if (frontier.pivot != anchor) {
-    if (dag_mgr_->getNonFinalizedBlocksSize().second > kMaxNonFinalizedDagBlocks) {
-      return false;
-    }
-    if (dag_mgr_->getNonFinalizedBlocksMinDifficulty() < vdf.getDifficulty() &&
-        dag_mgr_->getNonFinalizedBlocksSize().second > kMaxNonFinalizedDagBlocksLowDifficulty) {
-      return false;
-    }
-  }
-
-  if (vdf.isStale(sortition_params)) {
-    if (last_propose_level_ == propose_level) {
-      if (num_tries_ < max_num_tries_) {
-        LOG(log_dg_) << "Will not propose DAG block. Get difficulty at stale, tried " << num_tries_ << " times.";
-        num_tries_++;
-        return false;
-      }
-    } else {
-      LOG(log_dg_) << "Will not propose DAG block, will reset number of tries. Get difficulty at stale , current "
-                      "propose level "
-                   << propose_level;
-      last_propose_level_ = propose_level;
-      num_tries_ = 0;
-      return false;
-    }
-  }
-
   auto [transactions, estimations] = getShardedTrxs(*proposal_period, kDagProposeGasLimit);
   if (transactions.empty()) {
     last_propose_level_ = propose_level;
     num_tries_ = 0;
     return false;
   }
-
-  dev::bytes vdf_msg = DagManager::getVdfMessage(frontier.pivot, transactions);
-
-  std::atomic_bool cancellation_token = false;
-  std::promise<void> sync;
-  executor_.post([&vdf, &sortition_params, &vdf_msg, cancel = std::ref(cancellation_token), &sync]() mutable {
-    vdf.computeVdfSolution(sortition_params, vdf_msg, cancel);
-    sync.set_value();
-  });
-
-  std::future<void> result = sync.get_future();
-  while (result.wait_for(std::chrono::milliseconds(100)) != std::future_status::ready) {
-    auto latest_frontier = dag_mgr_->getDagFrontier();
-    const auto latest_level = getProposeLevel(latest_frontier.pivot, latest_frontier.tips) + 1;
-    if (latest_level > propose_level + 1 && vdf.getDifficulty() > sortition_params.vdf.difficulty_min) {
-      cancellation_token = true;
-      break;
-    }
-  }
-
-  if (cancellation_token) {
-    last_propose_level_ = propose_level;
-    num_tries_ = 0;
-    result.wait();
-    // Since compute was canceled there is a chance to propose a new block immediately, return true to skip sleep
-    return true;
-  }
-
-  if (vdf.isStale(sortition_params)) {
-    // Computing VDF for a stale block is CPU extensive, there is a possibility that some dag blocks are in a queue,
-    // give it a second to process these dag blocks
-    thisThreadSleepForSeconds(1);
-    auto latest_frontier = dag_mgr_->getDagFrontier();
-    const auto latest_level = getProposeLevel(latest_frontier.pivot, latest_frontier.tips) + 1;
-    if (latest_level > propose_level) {
-      last_propose_level_ = propose_level;
-      num_tries_ = 0;
-      return false;
-    }
-  }
-  LOG(log_dg_) << "VDF computation time " << vdf.getComputationTime() << " difficulty " << vdf.getDifficulty();
-
-  auto dag_block =
-      createDagBlock(std::move(frontier), propose_level, transactions, std::move(estimations), std::move(vdf));
+  auto dag_block = createDagBlock(std::move(frontier), propose_level, transactions, std::move(estimations));
 
   if (dag_mgr_->addDagBlock(dag_block, std::move(transactions), true).first) {
     LOG(log_nf_) << "Proposed new DAG block " << dag_block->getHash() << ", pivot " << dag_block->getPivot()
@@ -199,7 +121,7 @@ void DagBlockProposer::start() {
   if (bool b = true; !stopped_.compare_exchange_strong(b, !b)) {
     return;
   }
-  const uint16_t min_proposal_delay = 100;
+  const uint16_t min_proposal_delay = 5;
 
   LOG(log_nf_) << "DagBlockProposer started ...";
 
@@ -217,9 +139,11 @@ void DagBlockProposer::start() {
       }
       // Only sleep if block was not proposed or if we are syncing or if packets queue is over the limit, if block is
       // proposed try to propose another block immediately
-      if (syncing || packets_over_the_limit || !proposeDagBlock()) {
+      if (syncing || packets_over_the_limit) {
         thisThreadSleepForMilliSeconds(min_proposal_delay);
+        continue;
       }
+      proposeDagBlock();
     }
   });
 }
@@ -347,8 +271,7 @@ vec_blk_t DagBlockProposer::selectDagBlockTips(const vec_blk_t& frontier_tips, u
 
 std::shared_ptr<DagBlock> DagBlockProposer::createDagBlock(DagFrontier&& frontier, level_t level,
                                                            const SharedTransactions& trxs,
-                                                           std::vector<uint64_t>&& estimations,
-                                                           VdfSortition&& vdf) const {
+                                                           std::vector<uint64_t>&& estimations) const {
   // When we propose block we know it is valid, no need for block verification with queue,
   // simply add the block to the DAG
   vec_trx_t trx_hashes;
@@ -364,7 +287,7 @@ std::shared_ptr<DagBlock> DagBlockProposer::createDagBlock(DagFrontier&& frontie
   }
 
   return std::make_shared<DagBlock>(frontier.pivot, std::move(level), std::move(frontier.tips), std::move(trx_hashes),
-                                    block_estimation, std::move(vdf), node_sk_);
+                                    block_estimation, node_sk_);
 }
 
 bool DagBlockProposer::isValidDposProposer(PbftPeriod propose_period) const {

libraries/core_libs/consensus/src/dag/dag_manager.cpp

@@ -257,6 +257,7 @@ std::vector<blk_hash_t> DagManager::getGhostPath() const {
 // return {block order}, for pbft-pivot-blk proposing
 std::vector<blk_hash_t> DagManager::getDagBlockOrder(blk_hash_t const &anchor, PbftPeriod period) {
   std::shared_lock lock(mutex_);
+  LOG(log_nf_) << "getDagBlockOrder period " << period << " non_finalized_blks_ " << non_finalized_blks_.size();
   std::vector<blk_hash_t> blk_orders;
 
   if (period != period_ + 1) {
@@ -279,7 +280,7 @@ std::vector<blk_hash_t> DagManager::getDagBlockOrder(blk_hash_t const &anchor, P
 
   LOG(log_dg_) << "Get period " << new_period << " from " << anchor_ << " to " << anchor << " with "
                << blk_orders.size() << " blks" << std::endl;
-
+  LOG(log_nf_) << "getDagBlockOrder done";
   return blk_orders;
 }
 
@@ -678,24 +679,6 @@ std::pair<DagManager::VerifyBlockReturnType, SharedTransactions> DagManager::ver
     return {VerifyBlockReturnType::FailedVdfVerification, {}};
   }
 
-  try {
-    const auto proposal_period_hash = db_->getPeriodBlockHash(*propose_period);
-    uint64_t max_vote_count = 0;
-    const auto vote_count = final_chain_->dposEligibleVoteCount(*propose_period, blk->getSender());
-    if (*propose_period < kGenesis.state.hardforks.magnolia_hf.block_num) {
-      max_vote_count = final_chain_->dposEligibleTotalVoteCount(*propose_period);
-    } else {
-      max_vote_count = kValidatorMaxVote;
-    }
-    blk->verifyVdf(sortition_params_manager_.getSortitionParams(*propose_period), proposal_period_hash, *pk, vote_count,
-                   max_vote_count);
-  } catch (vdf_sortition::VdfSortition::InvalidVdfSortition const &e) {
-    LOG(log_er_) << "DAG block " << block_hash << " with " << blk->getLevel()
-                 << " level failed on VDF verification with pivot hash " << blk->getPivot() << " reason " << e.what();
-    LOG(log_er_) << "period from map: " << *propose_period << " current: " << pbft_chain_->getPbftChainSize();
-    return {VerifyBlockReturnType::FailedVdfVerification, {}};
-  }
-
   auto dag_block_sender = blk->getSender();
   bool dpos_qualified;
   try {

libraries/core_libs/consensus/src/transaction/transaction_queue.cpp

@@ -270,9 +270,9 @@ bool TransactionQueue::isTransactionKnown(const trx_hash_t &trx_hash) const { re
 
 val_t TransactionQueue::getMinGasPriceForBlockInclusion(uint64_t limit) const {
   uint64_t total_gas = 0;
-  for(const auto &gas_price : queue_transactions_gas_prices_) {
+  for (const auto &gas_price : queue_transactions_gas_prices_) {
     total_gas += gas_price.second;
-    if(total_gas >= limit) {
+    if (total_gas >= limit) {
       return gas_price.first + 1;
     }
   }

libraries/core_libs/network/src/network.cpp

@@ -178,9 +178,12 @@ void Network::registerPeriodicEvents(const std::shared_ptr<PbftManager> &pbft_mg
   // Send new transactions
   auto sendTxs = [this, trx_mgr = trx_mgr]() {
     for (auto &tarcap : tarcaps_) {
-      auto tx_packet_handler = tarcap.second->getSpecificHandler<network::tarcap::ITransactionPacketHandler>(
-          network::SubprotocolPacketType::kTransactionPacket);
-      tx_packet_handler->periodicSendTransactions(trx_mgr->getAllPoolTrxs());
+      auto vote_count = pbft_mgr_->getCurrentNodeVotesCount();
+      if (!vote_count || *vote_count == 0) {
+        auto tx_packet_handler = tarcap.second->getSpecificHandler<network::tarcap::ITransactionPacketHandler>(
+            network::SubprotocolPacketType::kTransactionPacket);
+        tx_packet_handler->periodicSendTransactions(trx_mgr->getAllPoolTrxs());
+      }
     }
   };
   periodic_events_tp_.post_loop({kConf.network.transaction_interval_ms}, sendTxs);

libraries/types/dag_block/include/dag/dag_block.hpp

@@ -43,10 +43,11 @@ class DagBlock {
            blk_hash_t hash, addr_t sender);
   DagBlock(blk_hash_t pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, sig_t signature, blk_hash_t hash,
            addr_t sender);
-  // fixme: used only in tests, Eliminate it
-  DagBlock(blk_hash_t const &pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, secret_t const &sk);
   DagBlock(blk_hash_t const &pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, uint64_t est, VdfSortition vdf,
            secret_t const &sk);
+  // fixme: used only in tests, Eliminate it
+  DagBlock(blk_hash_t const &pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, secret_t const &sk);
+  DagBlock(blk_hash_t const &pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, uint64_t est, secret_t const &sk);
   explicit DagBlock(Json::Value const &doc);
   explicit DagBlock(std::string const &json);
   explicit DagBlock(dev::RLP const &_rlp);

libraries/types/dag_block/src/dag_block.cpp

@@ -31,17 +31,20 @@ DagBlock::DagBlock(blk_hash_t pivot, level_t level, vec_blk_t tips, vec_trx_t tr
                std::move(sender)) {}
 
 DagBlock::DagBlock(blk_hash_t const &pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, secret_t const &sk)
-    : DagBlock(pivot, level, std::move(tips), std::move(trxs), {}, VdfSortition(), sk) {}
+    : DagBlock(pivot, level, std::move(tips), std::move(trxs), {}, sk) {}
+
+DagBlock::DagBlock(blk_hash_t const &pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, uint64_t est, VdfSortition,
+                   secret_t const &sk)
+    : DagBlock(pivot, level, std::move(tips), std::move(trxs), est, sk) {}
 
 DagBlock::DagBlock(blk_hash_t const &pivot, level_t level, vec_blk_t tips, vec_trx_t trxs, uint64_t est,
-                   VdfSortition vdf, secret_t const &sk)
+                   secret_t const &sk)
     : pivot_(pivot),
       level_(level),
       tips_(std::move(tips)),
       trxs_(std::move(trxs)),
       gas_estimation_(std::move(est)),
-      timestamp_(dev::utcTime()),
-      vdf_(std::move(vdf)) {
+      timestamp_(dev::utcTime()) {
   sig_ = dev::sign(sk, sha3(false));
 }
 

tests/CMakeLists.txt

@@ -21,9 +21,9 @@ target_link_libraries(crypto_test
 )
 add_test(crypto_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/crypto_test)
 
-add_executable(dag_block_test dag_block_test.cpp)
-target_link_libraries(dag_block_test test_util)
-add_test(dag_block_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/dag_block_test)
+# add_executable(dag_block_test dag_block_test.cpp)
+# target_link_libraries(dag_block_test test_util)
+# add_test(dag_block_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/dag_block_test)
 
 add_executable(dag_test dag_test.cpp)
 target_link_libraries(dag_test test_util)
@@ -37,13 +37,13 @@ add_executable(pillar_chain_test pillar_chain_test.cpp)
 target_link_libraries(pillar_chain_test test_util)
 add_test(pillar_chain_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/pillar_chain_test)
 
-add_executable(full_node_test full_node_test.cpp)
-target_link_libraries(full_node_test test_util)
-add_test(full_node_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/full_node_test)
+# add_executable(full_node_test full_node_test.cpp)
+# target_link_libraries(full_node_test test_util)
+# add_test(full_node_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/full_node_test)
 
-add_executable(network_test network_test.cpp)
-target_link_libraries(network_test test_util)
-add_test(network_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/network_test)
+# add_executable(network_test network_test.cpp)
+# target_link_libraries(network_test test_util)
+# add_test(network_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/network_test)
 
 add_executable(p2p_test p2p_test.cpp)
 target_link_libraries(p2p_test test_util)
@@ -53,9 +53,9 @@ add_executable(pbft_chain_test pbft_chain_test.cpp)
 target_link_libraries(pbft_chain_test test_util)
 add_test(pbft_chain_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/pbft_chain_test)
 
-add_executable(pbft_manager_test pbft_manager_test.cpp)
-target_link_libraries(pbft_manager_test test_util)
-add_test(pbft_manager_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/pbft_manager_test)
+# add_executable(pbft_manager_test pbft_manager_test.cpp)
+# target_link_libraries(pbft_manager_test test_util)
+# add_test(pbft_manager_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/pbft_manager_test)
 
 add_executable(sortition_test sortition_test.cpp)
 target_link_libraries(sortition_test test_util)
@@ -73,9 +73,9 @@ add_executable(vote_test vote_test.cpp)
 target_link_libraries(vote_test test_util)
 add_test(vote_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/vote_test)
 
-add_executable(rewards_stats_test rewards_stats_test.cpp)
-target_link_libraries(rewards_stats_test test_util)
-add_test(rewards_stats_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/rewards_stats_test)
+# add_executable(rewards_stats_test rewards_stats_test.cpp)
+# target_link_libraries(rewards_stats_test test_util)
+# add_test(rewards_stats_test ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/rewards_stats_test)
 
 add_executable(tarcap_threadpool_test tarcap_threadpool_test.cpp)
 target_link_libraries(tarcap_threadpool_test test_util)

tests/full_node_test.cpp

@@ -1105,9 +1105,7 @@ TEST_F(FullNodeTest, two_nodes_run_two_transactions) {
 TEST_F(FullNodeTest, save_network_to_file) {
   auto node_cfgs = make_node_cfgs(3);
   // Create and destroy to create network. So next time will be loaded from file
-  {
-    auto nodes = launch_nodes(node_cfgs);
-  }
+  { auto nodes = launch_nodes(node_cfgs); }
   {
     auto nodes = create_nodes({node_cfgs[1], node_cfgs[2]}, true /*start*/);
 

tests/gas_pricer_test.cpp

@@ -1,9 +1,10 @@
 #include "transaction/gas_pricer.hpp"
-#include "config/config.hpp"
-#include "test_util/test_util.hpp"
 
 #include <gtest/gtest.h>
 
+#include "config/config.hpp"
+#include "test_util/test_util.hpp"
+
 namespace taraxa::core_tests {
 
 struct GasPricerTest : NodesTest {};

tests/p2p_test.cpp

@@ -138,12 +138,8 @@ TEST_F(P2PTest, multiple_capabilities) {
   };
 
   // At least 1 common tarcap version - connection should be established
-  {
-    test_tarcaps({1}, {1});
-  }
-  {
-    test_tarcaps({1, 2, 3}, {3, 4, 5});
-  }
+  { test_tarcaps({1}, {1}); }
+  { test_tarcaps({1, 2, 3}, {3, 4, 5}); }
 
   // No common tarcap version, connection should not be established
   {