feat(choreo): dup handling, gossip int, bug fixes, missing logic

Author: lidatong

src/app/firedancer-dev/commands/send_test/send_test_helpers.c

@@ -215,7 +215,7 @@ encode_vote( send_test_ctx_t * ctx, fd_tower_slot_done_t * slot_done ) {
   /* Create minimal mock tower with one vote */
   uchar tower_mem[ FD_TOWER_FOOTPRINT ] __attribute__((aligned(FD_TOWER_ALIGN)));
   fd_tower_t * tower = fd_tower_join( fd_tower_new( tower_mem ) );
-  fd_tower_votes_push_tail( tower, (fd_tower_vote_t){ .slot = vote_slot, .conf = 1 } );
+  fd_tower_push_tail( tower, (fd_tower_vote_t){ .slot = vote_slot, .conf = 1 } );
 
   /* Mock values */
   fd_lockout_offset_t lockouts_scratch[1];

src/app/firedancer/config/default.toml

@@ -463,6 +463,15 @@ user = ""
     # abort with an error.
     expected_genesis_hash = ""
 
+    # Firedancer can process at most this many slots without rooting in
+    # the consensus rules before it must begin evicting.
+    #
+    # This is an estimate and should be set as generously as possible to
+    # allow for brief anomalies such as the validator disconnecting from
+    # part of the cluster due to data center issues.  Roughly, at 400 ms
+    # per slot, the default allows for 30 minutes without rooting.
+    max_unrooted_slots = 4096
+
 # This section configures the "funk" account database.  Currently, funk
 # stores all Solana accounts.  In future versions of Firedancer, most
 # accounts will be offloaded to the "groove" database.

src/app/firedancer/topology.c

@@ -1,6 +1,7 @@
 #include "topology.h"
 
 #include "../../ballet/lthash/fd_lthash.h"
+#include "../../choreo/fd_choreo_base.h"
 #include "../../discof/reasm/fd_reasm.h"
 #include "../../discof/poh/fd_poh.h"
 #include "../../discof/replay/fd_exec.h"
@@ -227,6 +228,7 @@ resolve_gossip_entrypoints( config_t * config ) {
 
 void
 fd_topo_initialize( config_t * config ) {
+  FD_LOG_NOTICE(( "max unrooted %lu", config->firedancer.consensus.max_unrooted_slots ));
   /* TODO: Not here ... */
   resolve_gossip_entrypoints( config );
 
@@ -347,6 +349,7 @@ fd_topo_initialize( config_t * config ) {
 
   /* TODO: Explain this .... USHORT_MAX is not dcache max */
   ulong pending_fec_shreds_depth = fd_ulong_min( fd_ulong_pow2_up( config->tiles.shred.max_pending_shred_sets * FD_REEDSOL_DATA_SHREDS_MAX ), USHORT_MAX + 1 /* dcache max */ );
+  ulong max_unrooted_slots       = config->firedancer.consensus.max_unrooted_slots;
 
   /*                                  topo, link_name,      wksp_name,      depth,                                    mtu,                           burst */
   /**/                 fd_topob_link( topo, "gossip_net",   "net_gossip",   config->net.ingress_buffer_size,          FD_NET_MTU,                    1UL );
@@ -405,7 +408,7 @@ fd_topo_initialize( config_t * config ) {
 
   FOR(shred_tile_cnt)  fd_topob_link( topo, "shred_out",    "shred_out",    pending_fec_shreds_depth,                 FD_SHRED_OUT_MTU,              3UL ); /* TODO: Pretty sure burst of 3 is incorrect here */
   FOR(shred_tile_cnt)  fd_topob_link( topo, "repair_shred", "shred_out",    pending_fec_shreds_depth,                 sizeof(fd_ed25519_sig_t),      1UL ); /* TODO: Also pending_fec_shreds_depth? Seems wrong */
-  /**/                 fd_topob_link( topo, "tower_out",    "tower_out",    1024UL,                                   sizeof(fd_tower_slot_done_t),  1UL );
+  /**/                 fd_topob_link( topo, "tower_out",    "tower_out",    max_unrooted_slots,                       sizeof(fd_tower_msg_t),        1UL );
   /**/                 fd_topob_link( topo, "send_out",     "send_out",     128UL,                                    FD_TPU_RAW_MTU,                1UL );
 
                        fd_topob_link( topo, "replay_exec",  "replay_exec",  16384UL,                                  sizeof(fd_exec_task_msg_t),    1UL );
@@ -558,6 +561,7 @@ fd_topo_initialize( config_t * config ) {
   /**/                 fd_topob_tile_in(    topo, "replay",  0UL,          "metric_in", "poh_replay",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   FOR(exec_tile_cnt)   fd_topob_tile_in(    topo, "exec",    i,            "metric_in", "replay_exec",  0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "genesi_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+  /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "gossip_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "replay_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   if( snapshots_enabled ) {
                        fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "snapin_manif", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
@@ -1017,8 +1021,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "tower" ) ) ) {
 
-    strncpy( tile->tower.identity_key_path, config->paths.identity_key, sizeof(tile->tower.identity_key_path) );
-    strncpy( tile->tower.vote_acc_path, config->paths.vote_account, sizeof(tile->tower.vote_acc_path) );
+    tile->tower.slot_max = config->firedancer.consensus.max_unrooted_slots;
+    strncpy( tile->tower.identity_key, config->paths.identity_key, sizeof(tile->tower.identity_key) );
+    strncpy( tile->tower.vote_account, config->paths.vote_account, sizeof(tile->tower.vote_account) );
     strncpy( tile->tower.ledger_path, config->paths.ledger, sizeof(tile->tower.ledger_path) );
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "send" ) ) ) {

src/app/shared/fd_config.h

@@ -123,6 +123,10 @@ struct fd_configf {
     char host[ 256 ];
   } gossip;
 
+  struct {
+    ulong max_unrooted_slots;
+  } consensus;
+
   struct {
     struct {
       uint max_local_full_effective_age;

src/app/shared/fd_config_parse.c

@@ -80,6 +80,8 @@ fd_config_extract_podf( uchar *        pod,
                         fd_configf_t * config ) {
   CFG_POP      ( cstr,   gossip.host                                         );
 
+  CFG_POP      ( ulong,  consensus.max_unrooted_slots                        );
+
   CFG_POP      ( uint,   layout.exec_tile_count                              );
   CFG_POP      ( uint,   layout.sign_tile_count                              );
   CFG_POP      ( uint,   layout.gossvf_tile_count                            );

src/app/shared_dev/commands/dev.c

@@ -108,13 +108,6 @@ update_config_for_dev( fd_config_t * config ) {
       shred->shred.expected_shred_version = shred_version;
     }
   }
-  ulong store_id = fd_topo_find_tile( &config->topo, "storei", 0 );
-  if( FD_UNLIKELY( store_id!=ULONG_MAX ) ) {
-    fd_topo_tile_t * storei = &config->topo.tiles[ store_id ];
-    if( FD_LIKELY( storei->store_int.expected_shred_version==(ushort)0 ) ) {
-      storei->store_int.expected_shred_version = shred_version;
-    }
-  }
 }
 
 /* Run Firedancer entirely in a single process for development and