snapshots: restructure config options

Author: amass-jump

src/app/firedancer-dev/commands/snapshot_load.c

@@ -138,8 +138,8 @@ extern int * fd_log_private_shared_lock;
 static void
 snapshot_load_cmd_fn( args_t *   args,
                       config_t * config ) {
-  if( FD_UNLIKELY( config->firedancer.snapshots.sources.gossip.enabled || config->firedancer.snapshots.sources.entrypoints.enabled ) ) {
-    FD_LOG_ERR(( "snapshot-load command is incompatible with gossip or entrypoint snapshot sources" ));
+  if( FD_UNLIKELY( config->firedancer.snapshots.sources.gossip.allow_any || 0UL!=config->firedancer.snapshots.sources.gossip.allow_list_cnt ) ) {
+    FD_LOG_ERR(( "snapshot-load command is incompatible with gossip snapshot sources" ));
   }
   snapshot_load_topo( config, args );
   fd_topo_t * topo = &config->topo;

src/app/firedancer/config/default.toml

@@ -267,6 +267,7 @@ user = ""
     # specified in `[net.interface]`.
     host = ""
 
+#TODO-AM: Tweak all the wording
 # Snapshots are a periodic view of the ledger at a point in time.  They
 # are used to enable validators to join the network quickly, as they do
 # not need to replay all transactions since genesis, just those since a
@@ -283,20 +284,6 @@ user = ""
     # a validator to catch up more quickly.
     incremental_snapshots = true
 
-    # How old a local incremental snapshot on disk can be, in slots, and
-    # still be loaded on startup.  The age of a snapshot is determined
-    # by looking at the highest incremental snapshot slot available for
-    # download from cluster peers.
-    #
-    # If a local snapshot exists on disk, within the given age, other
-    # options are ignored and no snapshot will be downloaded.
-    #
-    # If the local snapshot is older than this, we will instead download
-    # a new snapshot.  If the local snapshot is too old, and we cannot
-    # download a snapshot because `download` is `false`, the validator
-    # will exit with an error.
-    maximum_local_snapshot_age = 2500
-
     # Whether to allow downloading a new genesis file from a peer when
     # booting.  If set to false, the validator will only reuse a locally
     # stored genesis file, and if one does not exist, the validator will
@@ -318,64 +305,6 @@ user = ""
     # TODO: Not yet implemented.
     genesis_download = true
 
-    # Whether to allow downloading a new snapshot from a peer when
-    # booting.
-    #
-    # If set to true, the validator will contact peers from the gossip
-    # network to locate both a full and incremental snapshot and
-    # download it, assuming there is no loadable local snapshot.  If
-    # false, the validator will attempt to load the most recently
-    # downloaded snapshot that exists locally on the disk.  If there is
-    # no local snapshot, the validator will abort with an error.
-    #
-    # When downloading a snapshot, it will optionally be stored on disk
-    # if `[paths.snapshots]` is specified.  The number of downloaded
-    # snapshots that will be stored on disk is controlled below by the
-    # max_*_snapshots to keep options.
-    #
-    # TODO: implement the "at most one downloaded" logic. "
-    download = true
-
-    # A snapshot hash must be published in gossip by one of the
-    # validator identities listed here for it to be accepted.  The list
-    # should be a list of identity public keys, specified as base58
-    # strings.
-    #
-    # If a snapshot hash is received that is not published by a known
-    # validator the validator will exit with an error.
-    #
-    # If no known validators are specified here, any snapshot hash will
-    # be accepted.
-    #
-    # TODO: Implement this once gossip is wired up.
-    known_validators = []
-
-    # The minimum acceptable speed for snapshot download from a peer, in
-    # megabits per second.  If the initial download speed falls below
-    # this threshold, the validator will abandon the download, find a
-    # new peer and start downloading the snapshot from them.
-    #
-    # The initial download speed is measured over every 10 mib for the
-    # first 400 mib of downloaded content.  The validator continues
-    # to measure download speed over every 10 mib until the download is
-    # complete, but only the initial download speed is used to decide
-    # whether to abort and retry the download.
-    # TODO: refine the initial download speed measurement and threshold.
-    #
-    # If set to zero, no minimum threshold will be applied.
-    minimum_download_speed_mib = 0
-
-    # The maximum number of times to abort and retry when encountering a
-    # slow snapshot download.  If the maximum attempts is reached while
-    # downloading a snapshot the validator will exit with an error.
-    #
-    # The number of attempts is reset when starting a download for a
-    # new full or incremental snapshot.
-    #
-    # If set to zero, the validator will continually retry until
-    # completing snapshot download.
-    maximum_download_retry_abort = 5
-
     # Controls how many full snapshots are allowed to be kept in the
     # snapshots directory on disk.
     #
@@ -385,6 +314,15 @@ user = ""
     max_full_snapshots_to_keep = 1
     max_incremental_snapshots_to_keep = 1
 
+    # If an in-progress loading (local, or downloaded) of a full
+    # snapshot has its effective age rise above this threshold, we will
+    # cancel the load and start over.  The most common reason this would
+    # happen is if the network is all using the same full snapshot
+    # interval (e.g. 100_000) and while we are downloading a full
+    # snapshot from interval N, the network moves on to N+1, removing
+    # availability of all incremental snapshots that build on N.
+    full_effective_age_cancel_threshold = 20_000
+
     # Controls for where we look to find snapshots.  If multiple sources
     # and peers are available, the one with the lowest latency and
     # fastest download will be automatically selected among them.
@@ -405,11 +343,47 @@ user = ""
     # you highly trust, and this is the default behavior of the
     # configuration.
     [snapshots.sources]
-        # Allow fetching of snapshots from peers specified in the gossip
-        # entrypoints.  Gossip entrypoints must be peers that are highly
-        # trusted as they are used to bootstrap your node.
-        [snapshots.sources.entrypoints]
-            enabled = false
+        # How old a local incremental snapshot on disk can be, in slots, and
+        # still be loaded on startup.  The age of a snapshot is determined
+        # by looking at the highest incremental snapshot slot available for
+        # download from cluster peers.
+        #
+        # If a local snapshot exists on disk, within the given age, other
+        # options are ignored and no snapshot will be downloaded.
+        #
+        # If the local snapshot is older than this, we will instead download
+        # a new snapshot.  If the local snapshot is too old, and we cannot
+        # download a snapshot because `download` is `false`, the validator
+        # will exit with an error.
+        #
+        # The local age parameters control whether or not we use locally
+        # stored full / incremental snapshots, if they exist.  Slot age
+        # is measured against the highest slot we could currently arrive at
+        # by downloading new full and incremental snapshots from any of
+        # our trusted sources.
+        #
+        # The effective usable slot of the local full snapshot is
+        # defined as the max slot of all trusted incremental snapshots
+        # that we could possibly use (local, or downloaded) which apply
+        # on top of this full snapshot.
+        max_local_full_effective_age = 1000
+        max_local_incremental_age = 200
+
+        # TODO-AM
+        # If any HTTP peers are listed, the following paths will be
+        # fetched from these peers to determine if they have a snapshot
+        # available:
+        #
+        #  /snapshot.tar.bz2
+        #  /incremental-snapshot.tar.bz2
+        #
+        # If these peers have snapshots at the given paths, they will be
+        # considered as additional sources for downloading snapshots
+        # when enabled alongside gossip and entrypoint sources.
+        # The url to fetch snapshots from.  The paths /snapshot.tar.bz2
+        # and /incremental-snapshot.tar.bz2 will be queried from this
+        # URL.
+        servers = []
 
         # Allow fetching of snapshots from arbitrary gossip peers that
         # are hosting a snapshot server.  This may allow faster snapshot
@@ -418,27 +392,48 @@ user = ""
         # serve you a specially crafted malicious snapshot with no way
         # to tell.
         [snapshots.sources.gossip]
-            enabled = false
+            # TODO-AM
+            allow_any = true
+
+            # TODO-AM
+            allow_list = []
+
+            # TODO-AM
+            # Do not attempt downloads from the given gossip pubkeys. This can
+            # be useful if a peer is frequently selected by the automatic logic
+            # but does not yield successful / usable snapshots.
+            block_list = []
+
+    # TODO-AM
+    [snapshots.download]
+        # The minimum acceptable speed for snapshot download from a peer, in
+        # megabits per second.  If the initial download speed falls below
+        # this threshold, the validator will abandon the download, find a
+        # new peer and start downloading the snapshot from them.
+        #
+        # The initial download speed is measured over every 10 mib for the
+        # first 400 mib of downloaded content.  The validator continues
+        # to measure download speed over every 10 mib until the download is
+        # complete, but only the initial download speed is used to decide
+        # whether to abort and retry the download.
+        # TODO: refine the initial download speed measurement and threshold.
+        #
+        # If set to zero, no minimum threshold will be applied.
+        min_speed_mib = 0
 
-        # If any HTTP peers are listed, the following paths will be
-        # fetched from these peers to determine if they have a snapshot
-        # available:
+        # The maximum number of times to abort and retry when encountering a
+        # slow snapshot download.  If the maximum attempts is reached while
+        # downloading a snapshot the validator will exit with an error.
         #
-        #  /snapshot.tar.bz2
-        #  /incremental-snapshot.tar.bz2
+        # The number of attempts is reset when starting a download for a
+        # new full or incremental snapshot.
         #
-        # If these peers have snapshots at the given paths, they will be
-        # considered as additional sources for downloading snapshots
-        # when enabled alongside gossip and entrypoint sources.
-        [[snapshots.sources.http]]
-            # Whether to enable fetching snapshots from a list of
-            # HTTP peers.
-            enabled = false
+        # If set to zero, the validator will continually retry until
+        # completing snapshot download.
+        max_retry_abort = 5
 
-            # The url to fetch snapshots from.  The paths /snapshot.tar.bz2
-            # and /incremental-snapshot.tar.bz2 will be queried from this
-            # URL.
-            url = "http://example.com:1234"
+    # TODO-AM
+    [snapshots.timeouts]
 
 [rpc]
     # If nonzero, enable JSON RPC on this port, and use the next port
@@ -544,10 +539,6 @@ user = ""
         # is not recommended to change this setting.
         mean_cache_entry_size = 131072
 
-# This section configures the "groove" persistent account database.
-# [groove]
-# ...
-
 [store]
     # Similar to max_pending_shred_sets, this parameter configures the
     # maximum number of shred sets that can be buffered.  However, this

src/app/firedancer/config/testnet.toml

@@ -8,7 +8,3 @@
     max_account_records = 200_000_000
 [layout]
     gossvf_tile_count = 4
-[snapshots]
-    [[snapshots.sources.http]]
-        enabled = true
-        url = "http://solana-testnet-rpc.jumpisolated.com:8899"

src/app/firedancer/topology.c

@@ -901,35 +901,33 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
   } else if( FD_UNLIKELY( !strcmp( tile->name, "snapct" ) ) ) {
 
     fd_memcpy( tile->snapct.snapshots_path, config->paths.snapshots, PATH_MAX );
-    tile->snapct.incremental_snapshot_fetch        = config->firedancer.snapshots.incremental_snapshots;
-    tile->snapct.do_download                       = config->firedancer.snapshots.download;
-    tile->snapct.maximum_local_snapshot_age        = config->firedancer.snapshots.maximum_local_snapshot_age;
-    tile->snapct.minimum_download_speed_mib        = config->firedancer.snapshots.minimum_download_speed_mib;
-    tile->snapct.maximum_download_retry_abort      = config->firedancer.snapshots.maximum_download_retry_abort;
-    tile->snapct.max_full_snapshots_to_keep        = config->firedancer.snapshots.max_full_snapshots_to_keep;
-    tile->snapct.max_incremental_snapshots_to_keep = config->firedancer.snapshots.max_incremental_snapshots_to_keep;
-
-    tile->snapct.entrypoints_enabled               = config->firedancer.snapshots.sources.entrypoints.enabled;
-    tile->snapct.gossip_peers_enabled              = config->firedancer.snapshots.sources.gossip.enabled;
-    tile->snapct.gossip_entrypoints_cnt            = config->gossip.entrypoints_cnt;
-
-    for( ulong i=0UL; i<tile->snapct.gossip_entrypoints_cnt; i++ ) tile->snapct.gossip_entrypoints[ i ] = config->gossip.resolved_entrypoints[ i ];
-
-    ulong peers_cnt          = config->firedancer.snapshots.sources.http.peers_cnt;
-    ulong resolved_peers_cnt = 0UL;
-
-    for( ulong j=0UL; j<peers_cnt; j++ ) {
-      if( FD_UNLIKELY( !config->firedancer.snapshots.sources.http.peers[ j ].enabled ) ) continue;
-
-      if( FD_UNLIKELY( 0==resolve_peer( config->firedancer.snapshots.sources.http.peers[ j ].url, &tile->snapct.http.peers[ resolved_peers_cnt ] ) ) ) {
-        FD_LOG_ERR(( "failed to resolve address of [snapshots.sources.http.peers] entry \"%s\"", config->firedancer.snapshots.sources.http.peers[ j ].url ));
-      } else {
-        resolved_peers_cnt++;
+    tile->snapct.sources.max_local_full_effective_age = config->firedancer.snapshots.sources.max_local_full_effective_age;
+    tile->snapct.sources.max_local_incremental_age    = config->firedancer.snapshots.sources.max_local_incremental_age;
+    tile->snapct.incremental_snapshots                = config->firedancer.snapshots.incremental_snapshots;
+    tile->snapct.max_full_snapshots_to_keep           = config->firedancer.snapshots.max_full_snapshots_to_keep;
+    tile->snapct.max_incremental_snapshots_to_keep    = config->firedancer.snapshots.max_incremental_snapshots_to_keep;
+    tile->snapct.full_effective_age_cancel_threshold  = config->firedancer.snapshots.full_effective_age_cancel_threshold;
+    tile->snapct.download.min_speed_mib               = config->firedancer.snapshots.download.min_speed_mib;
+    tile->snapct.download.max_retry_abort             = config->firedancer.snapshots.download.max_retry_abort;
+    tile->snapct.sources.gossip.allow_any             = config->firedancer.snapshots.sources.gossip.allow_any;
+    tile->snapct.sources.gossip.allow_list_cnt        = config->firedancer.snapshots.sources.gossip.allow_list_cnt;
+    tile->snapct.sources.gossip.block_list_cnt        = config->firedancer.snapshots.sources.gossip.block_list_cnt;
+    tile->snapct.sources.servers_cnt                  = config->firedancer.snapshots.sources.servers_cnt;
+    for( ulong i=0UL; i<tile->snapct.sources.gossip.allow_list_cnt; i++ ) {
+      if( FD_UNLIKELY( !fd_base58_decode_32( config->firedancer.snapshots.sources.gossip.allow_list[ i ], tile->snapct.sources.gossip.allow_list[ i ].uc ) ) ) {
+        FD_LOG_ERR(( "[snapshots.sources.gossip.allow_list[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.gossip.allow_list[ i ] ));
+      }
+    }
+    for( ulong i=0UL; i<tile->snapct.sources.gossip.block_list_cnt; i++ ) {
+      if( FD_UNLIKELY( !fd_base58_decode_32( config->firedancer.snapshots.sources.gossip.block_list[ i ], tile->snapct.sources.gossip.block_list[ i ].uc ) ) ) {
+        FD_LOG_ERR(( "[snapshots.sources.gossip.block_list[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.gossip.block_list[ i ] ));
+      }
+    }
+    for( ulong i=0UL; i<tile->snapct.sources.servers_cnt; i++ ) {
+      if( FD_UNLIKELY( !resolve_peer( config->firedancer.snapshots.sources.servers[ i ], &tile->snapct.sources.servers[ i ] ) ) ) {
+        FD_LOG_ERR(( "[snapshots.sources.servers[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.servers[ i ] ));
       }
     }
-
-    tile->snapct.http.peers_cnt = resolved_peers_cnt;
-    /* TODO: set up known validators and known validators cnt */
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "snapld" ) ) ) {
 

src/app/shared/fd_config.h

@@ -11,7 +11,6 @@
 #define AFFINITY_SZ                      (256UL)
 #define CONFIGURE_STAGE_COUNT            ( 12UL)
 #define GOSSIP_TILE_ENTRYPOINTS_MAX      ( 16UL)
-#define SNAPSHOT_TILE_HTTP_PEERS_MAX     ( 16UL)
 #define IP4_PORT_STR_MAX                 ( 22UL)
 
 struct fd_configh {
@@ -124,37 +123,33 @@ struct fd_configf {
   } gossip;
 
   struct {
-
     struct {
+      uint max_local_full_effective_age;
+      uint max_local_incremental_age;
 
       struct {
-        int enabled;
-      } entrypoints;
-
-      struct {
-        int enabled;
+        int   allow_any;
+        ulong allow_list_cnt;
+        char  allow_list[ 16 ][ FD_BASE58_ENCODED_32_SZ ];
+        ulong block_list_cnt;
+        char  block_list[ 16 ][ FD_BASE58_ENCODED_32_SZ ];
       } gossip;
 
-      struct {
-        ulong            peers_cnt;
-        struct {
-          int  enabled;
-          char url[ PATH_MAX ];
-        } peers[ SNAPSHOT_TILE_HTTP_PEERS_MAX ];
-      } http;
-
+      ulong servers_cnt;
+      char  servers[ 16 ][ 128 ];
     } sources;
 
-    int   incremental_snapshots;
-    uint  maximum_local_snapshot_age;
-    int   genesis_download;
-    int   download;
-    ulong known_validators_cnt;
-    char  known_validators[ 16 ][ 256 ];
-    uint  minimum_download_speed_mib;
-    uint  maximum_download_retry_abort;
-    uint  max_full_snapshots_to_keep;
-    uint  max_incremental_snapshots_to_keep;
+    int  incremental_snapshots;
+    int  genesis_download;
+    uint max_full_snapshots_to_keep;
+    uint max_incremental_snapshots_to_keep;
+    uint full_effective_age_cancel_threshold;
+
+    struct {
+      uint min_speed_mib;
+      uint max_retry_abort;
+    } download;
+
   } snapshots;
 
   struct {