Skip to content

[WIP] snapshots: restructure config options #6832

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+389 −393
Draft

[WIP] snapshots: restructure config options #6832

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions contrib/offline-replay/offline_replay.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,4 @@
maximum_download_retry_abort = 0
[gossip]
entrypoints = [ "0.0.0.0:1" ]
#TODO-AM
2 changes: 1 addition & 1 deletion src/app/firedancer-dev/commands/snapshot_load.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ extern int * fd_log_private_shared_lock;
static void
snapshot_load_cmd_fn( args_t * args,
config_t * config ) {
if( FD_UNLIKELY( config->firedancer.snapshots.sources.gossip.enabled ) ) {
if( FD_UNLIKELY( config->firedancer.snapshots.sources.gossip.allow_any || 0UL!=config->firedancer.snapshots.sources.gossip.allow_list_cnt ) ) {
FD_LOG_ERR(( "snapshot-load command is incompatible with gossip snapshot sources" ));
}
snapshot_load_topo( config, args );
Expand Down
183 changes: 90 additions & 93 deletions src/app/firedancer/config/default.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ user = ""
# specified in `[net.interface]`.
host = ""

#TODO-AM: Tweak all the wording
# Snapshots are a periodic view of the ledger at a point in time. They
# are used to enable validators to join the network quickly, as they do
# not need to replay all transactions since genesis, just those since a
Expand All @@ -280,20 +281,6 @@ user = ""
# a validator to catch up more quickly.
incremental_snapshots = true

# How old a local incremental snapshot on disk can be, in slots, and
# still be loaded on startup. The age of a snapshot is determined
# by looking at the highest incremental snapshot slot available for
# download from cluster peers.
#
# If a local snapshot exists on disk, within the given age, other
# options are ignored and no snapshot will be downloaded.
#
# If the local snapshot is older than this, we will instead download
# a new snapshot. If the local snapshot is too old, and we cannot
# download a snapshot because `download` is `false`, the validator
# will exit with an error.
maximum_local_snapshot_age = 2500

# Whether to allow downloading a new genesis file from a peer when
# booting. If set to false, the validator will only reuse a locally
# stored genesis file, and if one does not exist, the validator will
Expand All @@ -313,64 +300,6 @@ user = ""
# will exit with an error.
genesis_download = true

# Whether to allow downloading a new snapshot from a peer when
# booting.
#
# If set to true, the validator will contact peers from the gossip
# network to locate both a full and incremental snapshot and
# download it, assuming there is no loadable local snapshot. If
# false, the validator will attempt to load the most recently
# downloaded snapshot that exists locally on the disk. If there is
# no local snapshot, the validator will abort with an error.
#
# When downloading a snapshot, it will optionally be stored on disk
# if `[paths.snapshots]` is specified. The number of downloaded
# snapshots that will be stored on disk is controlled below by the
# max_*_snapshots to keep options.
#
# TODO: implement the "at most one downloaded" logic. "
download = true

# A snapshot hash must be published in gossip by one of the
# validator identities listed here for it to be accepted. The list
# should be a list of identity public keys, specified as base58
# strings.
#
# If a snapshot hash is received that is not published by a known
# validator the validator will exit with an error.
#
# If no known validators are specified here, any snapshot hash will
# be accepted.
#
# TODO: Implement this once gossip is wired up.
known_validators = []

# The minimum acceptable speed for snapshot download from a peer, in
# megabits per second. If the initial download speed falls below
# this threshold, the validator will abandon the download, find a
# new peer and start downloading the snapshot from them.
#
# The initial download speed is measured over every 10 mib for the
# first 400 mib of downloaded content. The validator continues
# to measure download speed over every 10 mib until the download is
# complete, but only the initial download speed is used to decide
# whether to abort and retry the download.
# TODO: refine the initial download speed measurement and threshold.
#
# If set to zero, no minimum threshold will be applied.
minimum_download_speed_mib = 0

# The maximum number of times to abort and retry when encountering a
# slow snapshot download. If the maximum attempts is reached while
# downloading a snapshot the validator will exit with an error.
#
# The number of attempts is reset when starting a download for a
# new full or incremental snapshot.
#
# If set to zero, the validator will continually retry until
# completing snapshot download.
maximum_download_retry_abort = 5

# Controls how many full snapshots are allowed to be kept in the
# snapshots directory on disk.
#
Expand All @@ -380,6 +309,15 @@ user = ""
max_full_snapshots_to_keep = 1
max_incremental_snapshots_to_keep = 1

# If an in-progress loading (local, or downloaded) of a full
# snapshot has its effective age rise above this threshold, we will
# cancel the load and start over. The most common reason this would
# happen is if the network is all using the same full snapshot
# interval (e.g. 100_000) and while we are downloading a full
# snapshot from interval N, the network moves on to N+1, removing
# availability of all incremental snapshots that build on N.
full_effective_age_cancel_threshold = 20_000

# Controls for where we look to find snapshots. If multiple sources
# and peers are available, the one with the lowest latency and
# fastest download will be automatically selected among them.
Expand All @@ -400,34 +338,97 @@ user = ""
# you highly trust, and this is the default behavior of the
# configuration.
[snapshots.sources]
# How old a local incremental snapshot on disk can be, in slots, and
# still be loaded on startup. The age of a snapshot is determined
# by looking at the highest incremental snapshot slot available for
# download from cluster peers.
#
# If a local snapshot exists on disk, within the given age, other
# options are ignored and no snapshot will be downloaded.
#
# If the local snapshot is older than this, we will instead download
# a new snapshot. If the local snapshot is too old, and we cannot
# download a snapshot because `download` is `false`, the validator
# will exit with an error.
#
# The local age parameters control whether or not we use locally
# stored full / incremental snapshots, if they exist. Slot age
# is measured against the highest slot we could currently arrive at
# by downloading new full and incremental snapshots from any of
# our trusted sources.
#
# The effective usable slot of the local full snapshot is
# defined as the max slot of all trusted incremental snapshots
# that we could possibly use (local, or downloaded) which apply
# on top of this full snapshot.
max_local_full_effective_age = 1000
max_local_incremental_age = 200

# TODO-AM
# If any HTTP peers are listed, the following paths will be
# fetched from these peers to determine if they have a snapshot
# available:
#
# /snapshot.tar.bz2
# /incremental-snapshot.tar.bz2
#
# If these peers have snapshots at the given paths, they will be
# considered as additional sources for downloading snapshots
# when enabled alongside gossip and entrypoint sources.
# The url to fetch snapshots from. The paths /snapshot.tar.bz2
# and /incremental-snapshot.tar.bz2 will be queried from this
# URL.
servers = []

# Allow fetching of snapshots from arbitrary gossip peers that
# are hosting a snapshot server. This may allow faster snapshot
# download if a peer in a local data-center is serving a good
# snapshot, but it is extremely dangerous as the peer could
# serve you a specially crafted malicious snapshot with no way
# to tell.
[snapshots.sources.gossip]
enabled = false
# TODO-AM
allow_any = true

# TODO-AM
allow_list = []

# TODO-AM
# Do not attempt downloads from the given gossip pubkeys. This can
# be useful if a peer is frequently selected by the automatic logic
# but does not yield successful / usable snapshots.
block_list = []

# TODO-AM
[snapshots.download]
# The minimum acceptable speed for snapshot download from a peer, in
# megabits per second. If the initial download speed falls below
# this threshold, the validator will abandon the download, find a
# new peer and start downloading the snapshot from them.
#
# The initial download speed is measured over every 10 mib for the
# first 400 mib of downloaded content. The validator continues
# to measure download speed over every 10 mib until the download is
# complete, but only the initial download speed is used to decide
# whether to abort and retry the download.
# TODO: refine the initial download speed measurement and threshold.
#
# If set to zero, no minimum threshold will be applied.
min_speed_mib = 0

# If any HTTP peers are listed, the following paths will be
# fetched from these peers to determine if they have a snapshot
# available:
# The maximum number of times to abort and retry when encountering a
# slow snapshot download. If the maximum attempts is reached while
# downloading a snapshot the validator will exit with an error.
#
# /snapshot.tar.bz2
# /incremental-snapshot.tar.bz2
# The number of attempts is reset when starting a download for a
# new full or incremental snapshot.
#
# If these peers have snapshots at the given paths, they will be
# considered as additional sources for downloading snapshots
# when enabled alongside gossip sources.
[[snapshots.sources.http]]
# Whether to enable fetching snapshots from a list of
# HTTP peers.
enabled = false
# If set to zero, the validator will continually retry until
# completing snapshot download.
max_retry_abort = 5

# The url to fetch snapshots from. The paths /snapshot.tar.bz2
# and /incremental-snapshot.tar.bz2 will be queried from this
# URL.
url = "http://example.com:1234"
# TODO-AM
[snapshots.timeouts]

[rpc]
# If nonzero, enable JSON RPC on this port, and use the next port
Expand Down Expand Up @@ -520,10 +521,6 @@ user = ""
# is not recommended to change this setting.
mean_cache_entry_size = 131072

# This section configures the "groove" persistent account database.
# [groove]
# ...

[store]
# Similar to max_pending_shred_sets, this parameter configures the
# maximum number of shred sets that can be buffered. However, this
Expand Down
4 changes: 0 additions & 4 deletions src/app/firedancer/config/testnet.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,3 @@
max_account_records = 200_000_000
[layout]
gossvf_tile_count = 4
[snapshots]
[[snapshots.sources.http]]
enabled = true
url = "http://solana-testnet-rpc.jumpisolated.com:8899"
47 changes: 25 additions & 22 deletions src/app/firedancer/topology.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -908,30 +908,33 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
} else if( FD_UNLIKELY( !strcmp( tile->name, "snapct" ) ) ) {

fd_memcpy( tile->snapct.snapshots_path, config->paths.snapshots, PATH_MAX );
tile->snapct.incremental_snapshot_fetch = config->firedancer.snapshots.incremental_snapshots;
tile->snapct.do_download = config->firedancer.snapshots.download;
tile->snapct.maximum_local_snapshot_age = config->firedancer.snapshots.maximum_local_snapshot_age;
tile->snapct.minimum_download_speed_mib = config->firedancer.snapshots.minimum_download_speed_mib;
tile->snapct.maximum_download_retry_abort = config->firedancer.snapshots.maximum_download_retry_abort;
tile->snapct.max_full_snapshots_to_keep = config->firedancer.snapshots.max_full_snapshots_to_keep;
tile->snapct.max_incremental_snapshots_to_keep = config->firedancer.snapshots.max_incremental_snapshots_to_keep;
tile->snapct.gossip_peers_enabled = config->firedancer.snapshots.sources.gossip.enabled;

ulong peers_cnt = config->firedancer.snapshots.sources.http.peers_cnt;
ulong resolved_peers_cnt = 0UL;

for( ulong j=0UL; j<peers_cnt; j++ ) {
if( FD_UNLIKELY( !config->firedancer.snapshots.sources.http.peers[ j ].enabled ) ) continue;

if( FD_UNLIKELY( 0==resolve_peer( config->firedancer.snapshots.sources.http.peers[ j ].url, &tile->snapct.http.peers[ resolved_peers_cnt ] ) ) ) {
FD_LOG_ERR(( "failed to resolve address of [snapshots.sources.http.peers] entry \"%s\"", config->firedancer.snapshots.sources.http.peers[ j ].url ));
} else {
resolved_peers_cnt++;
tile->snapct.sources.max_local_full_effective_age = config->firedancer.snapshots.sources.max_local_full_effective_age;
tile->snapct.sources.max_local_incremental_age = config->firedancer.snapshots.sources.max_local_incremental_age;
tile->snapct.incremental_snapshots = config->firedancer.snapshots.incremental_snapshots;
tile->snapct.max_full_snapshots_to_keep = config->firedancer.snapshots.max_full_snapshots_to_keep;
tile->snapct.max_incremental_snapshots_to_keep = config->firedancer.snapshots.max_incremental_snapshots_to_keep;
tile->snapct.full_effective_age_cancel_threshold = config->firedancer.snapshots.full_effective_age_cancel_threshold;
tile->snapct.download.min_speed_mib = config->firedancer.snapshots.download.min_speed_mib;
tile->snapct.download.max_retry_abort = config->firedancer.snapshots.download.max_retry_abort;
tile->snapct.sources.gossip.allow_any = config->firedancer.snapshots.sources.gossip.allow_any;
tile->snapct.sources.gossip.allow_list_cnt = config->firedancer.snapshots.sources.gossip.allow_list_cnt;
tile->snapct.sources.gossip.block_list_cnt = config->firedancer.snapshots.sources.gossip.block_list_cnt;
tile->snapct.sources.servers_cnt = config->firedancer.snapshots.sources.servers_cnt;
for( ulong i=0UL; i<tile->snapct.sources.gossip.allow_list_cnt; i++ ) {
if( FD_UNLIKELY( !fd_base58_decode_32( config->firedancer.snapshots.sources.gossip.allow_list[ i ], tile->snapct.sources.gossip.allow_list[ i ].uc ) ) ) {
FD_LOG_ERR(( "[snapshots.sources.gossip.allow_list[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.gossip.allow_list[ i ] ));
}
}
for( ulong i=0UL; i<tile->snapct.sources.gossip.block_list_cnt; i++ ) {
if( FD_UNLIKELY( !fd_base58_decode_32( config->firedancer.snapshots.sources.gossip.block_list[ i ], tile->snapct.sources.gossip.block_list[ i ].uc ) ) ) {
FD_LOG_ERR(( "[snapshots.sources.gossip.block_list[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.gossip.block_list[ i ] ));
}
}
for( ulong i=0UL; i<tile->snapct.sources.servers_cnt; i++ ) {
if( FD_UNLIKELY( !resolve_peer( config->firedancer.snapshots.sources.servers[ i ], &tile->snapct.sources.servers[ i ] ) ) ) {
FD_LOG_ERR(( "[snapshots.sources.servers[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.servers[ i ] ));
}
}

tile->snapct.http.peers_cnt = resolved_peers_cnt;
/* TODO: set up known validators and known validators cnt */

} else if( FD_UNLIKELY( !strcmp( tile->name, "snapld" ) ) ) {

Expand Down
41 changes: 20 additions & 21 deletions src/app/shared/fd_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
#define AFFINITY_SZ (256UL)
#define CONFIGURE_STAGE_COUNT ( 12UL)
#define GOSSIP_TILE_ENTRYPOINTS_MAX ( 16UL)
#define SNAPSHOT_TILE_HTTP_PEERS_MAX ( 16UL)
#define IP4_PORT_STR_MAX ( 22UL)

struct fd_configh {
Expand Down Expand Up @@ -123,33 +122,33 @@ struct fd_configf {
} gossip;

struct {

struct {
uint max_local_full_effective_age;
uint max_local_incremental_age;

struct {
int enabled;
int allow_any;
ulong allow_list_cnt;
char allow_list[ 16 ][ FD_BASE58_ENCODED_32_SZ ];
ulong block_list_cnt;
char block_list[ 16 ][ FD_BASE58_ENCODED_32_SZ ];
} gossip;

struct {
ulong peers_cnt;
struct {
int enabled;
char url[ PATH_MAX ];
} peers[ SNAPSHOT_TILE_HTTP_PEERS_MAX ];
} http;

ulong servers_cnt;
char servers[ 16 ][ 128 ];
} sources;

int incremental_snapshots;
uint maximum_local_snapshot_age;
int genesis_download;
int download;
ulong known_validators_cnt;
char known_validators[ 16 ][ 256 ];
uint minimum_download_speed_mib;
uint maximum_download_retry_abort;
uint max_full_snapshots_to_keep;
uint max_incremental_snapshots_to_keep;
int incremental_snapshots;
int genesis_download;
uint max_full_snapshots_to_keep;
uint max_incremental_snapshots_to_keep;
uint full_effective_age_cancel_threshold;

struct {
uint min_speed_mib;
uint max_retry_abort;
} download;

} snapshots;

struct {
Expand Down
Loading
Loading