Skip to content

Commit 04374b0

Browse files
committed
fix(tower): stem burst and pr comments
1 parent de56887 commit 04374b0

File tree

17 files changed

+149
-140
lines changed

17 files changed

+149
-140
lines changed

book/api/metrics-generated.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1135,6 +1135,9 @@
11351135

11361136
| Metric | Type | Description |
11371137
|--------|------|-------------|
1138+
| <span class="metrics-name">tower_&#8203;vote_&#8203;txn_&#8203;invalid</span> | counter | Number of times we dropped a vote txn because it was invalid (malformed, bad signature, etc.) |
1139+
| <span class="metrics-name">tower_&#8203;vote_&#8203;txn_&#8203;ignored</span> | counter | Number of times we ignored all or part of a vote txn because we didn't recognize a slot (eg. our replay was behind) |
1140+
| <span class="metrics-name">tower_&#8203;vote_&#8203;txn_&#8203;mismatch</span> | counter | Number of times a vote txn mismatched our own block id |
11381141
| <span class="metrics-name">tower_&#8203;ancestor_&#8203;rollback</span> | counter | Rollback to an ancestor of our prev vote (can't vote) |
11391142
| <span class="metrics-name">tower_&#8203;sibling_&#8203;confirmed</span> | counter | Duplicate sibling got confirmed (can't vote) |
11401143
| <span class="metrics-name">tower_&#8203;same_&#8203;fork</span> | counter | Same fork as prev vote (can vote) |

src/app/firedancer/config/default.toml

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -472,15 +472,6 @@ user = ""
472472
# abort with an error.
473473
expected_genesis_hash = ""
474474

475-
# Firedancer can process at most this many slots without rooting in
476-
# the consensus rules before it must begin evicting.
477-
#
478-
# This is an estimate and should be set as generously as possible to
479-
# allow for brief anomalies such as the validator disconnecting from
480-
# part of the cluster due to data center issues. Roughly, at 400 ms
481-
# per slot, the default allows for 30 minutes without rooting.
482-
max_unrooted_slots = 4096
483-
484475
# This section configures the "funk" account database. Currently, funk
485476
# stores all Solana accounts. In future versions of Firedancer, most
486477
# accounts will be offloaded to the "groove" database.
@@ -1370,6 +1361,27 @@ user = ""
13701361
# TODO: What is this ... needs to be deleted
13711362
cluster_version = "1.18.0"
13721363

1364+
# The tower tile runs the fork choice and tower rules to determine
1365+
# both what block to vote on and what block to build our own leader
1366+
# blocks on top of.
1367+
[tiles.tower]
1368+
# Firedancer can process at most this many slots without rooting
1369+
# in the consensus rules before it must begin evicting.
1370+
#
1371+
# This is an estimate and should be set as generously as
1372+
# possible to allow for temporary outages such as network
1373+
# partitions. For example, the validator might get disconnected
1374+
# from part of the cluster due to data center issues. Roughly,
1375+
# the default of 4096 allows for 30 minutes without rooting.
1376+
#
1377+
# Specifically, tower will ignore gossip votes that exceed max
1378+
# unrooted slots ahead of the current root. Additionally, both
1379+
# fork choice and tower structures will OOM and cause Firedancer
1380+
# to exit if it needs to maintain more than max unrooted slots
1381+
# tower forks (TODO in the future Firedancer will instead
1382+
# gracefully degrade by evicting forks).
1383+
max_unrooted_slots = 4096
1384+
13731385
[tiles.send]
13741386
# The port the send tile uses for QUIC, to send votes and other
13751387
# transactions. It also uses this as the UDP src port.

src/app/firedancer/topology.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,7 @@ fd_topo_initialize( config_t * config ) {
395395

396396
/* TODO: Explain this .... USHORT_MAX is not dcache max */
397397
ulong pending_fec_shreds_depth = fd_ulong_min( fd_ulong_pow2_up( config->tiles.shred.max_pending_shred_sets * FD_REEDSOL_DATA_SHREDS_MAX ), USHORT_MAX + 1 /* dcache max */ );
398-
ulong max_unrooted_slots = config->firedancer.consensus.max_unrooted_slots;
398+
ulong max_unrooted_slots = config->tiles.tower.max_unrooted_slots;
399399

400400
/* topo, link_name, wksp_name, depth, mtu, burst */
401401
/**/ fd_topob_link( topo, "gossip_net", "net_gossip", 32768UL, FD_NET_MTU, 1UL );
@@ -1164,7 +1164,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
11641164

11651165
} else if( FD_UNLIKELY( !strcmp( tile->name, "tower" ) ) ) {
11661166

1167-
tile->tower.slot_max = config->firedancer.consensus.max_unrooted_slots;
1167+
tile->tower.slot_max = config->tiles.tower.max_unrooted_slots;
11681168
strncpy( tile->tower.identity_key, config->paths.identity_key, sizeof(tile->tower.identity_key) );
11691169
strncpy( tile->tower.vote_account, config->paths.vote_account, sizeof(tile->tower.vote_account) );
11701170
strncpy( tile->tower.base_path, config->paths.base, sizeof(tile->tower.base_path) );

src/app/shared/fd_config.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,6 @@ struct fd_configf {
132132
char host[ 256 ];
133133
} gossip;
134134

135-
struct {
136-
ulong max_unrooted_slots;
137-
} consensus;
138-
139135
struct {
140136
struct {
141137
uint max_local_full_effective_age;
@@ -481,6 +477,10 @@ struct fd_config {
481477
ulong write_buffer_size;
482478
} shredcap;
483479

480+
struct {
481+
ulong max_unrooted_slots;
482+
} tower;
483+
484484
} tiles;
485485
struct {
486486
ulong capture_start_slot;

src/app/shared/fd_config_parse.c

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,6 @@ fd_config_extract_podf( uchar * pod,
8080
fd_configf_t * config ) {
8181
CFG_POP ( cstr, gossip.host );
8282

83-
CFG_POP ( ulong, consensus.max_unrooted_slots );
84-
8583
CFG_POP ( uint, layout.exec_tile_count );
8684
CFG_POP ( uint, layout.sign_tile_count );
8785
CFG_POP ( uint, layout.gossvf_tile_count );
@@ -255,13 +253,10 @@ fd_config_extract_pod( uchar * pod,
255253
CFG_POP ( cstr, tiles.replay.cluster_version );
256254
CFG_POP_ARRAY( cstr, tiles.replay.enable_features );
257255

258-
CFG_POP ( cstr, tiles.store_int.slots_pending );
259-
CFG_POP ( cstr, tiles.store_int.shred_cap_archive );
260-
CFG_POP ( cstr, tiles.store_int.shred_cap_replay );
261-
CFG_POP ( ulong, tiles.store_int.shred_cap_end_slot );
262-
263256
CFG_POP ( ushort, tiles.send.send_src_port );
264257

258+
CFG_POP ( ulong, tiles.tower.max_unrooted_slots );
259+
265260
CFG_POP ( bool, tiles.archiver.enabled );
266261
CFG_POP ( ulong, tiles.archiver.end_slot );
267262
CFG_POP ( cstr, tiles.archiver.rocksdb_path );

src/choreo/notar/fd_notar.c

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,6 @@ fd_notar_count_vote( fd_notar_t * notar,
101101
ulong vote_slot,
102102
fd_hash_t const * vote_block_id ) {
103103

104-
if( FD_UNLIKELY( !notar ) ) { FD_LOG_WARNING(( "NULL notar" )); return NULL; }
105-
106104
/* Ignore if this vote slot isn't in range. */
107105

108106
if( FD_UNLIKELY( vote_slot < notar->lo_wmark || vote_slot > notar->hi_wmark ) ) return NULL;
@@ -138,10 +136,14 @@ fd_notar_count_vote( fd_notar_t * notar,
138136

139137
fd_notar_blk_t * notar_blk = fd_notar_blk_query( notar->blk_map, *vote_block_id, NULL );
140138
if( FD_UNLIKELY( !notar_blk ) ) {
141-
notar_blk = fd_notar_blk_insert( notar->blk_map, *vote_block_id );
142-
notar_blk->slot = vote_slot;
143-
notar_blk->stake = 0;
144139
FD_TEST( notar_slot->block_ids_cnt < FD_VOTER_MAX ); /* at most one unique block id per voter in a slot */
140+
notar_blk = fd_notar_blk_insert( notar->blk_map, *vote_block_id );
141+
notar_blk->slot = vote_slot;
142+
notar_blk->stake = 0;
143+
notar_blk->dup_conf = 0;
144+
notar_blk->opt_conf = 0;
145+
notar_blk->dup_notif = 0;
146+
notar_blk->opt_notif = 0;
145147
notar_slot->block_ids[notar_slot->block_ids_cnt++] = *vote_block_id;
146148
}
147149
notar_blk->stake += vtr->stake;

src/choreo/notar/fd_notar.h

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,12 +110,14 @@ struct fd_notar_slot {
110110
typedef struct fd_notar_slot fd_notar_slot_t;
111111

112112
struct fd_notar_blk {
113-
fd_hash_t block_id; /* map key */
114-
uint hash; /* reserved for fd_map_dynamic */
115-
ulong slot; /* slot associated with this block */
116-
ulong stake; /* sum of stake that has voted for this block_id */
117-
int dup_conf; /* whether this block has reached 52% of stake */
118-
int opt_conf; /* whether this block has reached 2/3 of stake */
113+
fd_hash_t block_id; /* map key */
114+
uint hash; /* reserved for fd_map_dynamic */
115+
ulong slot; /* slot associated with this block */
116+
ulong stake; /* sum of stake that has voted for this block_id */
117+
int dup_conf; /* whether this block has reached 52% of stake */
118+
int opt_conf; /* whether this block has reached 2/3 of stake */
119+
int dup_notif; /* set by caller, whether we've notified consumers this is duplicate confirmed */
120+
int opt_notif; /* set by caller, whether we've notified consumers this is optimistically confirmed */
119121
};
120122
typedef struct fd_notar_blk fd_notar_blk_t;
121123

src/choreo/tower/fd_tower_forks.c

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,8 @@ fd_tower_forks_lowest_common_ancestor( fd_tower_forks_t * forks,
4242
}
4343

4444
fd_hash_t const *
45-
fd_tower_forks_canonical_block_id( fd_tower_forks_t * forks,
46-
ulong slot ) {
47-
fd_tower_forks_t * fork = fd_tower_forks_query( forks, slot, NULL );
48-
if( FD_UNLIKELY( !fork ) ) return NULL;
45+
fd_tower_forks_canonical_block_id( fd_tower_forks_t * fork ) {
4946
if ( FD_LIKELY( fork->confirmed ) ) return &fork->confirmed_block_id;
5047
else if( FD_LIKELY( fork->voted ) ) return &fork->voted_block_id;
51-
else if( FD_LIKELY( fork->replayed ) ) return &fork->replayed_block_id;
52-
else return NULL;
48+
else return &fork->replayed_block_id;
5349
}

src/choreo/tower/fd_tower_forks.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ struct fd_tower_forks {
3131
fd_hash_t confirmed_block_id; /* the block_id that was duplicate confirmed */
3232
int voted; /* whether we voted for this slot yet */
3333
fd_hash_t voted_block_id; /* the block_id we voted on for this slot */
34-
int replayed; /* whether we replayed this slot yet */
3534
fd_hash_t replayed_block_id; /* the block_id we _first_ replayed for this slot */
35+
ulong bank_idx; /* pool idx of the bank as of this replayed block */
3636
};
3737
typedef struct fd_tower_forks fd_tower_forks_t;
3838

@@ -78,8 +78,7 @@ fd_tower_forks_lowest_common_ancestor( fd_tower_forks_t * forks,
7878
so they're always comparing the confirmed block id */
7979

8080
fd_hash_t const *
81-
fd_tower_forks_canonical_block_id( fd_tower_forks_t * forks,
82-
ulong slot );
81+
fd_tower_forks_canonical_block_id( fd_tower_forks_t * fork );
8382

8483
FD_PROTOTYPES_END
8584

src/choreo/tower/fd_tower_serde.c

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
#define SHORTVEC 0
55

66
#define DE( T, name ) do { \
7-
if( FD_UNLIKELY( off+sizeof(T)>buf_sz ) ) { \
8-
FD_LOG_WARNING(( "de %s: overflow (off %lu > buf_sz: %lu)", #name, off, buf_sz )); \
9-
return -1; \
10-
} \
7+
if( FD_UNLIKELY( off+sizeof(T)>buf_sz ) ) return -1; \
118
serde->name = *(T const *)fd_type_pun_const( buf+off ); \
129
off += sizeof(T); \
1310
} while(0)
@@ -45,11 +42,8 @@ fd_compact_tower_sync_deserialize( fd_compact_tower_sync_serde_t * serde,
4542
ulong off = 0;
4643
DE( ulong, root );
4744
off += de_short_u16( &serde->lockouts_cnt, buf+off );
48-
if( FD_UNLIKELY( serde->lockouts_cnt > FD_TOWER_VOTE_MAX ) ) {
49-
FD_LOG_WARNING(( "lockouts_cnt > 31: %u", serde->lockouts_cnt ));
50-
return -1;
51-
}
52-
for( ulong i = 0; i < fd_ulong_min( serde->lockouts_cnt, 31 ); i++ ) {
45+
if( FD_UNLIKELY( serde->lockouts_cnt > FD_TOWER_VOTE_MAX ) ) return -1;
46+
for( ulong i = 0; i < serde->lockouts_cnt; i++ ) {
5347
off += de_var_int( &serde->lockouts[i].offset, buf+off );
5448
DE( uchar, lockouts[i].confirmation_count );
5549
}

0 commit comments

Comments
 (0)