Skip to content

repair: dump and retry API #6560

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

repair: dump and retry API #6560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c49abec
forest: dump slot API
emwang-jump Sep 29, 2025
fda2b0d
repair: add retry capabilities
emwang-jump Sep 30, 2025
488897d
store: initialize cmr on insert
emwang-jump Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/disco/shred/fd_shred_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1036,7 +1036,9 @@ after_frag( fd_shred_ctx_t * ctx,
long shacq_start, shacq_end, shrel_end;
fd_store_fec_t * fec = NULL;
FD_STORE_SHARED_LOCK( ctx->store, shacq_start, shacq_end, shrel_end ) {
fec = fd_store_insert( ctx->store, ctx->round_robin_id, (fd_hash_t *)fd_type_pun( &ctx->out_merkle_roots[fset_k] ) );
fec = fd_store_insert( ctx->store, ctx->round_robin_id,
(fd_hash_t *)fd_type_pun( &ctx->out_merkle_roots[fset_k] ),
(fd_hash_t *)fd_type_pun( ((uchar *)last + fd_shred_chain_off( last->variant )) ) );
} FD_STORE_SHARED_LOCK_END;

for( ulong i=0UL; i<set->data_shred_cnt; i++ ) {
Expand Down
6 changes: 3 additions & 3 deletions src/disco/store/fd_store.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#include "fd_store.h"
#include "../../flamenco/fd_flamenco_base.h"

static const fd_hash_t hash_null = { 0 };

#define null fd_store_pool_idx_null()

Expand Down Expand Up @@ -146,7 +145,8 @@ fd_store_delete( void * shstore ) {
fd_store_fec_t *
fd_store_insert( fd_store_t * store,
ulong part_idx,
fd_hash_t * merkle_root ) {
fd_hash_t * merkle_root,
fd_hash_t * chained_merkle ) {

# if FD_STORE_USE_HANDHOLDING
if( FD_UNLIKELY( fd_store_query_const( store, merkle_root ) ) ) { FD_LOG_WARNING(( "merkle root %s already in store", FD_BASE58_ENC_32_ALLOCA( merkle_root ) )); return NULL; }
Expand All @@ -162,7 +162,7 @@ fd_store_insert( fd_store_t * store,

fec->key.mr = *merkle_root;
fec->key.part = part_idx;
fec->cmr = hash_null;
fec->cmr = *chained_merkle;
fec->next = null;
fec->parent = null;
fec->child = null;
Expand Down
21 changes: 11 additions & 10 deletions src/disco/store/fd_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,16 +216,16 @@ typedef struct fd_store_fec fd_store_fec_t;
#include "../../util/tmpl/fd_map_chain.c"

struct fd_store {
ulong magic; /* ==FD_STORE_MAGIC */
ulong fec_max; /* max number of FEC sets that can be stored */
ulong part_cnt; /* number of partitions, also the number of writers */
ulong root; /* pool idx of the root */
ulong slot0; /* FIXME this hack is needed until the block_id is in the bank (manifest) */
ulong store_gaddr; /* wksp gaddr of store in the backing wksp, non-zero gaddr */
ulong map_gaddr; /* wksp gaddr of map of fd_store_key->fd_store_fec */
ulong magic; /* ==FD_STORE_MAGIC */
ulong fec_max; /* max number of FEC sets that can be stored */
ulong part_cnt; /* number of partitions, also the number of writers */
ulong root; /* pool idx of the root */
ulong slot0; /* FIXME this hack is needed until the block_id is in the bank (manifest) */
ulong store_gaddr; /* wksp gaddr of store in the backing wksp, non-zero gaddr */
ulong map_gaddr; /* wksp gaddr of map of fd_store_key->fd_store_fec */
ulong pool_mem_gaddr; /* wksp gaddr of shmem_t object in pool_para */
ulong pool_ele_gaddr; /* wksp gaddr of first ele_t object in pool_para */
fd_rwlock_t lock; /* rwlock for concurrent access */
fd_rwlock_t lock; /* rwlock for concurrent access */
};
typedef struct fd_store fd_store_t;

Expand Down Expand Up @@ -421,7 +421,7 @@ fd_store_query( fd_store_t * store, fd_hash_t const * merkle_root ) {
}

FD_FN_PURE static inline fd_store_fec_t const *
fd_store_query_const( fd_store_t const * store, fd_hash_t * merkle_root ) {
fd_store_query_const( fd_store_t const * store, fd_hash_t const * merkle_root ) {
fd_store_key_t key = { .mr = *merkle_root, .part = UINT_MAX };
for( uint i = 0; i < store->part_cnt; i++ ) {
key.part = i;
Expand Down Expand Up @@ -455,7 +455,8 @@ fd_store_query_const( fd_store_t const * store, fd_hash_t * merkle_root ) {
fd_store_fec_t *
fd_store_insert( fd_store_t * store,
ulong part_idx,
fd_hash_t * merkle_root );
fd_hash_t * merkle_root,
fd_hash_t * chained_merkle_root );

/* fd_store_link queries for and links the child keyed by merkle_root to
parent keyed by chained_merkle_root. Returns a pointer to the child.
Expand Down
56 changes: 30 additions & 26 deletions src/disco/store/test_store.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ test_simple( fd_wksp_t * wksp ) {
fd_hash_t mr4 = { { 4 } };
fd_hash_t mr5 = { { 5 } };
fd_hash_t mr6 = { { 6 } };
fd_store_insert( store, 0, &mr0 );
fd_store_insert( store, 0, &mr1 );
fd_store_insert( store, 0, &mr2 );
fd_store_insert( store, 0, &mr4 );
fd_store_insert( store, 0, &mr3 );
fd_store_insert( store, 0, &mr5 );
fd_store_insert( store, 0, &mr6 );
fd_store_insert( store, 0, &mr0, &mr0 );
fd_store_insert( store, 0, &mr1, &mr0 );
fd_store_insert( store, 0, &mr2, &mr1 );
fd_store_insert( store, 0, &mr4, &mr2 );
fd_store_insert( store, 0, &mr3, &mr1 );
fd_store_insert( store, 0, &mr5, &mr3 );
fd_store_insert( store, 0, &mr6, &mr5 );

fd_store_fec_t const * fec0 = fd_store_query_const( store, &mr0 );
fd_store_fec_t const * fec1 = fd_store_query_const( store, &mr1 );
Expand Down Expand Up @@ -112,46 +112,46 @@ test_mr( fd_wksp_t * wksp ) {
fd_hash_t mr5a = { { 5, 0xa } };
fd_hash_t mr5b = { { 5, 0xb } };
fd_hash_t mr6a = { { 6, 0xa } };
FD_TEST( fd_store_insert( store, 0, &mr0 ) );
FD_TEST( fd_store_insert( store, 0, &mr0, &mr0 ) );
FD_TEST( fd_store_query_const( store, &mr0 ) );

FD_TEST( fd_store_insert( store, 0, &mr1a ) );
FD_TEST( fd_store_insert( store, 0, &mr1a, &mr0 ) );
FD_TEST( fd_store_query_const( store, &mr1a ) );

FD_TEST( fd_store_insert( store, 0, &mr1b ) );
FD_TEST( fd_store_insert( store, 0, &mr1b, &mr1a ) );
FD_TEST( fd_store_query_const( store, &mr1b ) );

FD_TEST( fd_store_insert( store, 0, &mr2a ) );
FD_TEST( fd_store_insert( store, 0, &mr2a, &mr1b) );
FD_TEST( fd_store_query_const( store, &mr2a ) );

FD_TEST( fd_store_insert( store, 0, &mr2b ) );
FD_TEST( fd_store_insert( store, 0, &mr2b, &mr2a ) );
FD_TEST( fd_store_query_const( store, &mr2b ) );

FD_TEST( fd_store_insert( store, 0, &mr2c ) );
FD_TEST( fd_store_insert( store, 0, &mr2c, &mr2b ) );
FD_TEST( fd_store_query_const( store, &mr2c ) );

FD_TEST( fd_store_insert( store, 0, &mr3a ) );
FD_TEST( fd_store_insert( store, 0, &mr3a, &mr1b ) );
FD_TEST( fd_store_query_const( store, &mr3a ) );

FD_TEST( fd_store_insert( store, 0, &mr4a ) );
FD_TEST( fd_store_insert( store, 0, &mr4a, &mr2c ) );
FD_TEST( fd_store_query_const( store, &mr4a ) );

FD_TEST( fd_store_insert( store, 0, &mr4b ) );
FD_TEST( fd_store_insert( store, 0, &mr4b, &mr4a ) );
FD_TEST( fd_store_query_const( store, &mr4b ) );

FD_TEST( fd_store_insert( store, 0, &mr4c ) );
FD_TEST( fd_store_insert( store, 0, &mr4c, &mr4b ) );
FD_TEST( fd_store_query_const( store, &mr4c ) );

FD_TEST( fd_store_insert( store, 0, &mr4d ) );
FD_TEST( fd_store_insert( store, 0, &mr4d, &mr4c ) );
FD_TEST( fd_store_query_const( store, &mr4d ) );

FD_TEST( fd_store_insert( store, 0, &mr5a ) );
FD_TEST( fd_store_insert( store, 0, &mr5a, &mr3a ) );
FD_TEST( fd_store_query_const( store, &mr5a ) );

FD_TEST( fd_store_insert( store, 0, &mr5b ) );
FD_TEST( fd_store_insert( store, 0, &mr5b, &mr5a ) );
FD_TEST( fd_store_query_const( store, &mr5b ) );

FD_TEST( fd_store_insert( store, 1, &mr6a ) );
FD_TEST( fd_store_insert( store, 1, &mr6a, &mr5b ) );
FD_TEST( fd_store_query_const( store, &mr6a ) );

fd_store_link( store, &mr1a, &mr0 );
Expand Down Expand Up @@ -192,13 +192,16 @@ test_map_function( fd_wksp_t * wksp ) {
FD_TEST( store );
ulong part_sz = fec_max / store->part_cnt;


fd_hash_t hash_null = { 0 };
fd_store_key_t key1 = { .mr = { { 0 } }, .part = 0 };
fd_store_key_t key2 = { .mr = { { 0 } }, .part = 1 };


FD_TEST( fd_store_map_key_eq( &key1, &key2 ) );
FD_TEST( fd_store_map_key_hash( &key1, part_sz ) != fd_store_map_key_hash( &key2, part_sz ) );
fd_store_insert( store, key1.part, &key1.mr );
fd_store_insert( store, key2.part, &key2.mr ); /* will fail, mr already exists.*/
fd_store_insert( store, key1.part, &key1.mr, &hash_null );
fd_store_insert( store, key2.part, &key2.mr, &hash_null ); /* will fail, mr already exists.*/

/* at this point there are two merkle root = 0 keys in the map, but they live in different partitions */
/* the store_query APIs should return the same fec_t *, but the store_map_ele_query APIs should return different fec_t * */
Expand All @@ -210,8 +213,8 @@ test_map_function( fd_wksp_t * wksp ) {
/* purposefully collide a bunch of keys with key1 */
fd_store_key_t collide1 = { .mr = { .ul = { 0, 20, 0, 0 } }, .part = 0 };
fd_store_key_t collide2 = { .mr = { .ul = { 0, 30, 0, 0 } }, .part = 0 };
fd_store_insert( store, collide1.part, &collide1.mr );
fd_store_insert( store, collide2.part, &collide2.mr );
fd_store_insert( store, collide1.part, &collide1.mr, &hash_null );
fd_store_insert( store, collide2.part, &collide2.mr, &hash_null );

FD_TEST( fd_store_map_ele_query_const( map, &collide1, NULL, fec0 ) );
FD_TEST( fd_store_map_ele_query_const( map, &collide2, NULL, fec0 ) );
Expand Down Expand Up @@ -258,13 +261,14 @@ shred_tile_insert( int argc, char ** argv ) {
FD_LOG_NOTICE(( "shred_tile_insert called on tile %lu", fd_tile_idx() ));

while( !FD_VOLATILE_CONST( tile_go ) ) FD_SPIN_PAUSE();
fd_hash_t hash_null = { 0 };

ulong tile_idx = fd_tile_idx();
for( ulong i = 1; i < num_insert; i++ ) {
fd_hash_t mr = { .ul = { (i << 16) | tile_idx } };
fd_rwlock_read( &store->lock );
FD_LOG_NOTICE(( "inserting %lu at tile %lu", i, tile_idx ));
fd_store_insert( store, (uint)tile_idx, &mr );
fd_store_insert( store, (uint)tile_idx, &mr, &hash_null );
fd_rwlock_unread( &store->lock );
}
return 0;
Expand Down
5 changes: 3 additions & 2 deletions src/discof/backtest/fd_backtest_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,11 @@ after_credit( fd_backt_tile_t * ctx,
chained merkle root overwritten with their slot numbers and fec
set index. This is done in order to preserve behavior for older
ledgers which may not have merkle roots or chained merkle roots. */
fd_hash_t mr = { .ul[0] = shred->slot, .ul[1] = shred->fec_set_idx };
fd_hash_t mr = { .ul[0] = shred->slot, .ul[1] = shred->fec_set_idx };
fd_hash_t hash_null = { 0 };
if( FD_UNLIKELY( ctx->prev_slot==ULONG_MAX || shred->slot!=ctx->prev_slot || shred->fec_set_idx!=ctx->prev_fec_set_idx ) ) {
fd_store_shacq ( ctx->store );
fd_store_insert( ctx->store, 0, &mr );
fd_store_insert( ctx->store, 0, &mr, &hash_null );
fd_store_shrel ( ctx->store );
}

Expand Down
Loading
Loading