From 164090a40a5ef5062c83b2e30a03afdb5f060f8e Mon Sep 17 00:00:00 2001 From: Charles Li Date: Wed, 29 Jan 2025 16:48:24 +0000 Subject: [PATCH] feat(blockstore): para shred pool / map and slice (entry batch) exec --- src/app/backtest/Local.mk | 5 - src/app/backtest/fd_backtest_ctl.c | 60 -- src/app/fdctl/run/tiles/fd_repair.c | 8 +- src/app/fdctl/run/tiles/fd_replay.c | 49 +- src/app/fdctl/run/tiles/fd_rpcserv.c | 2 +- src/app/fdctl/run/tiles/fd_store_int.c | 29 +- src/app/ledger/main.c | 42 +- src/app/rpcserver/main.c | 8 +- src/app/shredcap/main.c | 5 +- src/ballet/shred/fd_shred.h | 6 +- src/choreo/eqvoc/fd_eqvoc.c | 36 +- src/choreo/forks/fd_forks.c | 16 +- src/disco/geyser/fd_geyser.c | 7 +- src/disco/rpcserver/fd_rpc_service.c | 38 +- src/disco/rpcserver/fd_rpc_service.h | 1 + src/disco/shred/fd_shred_cap.c | 2 +- src/disco/store/fd_store.c | 39 +- src/flamenco/repair/fd_repair.c | 18 +- src/flamenco/runtime/Local.mk | 1 + src/flamenco/runtime/fd_blockstore.c | 838 ++++++++++------------ src/flamenco/runtime/fd_blockstore.h | 300 ++++---- src/flamenco/runtime/fd_rocksdb.c | 22 +- src/flamenco/runtime/fd_runtime.c | 7 +- src/flamenco/runtime/test_archive_block.c | 49 +- src/flamenco/runtime/test_blockstore.c | 183 +++++ src/flamenco/shredcap/fd_shredcap.c | 4 - 26 files changed, 915 insertions(+), 860 deletions(-) delete mode 100644 src/app/backtest/Local.mk delete mode 100644 src/app/backtest/fd_backtest_ctl.c create mode 100644 src/flamenco/runtime/test_blockstore.c diff --git a/src/app/backtest/Local.mk b/src/app/backtest/Local.mk deleted file mode 100644 index c1fc6752a2..0000000000 --- a/src/app/backtest/Local.mk +++ /dev/null @@ -1,5 +0,0 @@ -ifdef FD_HAS_HOSTED -ifdef FD_HAS_INT128 -$(call make-bin,fd_backtest_ctl,fd_backtest_ctl,fd_flamenco fd_funk fd_ballet fd_util) -endif -endif diff --git a/src/app/backtest/fd_backtest_ctl.c b/src/app/backtest/fd_backtest_ctl.c deleted file mode 100644 index 7259ab3f41..0000000000 --- a/src/app/backtest/fd_backtest_ctl.c +++ /dev/null @@ -1,60 +0,0 @@ -#include "../../flamenco/runtime/fd_blockstore.h" -#include "../../funk/fd_funk.h" - -/* fd_backtest_ctl provides useful script utilities for recovering - blockstore and funk from a previous live run. - - For example, you can properly checkpt the existing blockstore under - /mnt/.fd/.gigantic/fd1_bstore.wksp into a checkpt file. - - Example usage: - - ./build/native/gcc/bin/fd_backtest_ctl \ - --blockstore-checkpt /data/chali/blockstore.checkpt \ - --funk-checkpt /data/chali/funk.checkpt - - SLOTS=$(echo /dev/shm/incremental-snapshot-* | grep -oP '\d+-\d+') \ - ./build/native/gcc/bin/fd_backtest_ctl \ - --blockstore-checkpt /data/chali/$SLOTS-blockstore.checkpt \ - --funk-checkpt /data/chali/$SLOTS-funk.checkpt - - */ - -int -main( int argc, char ** argv ) { - fd_boot( &argc, &argv ); - - char const * blockstore_checkpt = fd_env_strip_cmdline_cstr( &argc, - &argv, - "--blockstore-checkpt", - NULL, - NULL ); - char const * funk_checkpt = fd_env_strip_cmdline_cstr( &argc, - &argv, - "--funk-checkpt", - NULL, - NULL ); - - fd_wksp_t * blockstore_wksp = fd_wksp_attach( "fd1_bstore.wksp" ); - FD_TEST( blockstore_wksp ); - fd_wksp_tag_query_info_t blockstore_info; - ulong blockstore_tag = FD_BLOCKSTORE_MAGIC; - FD_TEST( fd_wksp_tag_query( blockstore_wksp, &blockstore_tag, 1, &blockstore_info, 1 ) > 0 ); - void * blockstore_mem = fd_wksp_laddr_fast( blockstore_wksp, blockstore_info.gaddr_lo ); - fd_blockstore_t * blockstore = fd_blockstore_join( blockstore_mem ); - FD_TEST( blockstore ); - FD_TEST( !fd_wksp_checkpt( blockstore_wksp, blockstore_checkpt, 0666, 0, NULL ) ); - - fd_wksp_t * funk_wksp = fd_wksp_attach( "fd1_funk.wksp" ); - FD_TEST( funk_wksp ); - fd_wksp_tag_query_info_t funk_info; - ulong funk_tag = FD_FUNK_MAGIC; - FD_TEST( fd_wksp_tag_query( funk_wksp, &funk_tag, 1, &funk_info, 1 ) > 0 ); - void * funk_mem = fd_wksp_laddr_fast( funk_wksp, funk_info.gaddr_lo ); - fd_funk_t * funk = fd_funk_join( funk_mem ); - FD_TEST( funk ); - FD_TEST( !fd_wksp_checkpt( funk_wksp, funk_checkpt, 0666, 0, NULL ) ); - - fd_halt(); - return 0; -} diff --git a/src/app/fdctl/run/tiles/fd_repair.c b/src/app/fdctl/run/tiles/fd_repair.c index 549cdbe692..8534d413c7 100644 --- a/src/app/fdctl/run/tiles/fd_repair.c +++ b/src/app/fdctl/run/tiles/fd_repair.c @@ -103,6 +103,7 @@ struct fd_repair_tile_ctx { fd_stem_context_t * stem; fd_wksp_t * blockstore_wksp; + fd_blockstore_t blockstore_ljoin; fd_blockstore_t * blockstore; fd_keyguard_client_t keyguard_client[1]; @@ -524,7 +525,8 @@ unprivileged_init( fd_topo_t * topo, FD_SCRATCH_ALLOC_INIT( l, scratch ); fd_repair_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_repair_tile_ctx_t), sizeof(fd_repair_tile_ctx_t) ); - ctx->repair = FD_SCRATCH_ALLOC_APPEND( l, fd_repair_align(), fd_repair_footprint() ); + ctx->blockstore = &ctx->blockstore_ljoin; + ctx->repair = FD_SCRATCH_ALLOC_APPEND( l, fd_repair_align(), fd_repair_footprint() ); void * smem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( FD_REPAIR_SCRATCH_MAX ) ); void * fmem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( FD_REPAIR_SCRATCH_DEPTH ) ); @@ -572,7 +574,7 @@ unprivileged_init( fd_topo_t * topo, FD_LOG_ERR(( "no blocktore workspace" )); } - ctx->blockstore = fd_blockstore_join( fd_topo_obj_laddr( topo, blockstore_obj_id ) ); + ctx->blockstore = fd_blockstore_join( &ctx->blockstore_ljoin, fd_topo_obj_laddr( topo, blockstore_obj_id ) ); FD_TEST( ctx->blockstore!=NULL ); fd_topo_link_t * netmux_link = &topo->links[ tile->in_link_id[ 0 ] ]; @@ -662,7 +664,7 @@ populate_allowed_seccomp( fd_topo_t const * topo, (void)topo; (void)tile; - populate_sock_filter_policy_repair( + populate_sock_filter_policy_repair( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)tile->repair.good_peer_cache_file_fd ); return sock_filter_policy_repair_instr_cnt; } diff --git a/src/app/fdctl/run/tiles/fd_replay.c b/src/app/fdctl/run/tiles/fd_replay.c index ba87bbad99..efc5cadd1f 100644 --- a/src/app/fdctl/run/tiles/fd_replay.c +++ b/src/app/fdctl/run/tiles/fd_replay.c @@ -230,8 +230,9 @@ struct fd_replay_tile_ctx { /* Depends on store_int and is polled in after_credit */ - fd_blockstore_t * blockstore; + fd_blockstore_t blockstore_ljoin; int blockstore_fd; /* file descriptor for archival file */ + fd_blockstore_t * blockstore; /* Updated during execution */ @@ -363,7 +364,7 @@ scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) { for( ulong i = 0UL; ireplay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) ); l = FD_LAYOUT_APPEND( l, fd_spad_align(), FD_RUNTIME_BLOCK_EXECUTION_FOOTPRINT ); /* FIXME: make this configurable */ @@ -1135,7 +1136,7 @@ publish_slot_notifications( fd_replay_tile_ctx_t * ctx, msg->type = FD_REPLAY_SLOT_TYPE; msg->slot_exec.slot = curr_slot; msg->slot_exec.parent = ctx->parent_slot; - msg->slot_exec.root = ctx->blockstore->smr; + msg->slot_exec.root = ctx->blockstore->shmem->smr; msg->slot_exec.height = ( block_map_entry ? block_map_entry->block_height : 0UL ); msg->slot_exec.transaction_count = fork->slot_ctx.slot_bank.transaction_count; memcpy( &msg->slot_exec.bank_hash, &fork->slot_ctx.slot_bank.banks_hash, sizeof( fd_hash_t ) ); @@ -1609,11 +1610,11 @@ after_frag( fd_replay_tile_ctx_t * ctx, fd_blockstore_end_read( ctx->blockstore ); if( FD_LIKELY( block_map_entry->data_complete_idx != FD_SHRED_IDX_NULL ) ) { - uint i = block_map_entry->replayed_idx + 1; + uint i = block_map_entry->consumed_idx + 1; uint j = block_map_entry->data_complete_idx; /* If this is the first batch being verified of this block, need to populate the slot_bank's tick height for tick verification */ - if( FD_UNLIKELY( block_map_entry->replayed_idx + 1 == 0 ) ){ + if( FD_UNLIKELY( block_map_entry->consumed_idx + 1 == 0 ) ){ FD_LOG_NOTICE(("Preparing first batch execution of slot %lu", ctx->curr_slot)); prepare_first_batch_execution( ctx, stem ); } @@ -1629,15 +1630,12 @@ after_frag( fd_replay_tile_ctx_t * ctx, required because txns can span multiple shreds. */ ulong mbatch_sz = 0; - fd_blockstore_start_read( ctx->blockstore ); - - int err = fd_blockstore_batch_assemble( ctx->blockstore, - ctx->curr_slot, - block_map_entry->replayed_idx + 1, - FD_MBATCH_MAX, - ctx->mbatch, - &mbatch_sz ); - fd_blockstore_end_read( ctx->blockstore ); + int err = fd_blockstore_slice_query( ctx->blockstore, + ctx->curr_slot, + block_map_entry->consumed_idx + 1, + FD_SLICE_MAX, + ctx->mbatch, + &mbatch_sz ); if( FD_UNLIKELY( err ) ){ FD_LOG_ERR(( "Failed to assemble microblock batch" )); @@ -1648,10 +1646,13 @@ after_frag( fd_replay_tile_ctx_t * ctx, // TODO: handle invalid batch how & do thread handling FD_LOG_ERR(( "Failed to process microblock batch" )); } + if( FD_UNLIKELY( idx == block_map_entry->slot_complete_idx ) ) { + for( uint idx = 0; idx <= block_map_entry->slot_complete_idx; idx++ ) { + fd_blockstore_shred_remove( ctx->blockstore, ctx->curr_slot, idx ); + } + } //replay_mbatch( ctx, block_map_entry->replayed_idx + 1 ); - fd_blockstore_start_write( ctx->blockstore ); - block_map_entry->replayed_idx = idx; - fd_blockstore_end_write( ctx->blockstore ); + block_map_entry->consumed_idx = idx; } } } @@ -1766,7 +1767,7 @@ after_frag( fd_replay_tile_ctx_t * ctx, if( FD_LIKELY( block_ ) ) { block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, FD_BLOCK_FLAG_PROCESSED ); block_map_entry->flags = fd_uchar_clear_bit( block_map_entry->flags, FD_BLOCK_FLAG_REPLAYING ); - ctx->blockstore->lps = block_map_entry->slot; + ctx->blockstore->shmem->lps = block_map_entry->slot; memcpy( &block_map_entry->bank_hash, &fork->slot_ctx.slot_bank.banks_hash, sizeof( fd_hash_t ) ); } fd_blockstore_end_write( ctx->blockstore ); @@ -1822,7 +1823,7 @@ after_frag( fd_replay_tile_ctx_t * ctx, break; } s = block_map_entry->parent_slot; - if( s < ctx->blockstore->smr ) { + if( s < ctx->blockstore->shmem->smr ) { break; } *(ulong*)(msg + 24U + i*8U) = s; @@ -2476,7 +2477,7 @@ during_housekeeping( void * _ctx ) { root and blockstore smr. */ fd_blockstore_start_read( ctx->blockstore ); - ulong wmk = fd_ulong_min( ctx->root, ctx->blockstore->smr ); + ulong wmk = fd_ulong_min( ctx->root, ctx->blockstore->shmem->smr ); fd_blockstore_end_read( ctx->blockstore ); if ( FD_LIKELY( wmk <= fd_fseq_query( ctx->wmk ) ) ) return; @@ -2557,7 +2558,7 @@ unprivileged_init( fd_topo_t * topo, for( ulong i = 0UL; ibmtree[i] = FD_SCRATCH_ALLOC_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) ); } - void * mbatch_mem = FD_SCRATCH_ALLOC_APPEND( l, 128UL, FD_MBATCH_MAX ); + void * mbatch_mem = FD_SCRATCH_ALLOC_APPEND( l, 128UL, FD_SLICE_MAX ); ulong thread_spad_size = fd_spad_footprint( FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT ); void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) + FD_RUNTIME_BLOCK_EXECUTION_FOOTPRINT ); ulong scratch_alloc_mem = FD_SCRATCH_ALLOC_FINI ( l, scratch_align() ); @@ -2574,7 +2575,6 @@ unprivileged_init( fd_topo_t * topo, /**********************************************************************/ ctx->wksp = topo->workspaces[ topo->objs[ tile->tile_obj_id ].wksp_id ].wksp; - ctx->blockstore = NULL; ulong blockstore_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "blockstore" ); FD_TEST( blockstore_obj_id!=ULONG_MAX ); @@ -2583,8 +2583,9 @@ unprivileged_init( fd_topo_t * topo, FD_LOG_ERR(( "no blockstore wksp" )); } - ctx->blockstore = fd_blockstore_join( fd_topo_obj_laddr( topo, blockstore_obj_id ) ); - FD_TEST( ctx->blockstore!=NULL ); + ctx->blockstore = fd_blockstore_join( &ctx->blockstore_ljoin, fd_topo_obj_laddr( topo, blockstore_obj_id ) ); + fd_buf_shred_pool_reset( ctx->blockstore->shred_pool, 0 ); + FD_TEST( ctx->blockstore->shmem->magic == FD_BLOCKSTORE_MAGIC ); ulong status_cache_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "txncache" ); FD_TEST( status_cache_obj_id != ULONG_MAX ); diff --git a/src/app/fdctl/run/tiles/fd_rpcserv.c b/src/app/fdctl/run/tiles/fd_rpcserv.c index 39d6f60fde..96b49e23ad 100644 --- a/src/app/fdctl/run/tiles/fd_rpcserv.c +++ b/src/app/fdctl/run/tiles/fd_rpcserv.c @@ -190,7 +190,7 @@ privileged_init( fd_topo_t * topo, /* Blockstore setup */ ulong blockstore_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "blockstore" ); FD_TEST( blockstore_obj_id!=ULONG_MAX ); - args->blockstore = fd_blockstore_join( fd_topo_obj_laddr( topo, blockstore_obj_id ) ); + args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, fd_topo_obj_laddr( topo, blockstore_obj_id ) ); FD_TEST( args->blockstore!=NULL ); ctx->blockstore_fd = open( tile->replay.blockstore_file, O_RDONLY ); if ( FD_UNLIKELY(ctx->blockstore_fd == -1) ){ diff --git a/src/app/fdctl/run/tiles/fd_store_int.c b/src/app/fdctl/run/tiles/fd_store_int.c index 5695e88add..a63ff04566 100644 --- a/src/app/fdctl/run/tiles/fd_store_int.c +++ b/src/app/fdctl/run/tiles/fd_store_int.c @@ -92,8 +92,9 @@ struct fd_store_tile_ctx { fd_pubkey_t identity_key[1]; /* Just the public key */ fd_store_t * store; - fd_blockstore_t * blockstore; + fd_blockstore_t blockstore_ljoin; int blockstore_fd; /* file descriptor for archival file */ + fd_blockstore_t * blockstore; fd_wksp_t * stake_in_mem; ulong stake_in_chunk0; @@ -271,13 +272,13 @@ after_frag( fd_store_tile_ctx_t * ctx, return; } - if( fd_store_shred_insert( ctx->store, shred ) < FD_BLOCKSTORE_OK ) { + if( fd_store_shred_insert( ctx->store, shred ) < FD_BLOCKSTORE_SUCCESS ) { FD_LOG_ERR(( "failed inserting to blockstore" )); } else if ( ctx->shred_cap_ctx.is_archive ) { - uchar shred_cap_flag = FD_SHRED_CAP_FLAG_MARK_REPAIR(0); - if ( fd_shred_cap_archive(&ctx->shred_cap_ctx, shred, shred_cap_flag) < FD_SHRED_CAP_OK ) { - FD_LOG_ERR(( "failed at archiving repair shred to file" )); - } + uchar shred_cap_flag = FD_SHRED_CAP_FLAG_MARK_REPAIR( 0 ); + if( fd_shred_cap_archive( &ctx->shred_cap_ctx, shred, shred_cap_flag ) < FD_SHRED_CAP_OK ) { + FD_LOG_ERR( ( "failed at archiving repair shred to file" ) ); + } } return; } @@ -310,16 +311,16 @@ after_frag( fd_store_tile_ctx_t * ctx, } // TODO: improve return value of api to not use < OK - if( fd_store_shred_insert( ctx->store, &ctx->s34_buffer->pkts[i].shred ) < FD_BLOCKSTORE_OK ) { + if( fd_store_shred_insert( ctx->store, shred ) < FD_BLOCKSTORE_SUCCESS ) { FD_LOG_ERR(( "failed inserting to blockstore" )); } else if ( ctx->shred_cap_ctx.is_archive ) { uchar shred_cap_flag = FD_SHRED_CAP_FLAG_MARK_TURBINE(0); - if ( fd_shred_cap_archive(&ctx->shred_cap_ctx, &ctx->s34_buffer->pkts[i].shred, shred_cap_flag) < FD_SHRED_CAP_OK ) { + if ( fd_shred_cap_archive(&ctx->shred_cap_ctx, shred, shred_cap_flag) < FD_SHRED_CAP_OK ) { FD_LOG_ERR(( "failed at archiving turbine shred to file" )); } } - fd_store_shred_update_with_shred_from_turbine( ctx->store, &ctx->s34_buffer->pkts[i].shred ); + fd_store_shred_update_with_shred_from_turbine( ctx->store, shred ); } } @@ -475,7 +476,7 @@ after_credit( fd_store_tile_ctx_t * ctx, if( FD_UNLIKELY( ctx->sim && ctx->store->pending_slots->start == ctx->store->pending_slots->end ) ) { - FD_LOG_WARNING(( "Sim is complete." )); + // FD_LOG_WARNING(( "Sim is complete." )); } for( ulong i = 0; iblockstore = &ctx->blockstore_ljoin; // TODO: set the lo_mark_slot to the actual snapshot slot! ctx->store = fd_store_join( fd_store_new( FD_SCRATCH_ALLOC_APPEND( l, fd_store_align(), fd_store_footprint() ), 1 ) ); ctx->repair_req_buffer = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_repair_request_t), MAX_REPAIR_REQS * sizeof(fd_repair_request_t) ); @@ -591,7 +593,7 @@ unprivileged_init( fd_topo_t * topo, do { expected_shred_version = fd_fseq_query( gossip_shred_version ); } while( expected_shred_version==ULONG_MAX ); - FD_LOG_INFO(( "using shred version %lu", expected_shred_version )); + FD_LOG_NOTICE(( "using shred version %lu", expected_shred_version )); } if( FD_UNLIKELY( expected_shred_version>USHORT_MAX ) ) FD_LOG_ERR(( "invalid shred version %lu", expected_shred_version )); FD_TEST( expected_shred_version ); @@ -608,7 +610,7 @@ unprivileged_init( fd_topo_t * topo, ulong tag = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.wksp_tag", blockstore_obj_id ); if( FD_LIKELY( fd_wksp_tag_query( ctx->blockstore_wksp, &tag, 1, &info, 1 ) > 0 ) ) { void * blockstore_mem = fd_wksp_laddr_fast( ctx->blockstore_wksp, info.gaddr_lo ); - ctx->blockstore = fd_blockstore_join( blockstore_mem ); + ctx->blockstore = fd_blockstore_join( &ctx->blockstore_ljoin, blockstore_mem ); } else { FD_LOG_WARNING(( "failed to find blockstore in workspace. making new blockstore." )); } @@ -618,7 +620,7 @@ unprivileged_init( fd_topo_t * topo, FD_LOG_ERR(( "failed to find blockstore" )); } - ctx->blockstore = fd_blockstore_join( blockstore_shmem ); + ctx->blockstore = fd_blockstore_join( &ctx->blockstore_ljoin, blockstore_shmem ); } FD_LOG_NOTICE(( "blockstore: %s", tile->store_int.blockstore_file )); @@ -736,6 +738,7 @@ unprivileged_init( fd_topo_t * topo, ctx->sim = 1; FD_TEST( fd_shred_cap_replay( tile->store_int.shred_cap_replay, ctx->store ) == FD_SHRED_CAP_OK ); } + } static ulong diff --git a/src/app/ledger/main.c b/src/app/ledger/main.c index 5891fec1e2..fb1fdffcd6 100644 --- a/src/app/ledger/main.c +++ b/src/app/ledger/main.c @@ -35,6 +35,7 @@ struct fd_ledger_args { fd_wksp_t * wksp; /* wksp for blockstore */ fd_wksp_t * funk_wksp; /* wksp for funk */ fd_wksp_t * status_cache_wksp; /* wksp for status cache. */ + fd_blockstore_t blockstore_ljoin; fd_blockstore_t * blockstore; /* blockstore for replay */ fd_funk_t * funk; /* handle to funk */ fd_alloc_t * alloc; /* handle to alloc */ @@ -382,7 +383,7 @@ runtime_replay( fd_ledger_args_t * ledger_args ) { bool block_exists = fd_blockstore_shreds_complete( blockstore, slot); fd_blockstore_end_read( blockstore ); if( !block_exists && slot_meta.slot == slot ) { - int err = fd_rocksdb_import_block_blockstore( &rocks_db, + int err = fd_rocksdb_import_block_blockstore( &rocks_db, &slot_meta, blockstore, ledger_args->copy_txn_status, slot == (ledger_args->trash_hash) ? trash_hash_buf : NULL, @@ -401,6 +402,9 @@ runtime_replay( fd_ledger_args_t * ledger_args ) { block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, FD_BLOCK_FLAG_PROCESSED ); /* Remove the old block from the blockstore */ + for( uint idx = 0; idx <= block_map_entry->slot_complete_idx; idx++ ) { + fd_blockstore_shred_remove( blockstore, block_slot, idx ); + } fd_blockstore_slot_remove( blockstore, block_slot ); } @@ -648,7 +652,7 @@ runtime_replay( fd_ledger_args_t * ledger_args ) { } /***************************** Helpers ****************************************/ -static fd_valloc_t +static fd_valloc_t allocator_setup( fd_wksp_t * wksp ) { if( FD_UNLIKELY( !wksp ) ) { @@ -728,7 +732,7 @@ fd_ledger_main_setup( fd_ledger_args_t * args ) { fd_bpf_scan_and_create_bpf_program_cache_entry_tpool( args->slot_ctx, args->slot_ctx->funk_txn, args->tpool, args->runtime_spad ); fd_funk_end_write( funk ); - /* First, load in the sysvars into the sysvar cache. This is required to + /* First, load in the sysvars into the sysvar cache. This is required to make the StakeHistory sysvar available to the rewards calculation. */ fd_runtime_sysvar_cache_load( args->slot_ctx ); @@ -925,8 +929,8 @@ init_blockstore( fd_ledger_args_t * args ) { void * shmem; if( fd_wksp_tag_query( args->wksp, &blockstore_tag, 1, &info, 1 ) > 0 ) { shmem = fd_wksp_laddr_fast( args->wksp, info.gaddr_lo ); - args->blockstore = fd_blockstore_join( shmem ); - if( args->blockstore == NULL ) { + args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, shmem ); + if( args->blockstore->shmem->magic != FD_BLOCKSTORE_MAGIC ) { FD_LOG_ERR(( "failed to join a blockstore" )); } FD_LOG_NOTICE(( "joined blockstore" )); @@ -936,9 +940,8 @@ init_blockstore( fd_ledger_args_t * args ) { if( shmem == NULL ) { FD_LOG_ERR(( "failed to allocate a blockstore" )); } - args->blockstore = fd_blockstore_join( fd_blockstore_new( shmem, 1, args->hashseed, args->shred_max, - args->slot_history_max, 16, txn_max ) ); - if( args->blockstore == NULL ) { + args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, fd_blockstore_new( shmem, 1, args->hashseed, args->shred_max, args->slot_history_max, 16, txn_max ) ); + if( args->blockstore->shmem->magic != FD_BLOCKSTORE_MAGIC ) { fd_wksp_free_laddr( shmem ); FD_LOG_ERR(( "failed to allocate a blockstore" )); } @@ -1121,10 +1124,10 @@ ingest( fd_ledger_args_t * args ) { if( args->snapshot ) { fd_snapshot_load_all( args->snapshot, slot_ctx, - NULL, - args->tpool, - args->verify_acc_hash, - args->check_acc_hash , + NULL, + args->tpool, + args->verify_acc_hash, + args->check_acc_hash , FD_SNAPSHOT_TYPE_FULL, args->exec_spads, args->exec_spad_cnt, @@ -1134,11 +1137,11 @@ ingest( fd_ledger_args_t * args ) { if( args->incremental ) { fd_snapshot_load_all( args->incremental, slot_ctx, - NULL, - args->tpool, - args->verify_acc_hash, - args->check_acc_hash, - FD_SNAPSHOT_TYPE_INCREMENTAL, + NULL, + args->tpool, + args->verify_acc_hash, + args->check_acc_hash, + FD_SNAPSHOT_TYPE_INCREMENTAL, args->exec_spads, args->exec_spad_cnt, args->runtime_spad ); @@ -1159,7 +1162,7 @@ ingest( fd_ledger_args_t * args ) { } fd_blockstore_t * blockstore = args->blockstore; if( blockstore ) { - blockstore->lps = blockstore->hcs = blockstore->smr = slot_ctx->slot_bank.slot; + blockstore->shmem->lps = blockstore->shmem->hcs = blockstore->shmem->smr = slot_ctx->slot_bank.slot; } if( args->funk_only ) { @@ -1247,7 +1250,7 @@ replay( fd_ledger_args_t * args ) { /* Setup slot_ctx */ fd_funk_t * funk = args->funk; - void * epoch_ctx_mem = fd_spad_alloc( spad, FD_EXEC_EPOCH_CTX_ALIGN, fd_exec_epoch_ctx_footprint( args->vote_acct_max ) ); + void * epoch_ctx_mem = fd_spad_alloc( spad, FD_EXEC_EPOCH_CTX_ALIGN, fd_exec_epoch_ctx_footprint( args->vote_acct_max ) ); fd_memset( epoch_ctx_mem, 0, fd_exec_epoch_ctx_footprint( args->vote_acct_max ) ); args->epoch_ctx = fd_exec_epoch_ctx_join( fd_exec_epoch_ctx_new( epoch_ctx_mem, args->vote_acct_max ) ); fd_exec_epoch_ctx_bank_mem_clear( args->epoch_ctx ); @@ -1325,6 +1328,7 @@ replay( fd_ledger_args_t * args ) { fd_ledger_main_setup( args ); fd_blockstore_init( args->blockstore, -1, FD_BLOCKSTORE_ARCHIVE_MIN_SIZE, &args->slot_ctx->slot_bank ); + fd_buf_shred_pool_reset( args->blockstore->shred_pool, 0 ); FD_LOG_WARNING(( "setup done" )); diff --git a/src/app/rpcserver/main.c b/src/app/rpcserver/main.c index 9100873d5a..b97e2cfef1 100644 --- a/src/app/rpcserver/main.c +++ b/src/app/rpcserver/main.c @@ -55,11 +55,11 @@ init_args( int * argc, char *** argv, fd_rpcserver_args_t * args ) { FD_LOG_ERR(( "workspace \"%s\" does not contain a blockstore", wksp_name )); } void * shmem = fd_wksp_laddr_fast( wksp, info.gaddr_lo ); - args->blockstore = fd_blockstore_join( shmem ); + args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, shmem ); if( args->blockstore == NULL ) { FD_LOG_ERR(( "failed to join a blockstore" )); } - FD_LOG_NOTICE(( "blockstore has slot root=%lu", args->blockstore->smr )); + FD_LOG_NOTICE(( "blockstore has slot root=%lu", args->blockstore->shmem->smr )); fd_wksp_mprotect( wksp, 1 ); fd_pubkey_t identity_key[1]; /* Just the public key */ @@ -135,11 +135,11 @@ init_args_offline( int * argc, char *** argv, fd_rpcserver_args_t * args ) { FD_LOG_ERR(( "workspace does not contain a blockstore" )); } void * shmem = fd_wksp_laddr_fast( wksp, info.gaddr_lo ); - args->blockstore = fd_blockstore_join( shmem ); + args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, shmem ); if( args->blockstore == NULL ) { FD_LOG_ERR(( "failed to join a blockstore" )); } - FD_LOG_NOTICE(( "blockstore has slot root=%lu", args->blockstore->smr )); + FD_LOG_NOTICE(( "blockstore has slot root=%lu", args->blockstore->shmem->smr )); fd_wksp_mprotect( wksp, 1 ); args->port = (ushort)fd_env_strip_cmdline_ulong( argc, argv, "--port", NULL, 8899 ); diff --git a/src/app/shredcap/main.c b/src/app/shredcap/main.c index a209c40d12..e44df2755f 100644 --- a/src/app/shredcap/main.c +++ b/src/app/shredcap/main.c @@ -73,9 +73,10 @@ main( int argc, char ** argv ) { fd_blockstore_t * blockstore; ulong tag = FD_BLOCKSTORE_MAGIC; fd_wksp_tag_query_info_t info; + fd_blockstore_t blockstore_ljoin; if ( fd_wksp_tag_query( wksp, &tag, 1, &info, 1 ) > 0) { shmem = fd_wksp_laddr_fast( wksp, info.gaddr_lo ); - blockstore = fd_blockstore_join( shmem ); + blockstore = fd_blockstore_join( &blockstore_ljoin, shmem ); if ( blockstore == NULL ) { FD_LOG_ERR(( "failed to join a blockstore" )); } @@ -85,7 +86,7 @@ main( int argc, char ** argv ) { FD_LOG_ERR(( "failed to allocate a blockstore" )); } - blockstore = fd_blockstore_join( fd_blockstore_new( shmem, 1, hashseed, shred_max, slot_history_max, 16, shred_max ) ); + blockstore = fd_blockstore_join( &blockstore_ljoin, fd_blockstore_new( shmem, 1, hashseed, shred_max, slot_history_max, 16, shred_max ) ); if ( blockstore == NULL ) { fd_wksp_free_laddr( shmem ); FD_LOG_ERR(( "failed to allocate a blockstore" )); diff --git a/src/ballet/shred/fd_shred.h b/src/ballet/shred/fd_shred.h index 6d07c68ac0..e2fbd7ce01 100644 --- a/src/ballet/shred/fd_shred.h +++ b/src/ballet/shred/fd_shred.h @@ -212,7 +212,7 @@ struct __attribute__((packed)) fd_shred { union { /* Common data shred header */ - struct __attribute__((packed)) fd_shred_data { + struct __attribute__((packed)) { /* Slot number difference between this block and the parent block. parent_off <= slot. Always greater than zero, except for slot 0, in which case the @@ -232,7 +232,7 @@ struct __attribute__((packed)) fd_shred { } data; /* Common coding shred header */ - struct __attribute__((packed)) fd_shred_code { + struct __attribute__((packed)) { /* Total number of data shreds in slot. Must be positive. */ /* 0x53 */ ushort data_cnt; @@ -245,8 +245,6 @@ struct __attribute__((packed)) fd_shred { } code; }; }; -typedef struct fd_shred_data fd_shred_data_t; -typedef struct fd_shred_code fd_shred_code_t; typedef struct fd_shred fd_shred_t; FD_PROTOTYPES_BEGIN diff --git a/src/choreo/eqvoc/fd_eqvoc.c b/src/choreo/eqvoc/fd_eqvoc.c index 910ff48da7..96056997ea 100644 --- a/src/choreo/eqvoc/fd_eqvoc.c +++ b/src/choreo/eqvoc/fd_eqvoc.c @@ -168,40 +168,6 @@ fd_eqvoc_fec_search( fd_eqvoc_t const * eqvoc, fd_shred_t const * shred ) { return NULL; /* No conflicts */ } -int -fd_eqvoc_fec_verify( FD_PARAM_UNUSED fd_eqvoc_t const * eqvoc, - fd_blockstore_t * blockstore, - ulong slot, - uint fec_set_idx, - fd_hash_t * chained_hash ) { - - fd_shred_t * shred = NULL; - uint idx = fec_set_idx; - do { - fd_blockstore_start_read( blockstore ); - shred = fd_buf_shred_query( blockstore, slot, idx ); - fd_blockstore_end_read( blockstore ); - -#if FD_EQVOC_USE_HANDHOLDING - if( FD_UNLIKELY( !shred ) ) { - FD_LOG_WARNING(( "[%s] couldn't find shred %lu %u", __func__, slot, fec_set_idx )); - return 0; - } -#endif - -#if FD_EQVOC_USE_HANDHOLDING - FD_TEST( fd_shred_is_chained( fd_shred_type( shred->variant ) ) ); -#endif - - if( FD_UNLIKELY( 0 != memcmp( chained_hash, shred + fd_shred_chain_off( shred->variant ), FD_SHRED_MERKLE_ROOT_SZ ) ) ) { - return 0; - } - - } while( shred->fec_set_idx == fec_set_idx ); - - return 1; -} - fd_eqvoc_proof_t * fd_eqvoc_proof_insert( fd_eqvoc_t * eqvoc, ulong slot, fd_pubkey_t const * from ) { fd_slot_pubkey_t key = { slot, *from }; @@ -239,7 +205,7 @@ fd_eqvoc_proof_chunk_insert( fd_eqvoc_proof_t * proof, fd_gossip_duplicate_shred FD_LOG_WARNING(( "[%s] received incompatible chunk (slot: %lu from: %s). ignoring.", __func__, proof->key.slot, FD_BASE58_ENC_32_ALLOCA( proof->key.hash.uc ) )); return; } - + if( FD_UNLIKELY( fd_eqvoc_proof_set_test( proof->set, chunk->chunk_index ) ) ) { FD_LOG_WARNING(( "[%s] already received chunk %u. slot: %lu from: %s. ignoring.", __func__, chunk->chunk_index, proof->key.slot, FD_BASE58_ENC_32_ALLOCA( proof->key.hash.uc ) )); diff --git a/src/choreo/forks/fd_forks.c b/src/choreo/forks/fd_forks.c index ff61bfef90..1e4dad65e7 100644 --- a/src/choreo/forks/fd_forks.c +++ b/src/choreo/forks/fd_forks.c @@ -367,10 +367,9 @@ fd_forks_update( fd_forks_t * forks, if( FD_UNLIKELY( !eqvocsafe ) ) { double pct = (double)node->replay_stake / (double)epoch->total_stake; if( FD_UNLIKELY( pct > FD_EQVOCSAFE_PCT ) ) { - FD_LOG_DEBUG( ( "eqvocsafe %lu", block_map_entry->slot ) ); - block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, - FD_BLOCK_FLAG_EQVOCSAFE ); - blockstore->hcs = fd_ulong_max( blockstore->hcs, block_map_entry->slot ); + FD_LOG_DEBUG(( "eqvocsafe %lu", block_map_entry->slot )); + block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, FD_BLOCK_FLAG_EQVOCSAFE ); + blockstore->shmem->hcs = fd_ulong_max( blockstore->shmem->hcs, block_map_entry->slot ); } } @@ -378,10 +377,9 @@ fd_forks_update( fd_forks_t * forks, if( FD_UNLIKELY( !confirmed ) ) { double pct = (double)node->replay_stake / (double)epoch->total_stake; if( FD_UNLIKELY( pct > FD_CONFIRMED_PCT ) ) { - FD_LOG_DEBUG( ( "confirmed %lu", block_map_entry->slot ) ); - block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, - FD_BLOCK_FLAG_CONFIRMED ); - blockstore->hcs = fd_ulong_max( blockstore->hcs, block_map_entry->slot ); + FD_LOG_DEBUG(( "confirmed %lu", block_map_entry->slot )); + block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, FD_BLOCK_FLAG_CONFIRMED ); + blockstore->shmem->hcs = fd_ulong_max( blockstore->shmem->hcs, block_map_entry->slot ); } } @@ -415,7 +413,7 @@ fd_forks_update( fd_forks_t * forks, double pct = (double)node->rooted_stake / (double)epoch->total_stake; if( FD_UNLIKELY( pct > FD_FINALIZED_PCT ) ) { ulong smr = block_map_entry->slot; - blockstore->smr = fd_ulong_max( blockstore->smr, smr ); + blockstore->shmem->smr = fd_ulong_max( blockstore->shmem->smr, smr ); FD_LOG_DEBUG(( "finalized %lu", block_map_entry->slot )); fd_block_map_t * ancestor = block_map_entry; while( ancestor ) { diff --git a/src/disco/geyser/fd_geyser.c b/src/disco/geyser/fd_geyser.c index 158814358e..08086b5fb0 100644 --- a/src/disco/geyser/fd_geyser.c +++ b/src/disco/geyser/fd_geyser.c @@ -26,6 +26,7 @@ struct fd_geyser { fd_funk_t * funk; + fd_blockstore_t blockstore_ljoin; fd_blockstore_t * blockstore; int blockstore_fd; fd_stake_ci_t * stake_ci; @@ -81,12 +82,12 @@ fd_geyser_new( void * mem, fd_geyser_args_t * args ) { FD_LOG_ERR(( "workspace \"%s\" does not contain a blockstore", args->blockstore_wksp )); } void * shmem = fd_wksp_laddr_fast( wksp, info.gaddr_lo ); - self->blockstore = fd_blockstore_join( shmem ); - if( self->blockstore == NULL ) { + self->blockstore = fd_blockstore_join( &self->blockstore_ljoin, shmem ); + if( self->blockstore->shmem->magic != FD_BLOCKSTORE_MAGIC ) { FD_LOG_ERR(( "failed to join a blockstore" )); } self->blockstore_fd = args->blockstore_fd; - FD_LOG_NOTICE(( "blockstore has slot root=%lu", self->blockstore->smr )); + FD_LOG_NOTICE(( "blockstore has slot root=%lu", self->blockstore->shmem->smr )); fd_wksp_mprotect( wksp, 1 ); fd_pubkey_t identity_key[1]; /* Just the public key */ diff --git a/src/disco/rpcserver/fd_rpc_service.c b/src/disco/rpcserver/fd_rpc_service.c index 06b2268ddb..eb5035d051 100644 --- a/src/disco/rpcserver/fd_rpc_service.c +++ b/src/disco/rpcserver/fd_rpc_service.c @@ -107,7 +107,7 @@ struct fd_rpc_global_ctx { fd_valloc_t valloc; fd_webserver_t ws; fd_funk_t * funk; - fd_blockstore_t * blockstore; + fd_blockstore_t blockstore[1]; int blockstore_fd; struct fd_ws_subscription sub_list[FD_WS_MAX_SUBS]; ulong sub_cnt; @@ -173,11 +173,11 @@ get_slot_from_commitment_level( struct json_values * values, fd_rpc_ctx_t * ctx ulong commit_str_sz = 0; const void * commit_str = json_get_value( values, PATH_COMMITMENT, 4, &commit_str_sz ); if( commit_str == NULL || MATCH_STRING( commit_str, commit_str_sz, "confirmed" ) ) { - return ctx->global->blockstore->hcs; + return ctx->global->blockstore->shmem->hcs; } else if( MATCH_STRING( commit_str, commit_str_sz, "processed" ) ) { - return ctx->global->blockstore->lps; + return ctx->global->blockstore->shmem->lps; } else if( MATCH_STRING( commit_str, commit_str_sz, "finalized" ) ) { - return ctx->global->blockstore->smr; + return ctx->global->blockstore->shmem->smr; } else { fd_method_error( ctx, -1, "invalid commitment %s", (const char *)commit_str ); return FD_SLOT_NULL; @@ -593,8 +593,8 @@ method_getBlockProduction(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_webserver_t * ws = &glob->ws; fd_blockstore_t * blockstore = glob->blockstore; FD_SCRATCH_SCOPE_BEGIN { - ulong startslot = blockstore->smr; - ulong endslot = blockstore->lps; + ulong startslot = blockstore->shmem->smr; + ulong endslot = blockstore->shmem->lps; fd_per_epoch_info_t const * ei = glob->stake_ci->epoch_info; startslot = fd_ulong_max( startslot, fd_ulong_min( ei[0].start_slot, ei[1].start_slot ) ); @@ -673,10 +673,10 @@ method_getBlocks(struct json_values* values, fd_rpc_ctx_t * ctx) { ulong endslotn = (endslot == NULL ? ULONG_MAX : (ulong)(*(long*)endslot)); fd_blockstore_t * blockstore = ctx->global->blockstore; - if (startslotn < blockstore->smr) /* FIXME query archival file */ - startslotn = blockstore->smr; - if (endslotn > blockstore->hcs) - endslotn = blockstore->hcs; + if (startslotn < blockstore->shmem->smr) /* FIXME query archival file */ + startslotn = blockstore->shmem->smr; + if (endslotn > blockstore->shmem->hcs) + endslotn = blockstore->shmem->hcs; fd_web_reply_sprintf(ws, "{\"jsonrpc\":\"2.0\",\"result\":["); uint cnt = 0; @@ -725,15 +725,15 @@ method_getBlocksWithLimit(struct json_values* values, fd_rpc_ctx_t * ctx) { ulong limitn = (ulong)(*(long*)limit); fd_blockstore_t * blockstore = ctx->global->blockstore; - if (startslotn < blockstore->smr) /* FIXME query archival file */ - startslotn = blockstore->smr; + if (startslotn < blockstore->shmem->smr) /* FIXME query archival file */ + startslotn = blockstore->shmem->smr; if (limitn > 500000) limitn = 500000; fd_web_reply_sprintf(ws, "{\"jsonrpc\":\"2.0\",\"result\":["); uint cnt = 0; uint skips = 0; - for ( ulong i = startslotn; i <= blockstore->lps && cnt < limitn && skips < 100U; ++i ) { + for ( ulong i = startslotn; i <= blockstore->shmem->lps && cnt < limitn && skips < 100U; ++i ) { fd_block_map_t meta[1]; int ret = fd_blockstore_block_map_query_volatile(blockstore, ctx->global->blockstore_fd, i, meta); if (!ret) { @@ -892,7 +892,7 @@ method_getFirstAvailableBlock(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_blockstore_t * blockstore = ctx->global->blockstore; fd_webserver_t * ws = &ctx->global->ws; fd_web_reply_sprintf(ws, "{\"jsonrpc\":\"2.0\",\"result\":%lu,\"id\":%s}" CRLF, - blockstore->smr, ctx->call_id); /* FIXME archival file */ + blockstore->shmem->smr, ctx->call_id); /* FIXME archival file */ return 0; } @@ -1121,7 +1121,7 @@ method_getMaxShredInsertSlot(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_blockstore_t * blockstore = ctx->global->blockstore; fd_webserver_t * ws = &ctx->global->ws; fd_web_reply_sprintf(ws, "{\"jsonrpc\":\"2.0\",\"result\":%lu,\"id\":%s}" CRLF, - blockstore->smr, ctx->call_id); /* FIXME archival file */ + blockstore->shmem->smr, ctx->call_id); /* FIXME archival file */ return 0; } @@ -1836,7 +1836,7 @@ method_minimumLedgerSlot(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_rpc_global_ctx_t * glob = ctx->global; fd_webserver_t * ws = &ctx->global->ws; fd_web_reply_sprintf(ws, "{\"jsonrpc\":\"2.0\",\"result\":%lu,\"id\":%s}" CRLF, - glob->blockstore->smr, ctx->call_id); /* FIXME archival file */ + glob->blockstore->shmem->smr, ctx->call_id); /* FIXME archival file */ return 0; } @@ -2496,13 +2496,13 @@ fd_rpc_start_service(fd_rpcserver_args_t * args, fd_rpc_ctx_t * ctx) { fd_rpc_global_ctx_t * gctx = ctx->global; gctx->funk = args->funk; - gctx->blockstore = args->blockstore; + memcpy( gctx->blockstore, args->blockstore, sizeof(fd_blockstore_t) ); gctx->blockstore_fd = args->blockstore_fd; fd_replay_notif_msg_t * msg = &gctx->last_slot_notify; msg->type = FD_REPLAY_SLOT_TYPE; - msg->slot_exec.slot = args->blockstore->smr; - msg->slot_exec.root = args->blockstore->smr; + msg->slot_exec.slot = args->blockstore->shmem->smr; + msg->slot_exec.root = args->blockstore->shmem->smr; } void diff --git a/src/disco/rpcserver/fd_rpc_service.h b/src/disco/rpcserver/fd_rpc_service.h index c8741167e0..d7cd62b792 100644 --- a/src/disco/rpcserver/fd_rpc_service.h +++ b/src/disco/rpcserver/fd_rpc_service.h @@ -19,6 +19,7 @@ struct fd_rpcserver_args { fd_valloc_t valloc; int offline; fd_funk_t * funk; + fd_blockstore_t blockstore_ljoin; fd_blockstore_t * blockstore; int blockstore_fd; fd_stake_ci_t * stake_ci; diff --git a/src/disco/shred/fd_shred_cap.c b/src/disco/shred/fd_shred_cap.c index 9790d3492f..3199dc5fec 100644 --- a/src/disco/shred/fd_shred_cap.c +++ b/src/disco/shred/fd_shred_cap.c @@ -56,7 +56,7 @@ fd_shred_cap_replay( const char * shred_cap_fpath, if ( bytes_read != shred_len ) break; fd_shred_t const * shred = fd_shred_parse( buffer, shred_len ); - if ( fd_store_shred_insert( store, shred ) < FD_BLOCKSTORE_OK ) return FD_SHRED_CAP_ERR; + if ( fd_store_shred_insert( store, shred ) < FD_BLOCKSTORE_SUCCESS ) return FD_SHRED_CAP_ERR; cnt++; /* if ( FD_SHRED_CAP_FLAG_IS_TURBINE(header.flags) ) { diff --git a/src/disco/store/fd_store.c b/src/disco/store/fd_store.c index 2223b16ee4..156a1deb6a 100644 --- a/src/disco/store/fd_store.c +++ b/src/disco/store/fd_store.c @@ -195,44 +195,41 @@ fd_store_slot_prepare( fd_store_t * store, int fd_store_shred_insert( fd_store_t * store, fd_shred_t const * shred ) { - if( FD_UNLIKELY( shred->version != store->expected_shred_version ) ) { FD_LOG_WARNING(( "received shred version %lu instead of %lu", (ulong)shred->version, store->expected_shred_version )); - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } fd_blockstore_t * blockstore = store->blockstore; - if (shred->slot < blockstore->smr) { - return FD_BLOCKSTORE_OK; + if (shred->slot < blockstore->shmem->smr) { + return FD_BLOCKSTORE_SUCCESS; } uchar shred_type = fd_shred_type( shred->variant ); - // FD_LOG_INFO(("is chained: %u", fd_shred_is_chained(shred_type) )); if( shred_type != FD_SHRED_TYPE_LEGACY_DATA && shred_type != FD_SHRED_TYPE_MERKLE_DATA && shred_type != FD_SHRED_TYPE_MERKLE_DATA_CHAINED && shred_type != FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ) { - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } if( store->root!=FD_SLOT_NULL && shred->slotroot ) { FD_LOG_WARNING(( "shred slot is behind root, dropping shred - root: %lu, shred_slot: %lu", store->root, shred->slot )); - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } - fd_blockstore_start_write( blockstore ); + fd_blockstore_start_read( blockstore ); if( fd_blockstore_shreds_complete( blockstore, shred->slot ) ) { - fd_blockstore_end_write( blockstore ); - return FD_BLOCKSTORE_OK; + fd_blockstore_end_read( blockstore ); + return FD_BLOCKSTORE_SUCCESS; } - int rc = fd_blockstore_shred_insert( blockstore, shred ); - fd_blockstore_end_write( blockstore ); + fd_blockstore_end_read( blockstore ); + fd_blockstore_shred_insert( blockstore, shred ); /* FIXME */ - if( FD_UNLIKELY( rc < FD_BLOCKSTORE_OK ) ) { - FD_LOG_ERR( ( "failed to insert shred. reason: %d", rc ) ); - } else if ( rc == FD_BLOCKSTORE_OK_SLOT_COMPLETE ) { + if( FD_UNLIKELY( fd_blockstore_shreds_complete( blockstore, shred->slot ) ) ) { fd_store_add_pending( store, shred->slot, (long)5e6, 0, 1 ); + return FD_BLOCKSTORE_SUCCESS_SLOT_COMPLETE; } else { fd_store_add_pending( store, shred->slot, FD_REPAIR_BACKOFF_TIME, 0, 0 ); fd_repair_backoff_t * backoff = fd_repair_backoff_map_query( store->repair_backoff_map, shred->slot, NULL ); @@ -246,8 +243,8 @@ fd_store_shred_insert( fd_store_t * store, backoff->last_backoff_duration = FD_REPAIR_BACKOFF_TIME; backoff->last_repair_time = store->now; } + return FD_BLOCKSTORE_SUCCESS; } - return rc; } void @@ -374,7 +371,7 @@ fd_store_slot_repair( fd_store_t * store, if( repair_req_cnt==out_repair_reqs_sz ) { backoff->last_backoff_duration += backoff->last_backoff_duration>>2; - FD_LOG_INFO( ( "[repair] MAX need %lu [%u, %u], sent %lu requests (backoff: %ld ms)", slot, block_map_entry->consumed_idx + 1, complete_idx, repair_req_cnt, backoff->last_backoff_duration/(long)1e6 ) ); + FD_LOG_INFO( ( "[repair] MAX need %lu [%u, %u], sent %lu requests (backoff: %ld ms)", slot, block_map_entry->buffered_idx + 1, complete_idx, repair_req_cnt, backoff->last_backoff_duration/(long)1e6 ) ); fd_blockstore_end_read( store->blockstore ); return repair_req_cnt; } @@ -399,8 +396,8 @@ fd_store_slot_repair( fd_store_t * store, } /* Fill in what's missing */ - for( uint i = block_map_entry->consumed_idx + 1; i <= complete_idx; i++ ) { - if( FD_UNLIKELY( fd_buf_shred_query( store->blockstore, slot, i ) != NULL) ) continue; + for( uint i = block_map_entry->buffered_idx + 1; i <= complete_idx; i++ ) { + if( FD_UNLIKELY( fd_blockstore_shred_test( store->blockstore, slot, i ) ) ) continue; fd_repair_request_t * repair_req = &out_repair_reqs[repair_req_cnt++]; repair_req->shred_index = i; @@ -409,14 +406,14 @@ fd_store_slot_repair( fd_store_t * store, if( repair_req_cnt == out_repair_reqs_sz ) { backoff->last_backoff_duration += backoff->last_backoff_duration>>2; - FD_LOG_INFO( ( "[repair] MAX need %lu [%u, %u], sent %lu requests (backoff: %ld ms)", slot, block_map_entry->consumed_idx + 1, complete_idx, repair_req_cnt, backoff->last_backoff_duration/(long)1e6 ) ); + FD_LOG_INFO( ( "[repair] MAX need %lu [%u, %u], sent %lu requests (backoff: %ld ms)", slot, block_map_entry->buffered_idx + 1, complete_idx, repair_req_cnt, backoff->last_backoff_duration/(long)1e6 ) ); fd_blockstore_end_read( store->blockstore ); return repair_req_cnt; } } if( repair_req_cnt ) { backoff->last_backoff_duration += backoff->last_backoff_duration>>2; - FD_LOG_INFO( ( "[repair] need %lu [%u, %u], sent %lu requests (backoff: %ld ms)", slot, block_map_entry->consumed_idx + 1, complete_idx, repair_req_cnt, backoff->last_backoff_duration/(long)1e6 ) ); + FD_LOG_INFO( ( "[repair] need %lu [%u, %u], sent %lu requests (backoff: %ld ms)", slot, block_map_entry->buffered_idx + 1, complete_idx, repair_req_cnt, backoff->last_backoff_duration/(long)1e6 ) ); } } diff --git a/src/flamenco/repair/fd_repair.c b/src/flamenco/repair/fd_repair.c index 03c10c7003..9f0fab1a43 100644 --- a/src/flamenco/repair/fd_repair.c +++ b/src/flamenco/repair/fd_repair.c @@ -502,7 +502,7 @@ fd_repair_send_requests( fd_repair_t * glob ) { fd_needed_table_remove( glob->needed, &n ); continue; } - + active->avg_reqs++; fd_repair_protocol_t protocol; @@ -602,7 +602,7 @@ long read_line(int fd, char *buf) { } static int -fd_read_in_good_peer_cache_file( fd_repair_t * repair ) { +fd_read_in_good_peer_cache_file( fd_repair_t * repair ) { if( repair->good_peer_cache_file_fd==-1 ) { FD_LOG_NOTICE(( "No repair good_peer_cache_file specified, not loading cached peers" )); return 0; @@ -667,7 +667,7 @@ fd_read_in_good_peer_cache_file( fd_repair_t * repair ) { /* Create the peer address struct (byte-swap the port to network order). */ fd_repair_peer_addr_t peer_addr; /* already in network byte order from inet_aton */ - peer_addr.addr = ip_addr; + peer_addr.addr = ip_addr; /* Flip to big-endian for network order */ peer_addr.port = fd_ushort_bswap( (ushort)port ); @@ -955,7 +955,7 @@ fd_actives_shuffle( fd_repair_t * repair ) { good[i]->sticky = (uchar)1; } if( leftovers_cnt ) { - /* Sample 64 new sticky peers using stake-weighted sampling */ + /* Sample 64 new sticky peers using stake-weighted sampling */ for( ulong i = 0; i < 64 && tot_cnt < FD_REPAIR_STICKY_MAX && tot_cnt < fd_active_table_key_cnt( repair->actives ); ++i ) { /* Generate a random amount of culmative stake at which to sample the peer */ ulong target_culm_stake = fd_rng_ulong( repair->rng ) % total_stake; @@ -979,7 +979,7 @@ fd_actives_shuffle( fd_repair_t * repair ) { peer->sticky = (uchar)1; } } - + } repair->actives_sticky_cnt = tot_cnt; @@ -1038,7 +1038,7 @@ fd_repair_create_needed_request( fd_repair_t * glob, int type, ulong slot, uint ids[i] = &peer->key; } - + if (!found_peer) { FD_LOG_DEBUG( ( "failed to find a good peer." ) ); fd_repair_unlock( glob ); @@ -1078,7 +1078,7 @@ fd_repair_create_needed_request( fd_repair_t * glob, int type, ulong slot, uint static int fd_write_good_peer_cache_file( fd_repair_t * repair ) { // return 0; - + if ( repair->good_peer_cache_file_fd == -1 ) { return 0; } @@ -1134,13 +1134,13 @@ fd_write_good_peer_cache_file( fd_repair_t * repair ) { int fd_repair_need_window_index( fd_repair_t * glob, ulong slot, uint shred_index ) { - // FD_LOG_INFO( ( "[repair] need window %lu, shred_index %lu", slot, shred_index ) ); + // FD_LOG_NOTICE(( "[repair] need window %lu, shred_index %u", slot, shred_index )); return fd_repair_create_needed_request( glob, fd_needed_window_index, slot, shred_index ); } int fd_repair_need_highest_window_index( fd_repair_t * glob, ulong slot, uint shred_index ) { - // FD_LOG_INFO( ( "[repair] need highest %lu", slot ) ); + // FD_LOG_NOTICE(( "[repair] need highest %lu", slot )); return fd_repair_create_needed_request( glob, fd_needed_highest_window_index, slot, shred_index ); } diff --git a/src/flamenco/runtime/Local.mk b/src/flamenco/runtime/Local.mk index 2fd53f0b2c..e1b29c19a4 100644 --- a/src/flamenco/runtime/Local.mk +++ b/src/flamenco/runtime/Local.mk @@ -10,6 +10,7 @@ $(call add-objs,fd_bank_hash_cmp,fd_flamenco) $(call add-hdrs,fd_blockstore.h fd_rwseq_lock.h) $(call add-objs,fd_blockstore,fd_flamenco) +$(call make-unit-test,test_blockstore,test_blockstore, fd_flamenco fd_util fd_ballet,$(SECP256K1_LIBS)) $(call add-hdrs,fd_borrowed_account.h) $(call add-objs,fd_borrowed_account,fd_flamenco) diff --git a/src/flamenco/runtime/fd_blockstore.c b/src/flamenco/runtime/fd_blockstore.c index 631b29c4a9..578345f0a1 100644 --- a/src/flamenco/runtime/fd_blockstore.c +++ b/src/flamenco/runtime/fd_blockstore.c @@ -1,4 +1,5 @@ #include "fd_blockstore.h" +#include "fd_rocksdb.h" #include #include #include @@ -13,15 +14,15 @@ fd_blockstore_new( void * shmem, ulong block_max, ulong idx_max, ulong txn_max ) { - fd_blockstore_t * blockstore = (fd_blockstore_t *)shmem; + fd_blockstore_shmem_t * blockstore_shmem = (fd_blockstore_shmem_t *)shmem; - if( FD_UNLIKELY( !blockstore ) ) { - FD_LOG_WARNING(( "NULL blockstore" )); + if( FD_UNLIKELY( !blockstore_shmem ) ) { + FD_LOG_WARNING(( "NULL blockstore_shmem" )); return NULL; } - if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)blockstore, fd_blockstore_align() ) )) { - FD_LOG_WARNING(( "misaligned blockstore" )); + if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)blockstore_shmem, fd_blockstore_align() ) )) { + FD_LOG_WARNING(( "misaligned blockstore_shmem" )); return NULL; } @@ -30,85 +31,98 @@ fd_blockstore_new( void * shmem, return NULL; } - fd_wksp_t * wksp = fd_wksp_containing( blockstore ); + fd_wksp_t * wksp = fd_wksp_containing( blockstore_shmem ); if( FD_UNLIKELY( !wksp ) ) { FD_LOG_WARNING(( "shmem must be part of a workspace" )); return NULL; } - fd_memset( blockstore, 0, fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ) ); + fd_memset( blockstore_shmem, 0, fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ) ); int lg_idx_max = fd_ulong_find_msb( fd_ulong_pow2_up( idx_max ) ); FD_SCRATCH_ALLOC_INIT( l, shmem ); - blockstore = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_blockstore_t), sizeof(fd_blockstore_t) ); - void * shred_pool = FD_SCRATCH_ALLOC_APPEND( l, fd_buf_shred_pool_align(), fd_buf_shred_pool_footprint( shred_max ) ); - void * shred_map = FD_SCRATCH_ALLOC_APPEND( l, fd_buf_shred_map_align(), fd_buf_shred_map_footprint( shred_max ) ); - void * block_map = FD_SCRATCH_ALLOC_APPEND( l, fd_block_map_align(), fd_block_map_footprint( block_max ) ); - void * block_idx = FD_SCRATCH_ALLOC_APPEND( l, fd_block_idx_align(), fd_block_idx_footprint( lg_idx_max ) ); - void * slot_deque = FD_SCRATCH_ALLOC_APPEND( l, fd_slot_deque_align(), fd_slot_deque_footprint( block_max ) ); - void * txn_map = FD_SCRATCH_ALLOC_APPEND( l, fd_txn_map_align(), fd_txn_map_footprint( txn_max ) ); - void * alloc = FD_SCRATCH_ALLOC_APPEND( l, fd_alloc_align(), fd_alloc_footprint() ); - FD_SCRATCH_ALLOC_FINI( l, fd_blockstore_align() ); - - blockstore->blockstore_gaddr = fd_wksp_gaddr_fast( wksp, blockstore ); - blockstore->wksp_tag = wksp_tag; - blockstore->seed = seed; + blockstore_shmem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_blockstore_shmem_t), sizeof(fd_blockstore_shmem_t) ); + void * shreds = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_buf_shred_t), sizeof(fd_buf_shred_t) * shred_max ); + void * shred_pool = FD_SCRATCH_ALLOC_APPEND( l, fd_buf_shred_pool_align(), fd_buf_shred_pool_footprint() ); + void * shred_map = FD_SCRATCH_ALLOC_APPEND( l, fd_buf_shred_map_align(), fd_buf_shred_map_footprint( shred_max ) ); + void * blocks = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_block_map_t), sizeof(fd_block_map_t) * block_max ); + void * block_map = FD_SCRATCH_ALLOC_APPEND( l, fd_block_map_align(), fd_block_map_footprint( block_max ) ); + void * block_idx = FD_SCRATCH_ALLOC_APPEND( l, fd_block_idx_align(), fd_block_idx_footprint( lg_idx_max ) ); + void * slot_deque = FD_SCRATCH_ALLOC_APPEND( l, fd_slot_deque_align(), fd_slot_deque_footprint( block_max ) ); + void * txn_map = FD_SCRATCH_ALLOC_APPEND( l, fd_txn_map_align(), fd_txn_map_footprint( txn_max ) ); + void * alloc = FD_SCRATCH_ALLOC_APPEND( l, fd_alloc_align(), fd_alloc_footprint() ); + ulong top = FD_SCRATCH_ALLOC_FINI( l, fd_blockstore_align() ); + FD_TEST( fd_ulong_align_up( top - (ulong)shmem, fd_alloc_align() ) == fd_ulong_align_up( fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ), fd_alloc_align() ) ); + + (void)shreds; + fd_buf_shred_pool_new( shred_pool ); + fd_buf_shred_map_new ( shred_map, shred_max, seed ); + + (void)blocks; + blockstore_shmem->block_map_gaddr = fd_wksp_gaddr( wksp, fd_block_map_join( fd_block_map_new( block_map, block_max, seed ) ) ); + blockstore_shmem->block_idx_gaddr = fd_wksp_gaddr( wksp, fd_block_idx_join( fd_block_idx_new( block_idx, lg_idx_max ) ) ); + blockstore_shmem->slot_deque_gaddr = fd_wksp_gaddr( wksp, fd_slot_deque_join (fd_slot_deque_new( slot_deque, block_max ) ) ); + blockstore_shmem->txn_map_gaddr = fd_wksp_gaddr( wksp, fd_txn_map_join (fd_txn_map_new( txn_map, txn_max, seed ) ) ); + blockstore_shmem->alloc_gaddr = fd_wksp_gaddr( wksp, fd_alloc_join (fd_alloc_new( alloc, wksp_tag ), wksp_tag ) ); + + FD_TEST( blockstore_shmem->block_idx_gaddr ); + FD_TEST( blockstore_shmem->slot_deque_gaddr ); + FD_TEST( blockstore_shmem->txn_map_gaddr ); + FD_TEST( blockstore_shmem->alloc_gaddr ); + + blockstore_shmem->blockstore_gaddr = fd_wksp_gaddr_fast( wksp, blockstore_shmem ); + blockstore_shmem->wksp_tag = wksp_tag; + blockstore_shmem->seed = seed; FD_COMPILER_MFENCE(); - fd_rwseq_new( &blockstore->lock ); + fd_rwseq_new( &blockstore_shmem->lock ); FD_COMPILER_MFENCE(); - blockstore->archiver = (fd_blockstore_archiver_t){ - .magic = FD_BLOCKSTORE_MAGIC, + blockstore_shmem->archiver = (fd_blockstore_archiver_t){ .fd_size_max = FD_BLOCKSTORE_ARCHIVE_MIN_SIZE, .head = FD_BLOCKSTORE_ARCHIVE_START, .tail = FD_BLOCKSTORE_ARCHIVE_START, .num_blocks = 0, }; - blockstore->lps = FD_SLOT_NULL; - blockstore->hcs = FD_SLOT_NULL; - blockstore->smr = FD_SLOT_NULL; - - blockstore->shred_max = shred_max; - blockstore->block_max = block_max; - blockstore->idx_max = idx_max; - blockstore->txn_max = txn_max; - - blockstore->shred_pool_gaddr = fd_wksp_gaddr( wksp, fd_buf_shred_pool_join( fd_buf_shred_pool_new( shred_pool, shred_max ) ) ); - blockstore->shred_map_gaddr = fd_wksp_gaddr( wksp, fd_buf_shred_map_join( fd_buf_shred_map_new( shred_map, shred_max, seed ) ) ); - blockstore->block_map_gaddr = fd_wksp_gaddr( wksp, fd_block_map_join( fd_block_map_new( block_map, block_max, seed ) ) ); - blockstore->block_idx_gaddr = fd_wksp_gaddr( wksp, fd_block_idx_join( fd_block_idx_new( block_idx, lg_idx_max ) ) ); - blockstore->slot_deque_gaddr = fd_wksp_gaddr( wksp, fd_slot_deque_join (fd_slot_deque_new( slot_deque, block_max ) ) ); - blockstore->txn_map_gaddr = fd_wksp_gaddr( wksp, fd_txn_map_join (fd_txn_map_new( txn_map, txn_max, seed ) ) ); - blockstore->alloc_gaddr = fd_wksp_gaddr( wksp, fd_alloc_join (fd_alloc_new( alloc, wksp_tag ), wksp_tag ) ); - - FD_TEST( blockstore->shred_pool_gaddr ); - FD_TEST( blockstore->shred_map_gaddr ); - FD_TEST( blockstore->block_map_gaddr ); - FD_TEST( blockstore->block_idx_gaddr ); - FD_TEST( blockstore->slot_deque_gaddr ); - FD_TEST( blockstore->txn_map_gaddr ); - FD_TEST( blockstore->alloc_gaddr ); + blockstore_shmem->lps = FD_SLOT_NULL; + blockstore_shmem->hcs = FD_SLOT_NULL; + blockstore_shmem->smr = FD_SLOT_NULL; + + blockstore_shmem->shred_max = shred_max; + blockstore_shmem->block_max = block_max; + blockstore_shmem->idx_max = idx_max; + blockstore_shmem->txn_max = txn_max; FD_COMPILER_MFENCE(); - FD_VOLATILE( blockstore->magic ) = FD_BLOCKSTORE_MAGIC; + FD_VOLATILE( blockstore_shmem->magic ) = FD_BLOCKSTORE_MAGIC; FD_COMPILER_MFENCE(); - return (void *)blockstore; + return (void *)blockstore_shmem; } fd_blockstore_t * -fd_blockstore_join( void * shblockstore ) { - fd_blockstore_t * blockstore = (fd_blockstore_t *)shblockstore; +fd_blockstore_join( void * ljoin, void * shblockstore ) { + fd_blockstore_t * join = (fd_blockstore_t *)ljoin; + fd_blockstore_shmem_t * blockstore = (fd_blockstore_shmem_t *)shblockstore; + + if( FD_UNLIKELY( !join ) ) { + FD_LOG_WARNING(( "NULL ljoin" )); + return NULL; + } + + if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)join, alignof(fd_blockstore_t) ) ) ) { + FD_LOG_WARNING(( "misaligned ljoin" )); + return NULL; + } if( FD_UNLIKELY( !blockstore ) ) { FD_LOG_WARNING(( "NULL shblockstore" )); return NULL; } - if( FD_UNLIKELY( !fd_ulong_is_aligned((ulong)blockstore, fd_blockstore_align() ) )) { + if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)blockstore, fd_blockstore_align() ) )) { FD_LOG_WARNING(( "misaligned shblockstore" )); return NULL; } @@ -118,7 +132,20 @@ fd_blockstore_join( void * shblockstore ) { return NULL; } - return blockstore; + FD_SCRATCH_ALLOC_INIT( l, shblockstore ); + blockstore = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_blockstore_shmem_t), sizeof(fd_blockstore_shmem_t) ); + void * shreds = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_buf_shred_t), sizeof(fd_buf_shred_t) * blockstore->shred_max ); + void * shred_pool = FD_SCRATCH_ALLOC_APPEND( l, fd_buf_shred_pool_align(), fd_buf_shred_pool_footprint() ); + void * shred_map = FD_SCRATCH_ALLOC_APPEND( l, fd_buf_shred_map_align(), fd_buf_shred_map_footprint( blockstore->shred_max ) ); + FD_SCRATCH_ALLOC_FINI( l, fd_blockstore_align() ); + + join->shmem = blockstore; + fd_buf_shred_pool_join( join->shred_pool, shred_pool, shreds, blockstore->shred_max ); + fd_buf_shred_map_join ( join->shred_map, shred_map, shreds, blockstore->shred_max ); + // FD_TEST( fd_buf_shred_pool_verify( join->shred_pool ) == FD_POOL_SUCCESS ); + // FD_TEST( fd_buf_shred_map_verify ( join->shred_map ) == FD_MAP_SUCCESS ); + + return join; } void * @@ -135,8 +162,8 @@ fd_blockstore_leave( fd_blockstore_t * blockstore ) { return NULL; } - FD_TEST( fd_buf_shred_pool_leave( fd_blockstore_shred_pool( blockstore ) ) ); - FD_TEST( fd_buf_shred_map_leave( fd_blockstore_shred_map( blockstore ) ) ); + FD_TEST( fd_buf_shred_pool_leave( blockstore->shred_pool ) ); + FD_TEST( fd_buf_shred_map_leave( blockstore->shred_map ) ); FD_TEST( fd_block_map_leave( fd_blockstore_block_map( blockstore ) ) ); FD_TEST( fd_block_idx_leave( fd_blockstore_block_idx( blockstore ) ) ); FD_TEST( fd_slot_deque_leave( fd_blockstore_slot_deque( blockstore ) ) ); @@ -160,7 +187,7 @@ fd_blockstore_delete( void * shblockstore ) { return NULL; } - if( FD_UNLIKELY( blockstore->magic != FD_BLOCKSTORE_MAGIC ) ) { + if( FD_UNLIKELY( blockstore->shmem->magic != FD_BLOCKSTORE_MAGIC ) ) { FD_LOG_WARNING(( "bad magic" )); return NULL; } @@ -173,8 +200,8 @@ fd_blockstore_delete( void * shblockstore ) { /* Delete all structures. */ - FD_TEST( fd_buf_shred_pool_delete( fd_blockstore_shred_pool( blockstore ) ) ); - FD_TEST( fd_buf_shred_map_delete( fd_blockstore_shred_map( blockstore ) ) ); + FD_TEST( fd_buf_shred_pool_delete( &blockstore->shred_pool ) ); + FD_TEST( fd_buf_shred_map_delete( &blockstore->shred_map ) ); FD_TEST( fd_block_map_delete( fd_blockstore_block_map( blockstore ) ) ); FD_TEST( fd_block_idx_delete( fd_blockstore_block_idx( blockstore ) ) ); FD_TEST( fd_slot_deque_delete( fd_blockstore_slot_deque( blockstore ) ) ); @@ -182,7 +209,7 @@ fd_blockstore_delete( void * shblockstore ) { FD_TEST( fd_alloc_delete( fd_blockstore_alloc( blockstore ) ) ); FD_COMPILER_MFENCE(); - FD_VOLATILE( blockstore->magic ) = 0UL; + FD_VOLATILE( blockstore->shmem->magic ) = 0UL; FD_COMPILER_MFENCE(); return blockstore; @@ -205,8 +232,8 @@ fd_blockstore_archiver_lrw_slot( fd_blockstore_t * blockstore, int fd, fd_block_ } fd_block_idx_t lrw_block_idx = { 0 }; - lrw_block_idx.off = blockstore->archiver.head; - int err = fd_blockstore_block_meta_restore(&blockstore->archiver, fd, &lrw_block_idx, lrw_block_map, lrw_block); + lrw_block_idx.off = blockstore->shmem->archiver.head; + int err = fd_blockstore_block_meta_restore(&blockstore->shmem->archiver, fd, &lrw_block_idx, lrw_block_map, lrw_block); check_read_write_err( err ); return lrw_block_map->slot; } @@ -215,8 +242,7 @@ bool fd_blockstore_archiver_verify( fd_blockstore_t * blockstore, fd_blockstore_archiver_t * fd_metadata ) { return ( fd_metadata->head < FD_BLOCKSTORE_ARCHIVE_START ) || ( fd_metadata->tail < FD_BLOCKSTORE_ARCHIVE_START ) - || ( fd_metadata->fd_size_max != blockstore->archiver.fd_size_max ) // should be initialized same as archive file - || ( fd_metadata->magic != FD_BLOCKSTORE_MAGIC ); + || ( fd_metadata->fd_size_max != blockstore->shmem->archiver.fd_size_max ); // should be initialized same as archive file } #define check_read_err_safe( cond, msg ) \ @@ -259,7 +285,7 @@ static int read_with_wraparound( fd_blockstore_archiver_t * archvr, *read_off = FD_BLOCKSTORE_ARCHIVE_START; } - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } static ulong @@ -306,7 +332,7 @@ build_idx( fd_blockstore_t * blockstore, int fd ) { return; } - blockstore->archiver = metadata; + blockstore->shmem->archiver = metadata; ulong off = metadata.head; ulong total_blocks = metadata.num_blocks; ulong blocks_read = 0; @@ -318,7 +344,7 @@ build_idx( fd_blockstore_t * blockstore, int fd ) { blocks_read++; fd_block_idx_t block_idx_entry = { 0 }; block_idx_entry.off = off; - err = fd_blockstore_block_meta_restore( &blockstore->archiver, fd, &block_idx_entry, &block_map_out, &block_out ); + err = fd_blockstore_block_meta_restore( &blockstore->shmem->archiver, fd, &block_idx_entry, &block_map_out, &block_out ); check_read_write_err( err ); if( FD_UNLIKELY( fd_block_idx_key_cnt( block_idx ) == fd_block_idx_key_max( block_idx ) ) ) { @@ -330,8 +356,8 @@ build_idx( fd_blockstore_t * blockstore, int fd ) { fd_block_idx_t * lrw_block_index = fd_block_idx_query( block_idx, lrw_slot, NULL ); fd_block_idx_remove( block_idx, lrw_block_index ); - blockstore->archiver.head = wrap_offset(&blockstore->archiver, blockstore->archiver.head + lrw_block.data_sz + sizeof(fd_block_map_t) + sizeof(fd_block_t));; - blockstore->archiver.num_blocks--; + blockstore->shmem->archiver.head = wrap_offset(&blockstore->shmem->archiver, blockstore->shmem->archiver.head + lrw_block.data_sz + sizeof(fd_block_map_t) + sizeof(fd_block_t));; + blockstore->shmem->archiver.num_blocks--; } fd_block_idx_t * idx_entry = fd_block_idx_query( block_idx, block_map_out.slot, NULL ); if ( FD_UNLIKELY( idx_entry ) ) { @@ -343,12 +369,12 @@ build_idx( fd_blockstore_t * blockstore, int fd ) { idx_entry->off = off; idx_entry->block_hash = block_map_out.block_hash; idx_entry->bank_hash = block_map_out.bank_hash; - blockstore->mrw_slot = block_map_out.slot; + blockstore->shmem->mrw_slot = block_map_out.slot; FD_LOG_NOTICE(( "[%s] read block (%lu/%lu) at offset: %lu. slot no: %lu", __func__, blocks_read, total_blocks, off, block_map_out.slot )); /* seek past data */ - off = wrap_offset( &blockstore->archiver, off + sizeof(fd_block_map_t) + sizeof(fd_block_t) + block_out.data_sz ); + off = wrap_offset( &blockstore->shmem->archiver, off + sizeof(fd_block_map_t) + sizeof(fd_block_t) + block_out.data_sz ); check_read_write_err( lseek( fd, (long)off, SEEK_SET ) == -1); } FD_LOG_NOTICE(( "[%s] successfully indexed blockstore archival file. entries: %lu", __func__, fd_block_idx_key_cnt( block_idx ) )); @@ -356,11 +382,12 @@ build_idx( fd_blockstore_t * blockstore, int fd ) { fd_blockstore_t * fd_blockstore_init( fd_blockstore_t * blockstore, int fd, ulong fd_size_max, fd_slot_bank_t const * slot_bank ) { + if ( fd_size_max < FD_BLOCKSTORE_ARCHIVE_MIN_SIZE ) { FD_LOG_ERR(( "archive file size too small" )); return NULL; } - blockstore->archiver.fd_size_max = fd_size_max; + blockstore->shmem->archiver.fd_size_max = fd_size_max; build_idx( blockstore, fd ); lseek( fd, 0, SEEK_END ); @@ -369,10 +396,10 @@ fd_blockstore_init( fd_blockstore_t * blockstore, int fd, ulong fd_size_max, fd_ ulong smr = slot_bank->slot; - blockstore->lps = smr; - blockstore->hcs = smr; - blockstore->smr = smr; - blockstore->wmk = smr; + blockstore->shmem->lps = smr; + blockstore->shmem->hcs = smr; + blockstore->shmem->smr = smr; + blockstore->shmem->wmk = smr; fd_block_map_t * block_map_entry = fd_block_map_insert( fd_blockstore_block_map( blockstore ), &smr ); @@ -399,8 +426,8 @@ fd_blockstore_init( fd_blockstore_t * blockstore, int fd, ulong fd_size_max, fd_ block_map_entry->ts = 0; block_map_entry->consumed_idx = 0; + block_map_entry->buffered_idx = 0; block_map_entry->received_idx = 0; - block_map_entry->replayed_idx = 0; block_map_entry->data_complete_idx = 0; block_map_entry->slot_complete_idx = 0; @@ -463,7 +490,7 @@ fd_txn_key_hash( fd_txn_key_t const * k, ulong seed ) { return h; } -static void +void fd_blockstore_scan_block( fd_blockstore_t * blockstore, ulong slot, fd_block_t * block ) { fd_block_micro_t * micros = fd_alloc_malloc( fd_blockstore_alloc( blockstore ), @@ -632,14 +659,8 @@ fd_blockstore_slot_remove( fd_blockstore_t * blockstore, ulong slot ) { /* Remove buf_shreds if there's no block yet (we haven't received all shreds). */ - fd_buf_shred_map_t * map = fd_blockstore_shred_map( blockstore ); - fd_buf_shred_t * pool = fd_blockstore_shred_pool( blockstore ); for( uint idx = 0; idx < block_map_entry->received_idx; idx++ ) { - fd_shred_key_t key = { .slot = slot, .idx = idx }; - fd_buf_shred_t * buf_shred = fd_buf_shred_map_ele_remove( map, &key, NULL, pool ); - if ( FD_LIKELY( buf_shred ) ) { - fd_buf_shred_pool_ele_release( pool, buf_shred ); - } + fd_blockstore_shred_remove( blockstore, block_map_entry->slot, idx ); } /* Return early because there are no allocations without a block. */ @@ -679,25 +700,6 @@ fd_blockstore_slot_remove( fd_blockstore_t * blockstore, ulong slot ) { return; } -int -fd_blockstore_buffered_shreds_remove( fd_blockstore_t * blockstore, ulong slot ) { - fd_block_map_t * block_map = fd_blockstore_block_map( blockstore ); - fd_block_map_t * block_map_entry = fd_block_map_query( block_map, &slot, NULL ); - if( FD_UNLIKELY( !block_map_entry ) ) return FD_BLOCKSTORE_OK; - fd_buf_shred_t * shred_pool = fd_blockstore_shred_pool( blockstore ); - fd_buf_shred_map_t * shred_map = fd_blockstore_shred_map( blockstore ); - ulong shred_cnt = block_map_entry->slot_complete_idx + 1; - for( uint i = 0; i < shred_cnt; i++ ) { - fd_shred_key_t key = { .slot = slot, .idx = i }; - fd_buf_shred_t * ele; - while( FD_UNLIKELY( - ele = fd_buf_shred_map_ele_remove( shred_map, &key, NULL, shred_pool ) ) ) - fd_buf_shred_pool_ele_release( shred_pool, ele ); - } - fd_block_map_remove( block_map, &slot ); - return FD_BLOCKSTORE_OK; -} - /** Where write_off is where we want to write to, and we return the next valid location to write to (either wraparound, or right after where we just wrote ) */ @@ -756,7 +758,7 @@ static void end_archive_write( fd_blockstore_archiver_t * archvr, /* Clears any to be overwritten blocks in the archive from the index and updates archvr */ static void fd_blockstore_lrw_archive_clear( fd_blockstore_t * blockstore, int fd, ulong wsz, ulong write_off ) { - fd_blockstore_archiver_t * archvr = &blockstore->archiver; + fd_blockstore_archiver_t * archvr = &blockstore->shmem->archiver; fd_block_idx_t * block_idx = fd_blockstore_block_idx( blockstore ); ulong non_wrapped_end = write_off + wsz; @@ -800,7 +802,7 @@ static void fd_blockstore_post_checkpt_update( fd_blockstore_t * blockstore, ulong slot, ulong wsz, ulong write_off ) { - fd_blockstore_archiver_t * archvr = &blockstore->archiver; + fd_blockstore_archiver_t * archvr = &blockstore->shmem->archiver; fd_block_idx_t * block_idx = fd_blockstore_block_idx( blockstore ); /* Successfully archived block, so update index and offset. */ @@ -824,7 +826,7 @@ static void fd_blockstore_post_checkpt_update( fd_blockstore_t * blockstore, archvr->num_blocks++; archvr->tail = wrap_offset( archvr, write_off + wsz);; - blockstore->mrw_slot = slot; + blockstore->shmem->mrw_slot = slot; } ulong @@ -832,7 +834,7 @@ fd_blockstore_block_checkpt( fd_blockstore_t * blockstore, fd_blockstore_ser_t * ser, int fd, ulong slot ) { - ulong write_off = blockstore->archiver.tail; + ulong write_off = blockstore->shmem->archiver.tail; ulong og_write_off = write_off; if ( FD_UNLIKELY( fd == -1 ) ) { FD_LOG_DEBUG(( "[%s] fd is -1", __func__ )); @@ -847,15 +849,15 @@ fd_blockstore_block_checkpt( fd_blockstore_t * blockstore, /* clear any potential overwrites */ fd_blockstore_lrw_archive_clear( blockstore, fd, total_wsz, write_off ); - start_archive_write( &blockstore->archiver, fd ); + start_archive_write( &blockstore->shmem->archiver, fd ); - write_off = write_with_wraparound( &blockstore->archiver, fd, (uchar*)ser->block_map, sizeof(fd_block_map_t), write_off ); - write_off = write_with_wraparound( &blockstore->archiver, fd, (uchar*)ser->block, sizeof(fd_block_t), write_off ); - write_off = write_with_wraparound( &blockstore->archiver, fd, ser->data, ser->block->data_sz, write_off ); + write_off = write_with_wraparound( &blockstore->shmem->archiver, fd, (uchar*)ser->block_map, sizeof(fd_block_map_t), write_off ); + write_off = write_with_wraparound( &blockstore->shmem->archiver, fd, (uchar*)ser->block, sizeof(fd_block_t), write_off ); + write_off = write_with_wraparound( &blockstore->shmem->archiver, fd, ser->data, ser->block->data_sz, write_off ); fd_blockstore_post_checkpt_update( blockstore, ser, fd, slot, total_wsz, og_write_off ); - end_archive_write( &blockstore->archiver, fd ); + end_archive_write( &blockstore->shmem->archiver, fd ); FD_LOG_NOTICE(( "[%s] archived block %lu at %lu: size %lu", __func__, slot, og_write_off, total_wsz )); return total_wsz; @@ -883,7 +885,7 @@ fd_blockstore_block_meta_restore( fd_blockstore_archiver_t * archvr, &rsz, &read_off ); check_read_err_safe( err, "failed to read block" ); - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } int @@ -905,17 +907,17 @@ fd_blockstore_block_data_restore( fd_blockstore_archiver_t * archvr, ulong rsz; int err = read_with_wraparound( archvr, fd, buf_out, data_sz, &rsz, &data_off ); check_read_err_safe( err, "failed to read block data" ); - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } void fd_blockstore_publish( fd_blockstore_t * blockstore, int fd, ulong wmk ) { - FD_LOG_NOTICE(( "[%s] wmk %lu => smr %lu", __func__, blockstore->wmk, wmk )); + FD_LOG_NOTICE(( "[%s] wmk %lu => smr %lu", __func__, blockstore->shmem->wmk, wmk )); /* Caller is incorrectly calling publish. */ - if( FD_UNLIKELY( blockstore->wmk == wmk ) ) { - FD_LOG_WARNING(( "[%s] attempting to re-publish when wmk %lu already at smr %lu", __func__, blockstore->wmk, wmk )); + if( FD_UNLIKELY( blockstore->shmem->wmk == wmk ) ) { + FD_LOG_WARNING(( "[%s] attempting to re-publish when wmk %lu already at smr %lu", __func__, blockstore->shmem->wmk, wmk )); return; } @@ -929,7 +931,7 @@ fd_blockstore_publish( fd_blockstore_t * blockstore, int fd, ulong wmk ) { /* Push the watermark onto the queue. */ - fd_slot_deque_push_tail( q, blockstore->wmk ); + fd_slot_deque_push_tail( q, blockstore->shmem->wmk ); /* Conduct a BFS to find slots to prune or archive. */ @@ -974,41 +976,69 @@ fd_blockstore_publish( fd_blockstore_t * blockstore, int fd, ulong wmk ) { /* Scan to clean up any orphaned blocks or shreds < new SMR. */ - for (ulong slot = blockstore->wmk; slot < wmk; slot++) { + for (ulong slot = blockstore->shmem->wmk; slot < wmk; slot++) { fd_blockstore_slot_remove( blockstore, slot ); } - blockstore->wmk = wmk; + blockstore->shmem->wmk = wmk; return; } +void +fd_blockstore_shred_remove( fd_blockstore_t * blockstore, ulong slot, uint idx ) { + // if ( fd_buf_shred_pool_verify( blockstore->shred_pool ) != FD_POOL_SUCCESS || fd_buf_shred_map_verify ( blockstore->shred_map ) != FD_MAP_SUCCESS ) { + // FD_LOG_NOTICE(( "slot %lu idx %u", slot, idx )); + // __asm__("int $3"); + // } + fd_shred_key_t key = { slot, idx }; + + fd_buf_shred_map_query_t query[1] = { 0 }; + int err = fd_buf_shred_map_remove( blockstore->shred_map, &key, NULL, query, FD_MAP_FLAG_BLOCKING ); + if( FD_UNLIKELY( err == FD_MAP_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] map corrupt: shred %lu %u", __func__, slot, idx )); + + if( FD_LIKELY( err == FD_MAP_SUCCESS ) ) { + fd_buf_shred_t * shred = fd_buf_shred_map_query_ele( query ); + int err = fd_buf_shred_pool_release( blockstore->shred_pool, shred, 1 ); + if( FD_UNLIKELY( err == FD_POOL_ERR_INVAL ) ) FD_LOG_ERR(( "[%s] pool error: shred %lu %u not in pool", __func__, slot, idx )); + if( FD_UNLIKELY( err == FD_POOL_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] pool corrupt: shred %lu %u", __func__, slot, idx )); + FD_TEST( !err ); + } + // FD_TEST( fd_buf_shred_pool_verify( blockstore->shred_pool ) == FD_POOL_SUCCESS ); + // FD_TEST( fd_buf_shred_map_verify ( blockstore->shred_map ) == FD_MAP_SUCCESS ); +} + /* Deshred into a block once we've received all shreds for a slot. */ static int deshred( fd_blockstore_t * blockstore, ulong slot ) { - FD_LOG_DEBUG(( "[%s] slot %lu", __func__, slot )); + FD_LOG_NOTICE(( "[%s] slot %lu", __func__, slot )); fd_block_map_t * block_map_entry = fd_blockstore_block_map_query( blockstore, slot ); FD_TEST( block_map_entry->block_gaddr == 0 ); /* FIXME duplicate blocks are not supported */ block_map_entry->ts = fd_log_wallclock(); - - fd_buf_shred_t * shred_pool = fd_blockstore_shred_pool( blockstore ); - fd_buf_shred_map_t * shred_map = fd_blockstore_shred_map( blockstore ); - ulong block_sz = 0UL; ulong shred_cnt = block_map_entry->slot_complete_idx + 1; ulong batch_cnt = 0UL; + fd_shred_t shred_hdr; for( uint idx = 0; idx < shred_cnt; idx++ ) { - fd_shred_key_t key = { .slot = slot, .idx = idx }; - fd_buf_shred_t const * query = fd_buf_shred_map_ele_query_const( shred_map, &key, NULL, shred_pool ); - if( FD_UNLIKELY( !query ) ) { - FD_LOG_ERR(( "[%s] missing shred slot: %lu idx: %u while deshredding", __func__, slot, idx )); + fd_shred_key_t key = { slot, idx }; + int err = FD_MAP_ERR_AGAIN; + while( err == FD_MAP_ERR_AGAIN ) { + fd_buf_shred_map_query_t query[1] = { 0 };; + err = fd_buf_shred_map_query_try( blockstore->shred_map, &key, NULL, query ); + if( FD_UNLIKELY( err == FD_MAP_ERR_KEY ) ) FD_LOG_ERR(( "[%s] map missing shred %lu %u while deshredding", __func__, slot, idx )); + if( FD_UNLIKELY( err == FD_MAP_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] map corrupt. shred %lu %u", __func__, slot, idx )); + fd_buf_shred_t const * shred = fd_buf_shred_map_query_ele_const( query ); + shred_hdr = shred->hdr; + err = fd_buf_shred_map_query_test( query ); } - block_sz += fd_shred_payload_sz( &query->hdr ); - if( FD_LIKELY( (query->hdr.data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE) || query->hdr.data.flags & FD_SHRED_DATA_FLAG_DATA_COMPLETE ) ) { + FD_TEST( !err ); + block_sz += fd_shred_payload_sz( &shred_hdr ); + if( FD_LIKELY( ( shred_hdr.data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE ) || + shred_hdr.data.flags & FD_SHRED_DATA_FLAG_DATA_COMPLETE ) ) { batch_cnt++; } } @@ -1038,176 +1068,117 @@ deshred( fd_blockstore_t * blockstore, ulong slot ) { block->batch_gaddr = fd_wksp_gaddr_fast( wksp, batch_laddr ); block->batch_cnt = batch_cnt; - /* deshred the shreds into the block mem */ - fd_deshredder_t deshredder; - fd_deshredder_init( &deshredder, data_laddr, block->data_sz, NULL, 0 ); - long rc = -FD_SHRED_EINVAL; ulong off = 0UL; ulong batch_i = 0UL; - for( uint i = 0; i < shred_cnt; i++ ) { + for( uint idx = 0; idx < shred_cnt; idx++ ) { // TODO can do this in one iteration with block sz loop... massage with deshredder API - fd_shred_key_t key = { .slot = slot, .idx = i }; - fd_buf_shred_t const * query = - fd_buf_shred_map_ele_query_const( shred_map, &key, NULL, shred_pool ); - if( FD_UNLIKELY( !query ) ) FD_LOG_ERR(( "missing shred idx %u during deshred. slot %lu.", i, slot )); - /* This is a hack to set up the internal state of the deshreddder - such that it processes exactly one shred. - TODO improve deshredder API - */ - fd_shred_t const * shred = &query->hdr; - deshredder.shreds = &shred; - deshredder.shred_cnt = 1; - rc = fd_deshredder_next( &deshredder ); - FD_TEST( rc >= 0 ); - - shreds_laddr[i].hdr = *shred; - ulong merkle_sz = shreds_laddr[i].merkle_sz = fd_shred_merkle_sz( shred->variant ); - FD_TEST( merkle_sz <= sizeof(shreds_laddr[i].merkle) ); - if( merkle_sz ) { - fd_memcpy( shreds_laddr[i].merkle, (uchar const*)shred + fd_shred_merkle_off( shred ), merkle_sz ); + fd_shred_key_t key = { slot, idx }; + ulong payload_sz = 0UL; + uchar flags = 0; + int err = FD_MAP_ERR_AGAIN; + while( err == FD_MAP_ERR_AGAIN ) { + fd_buf_shred_map_query_t query[1] = { 0 };; + err = fd_buf_shred_map_query_try( blockstore->shred_map, &key, NULL, query ); + fd_shred_t const * shred = &fd_buf_shred_map_query_ele_const( query )->hdr; + memcpy( data_laddr + off, fd_shred_data_payload( shred ), fd_shred_payload_sz( shred ) ); + + shreds_laddr[idx].hdr = *shred; + shreds_laddr[idx].off = off; + FD_TEST( 0 == memcmp( &shreds_laddr[idx].hdr, shred, sizeof( fd_shred_t ) ) ); + FD_TEST( 0 == memcmp( data_laddr + shreds_laddr[idx].off, fd_shred_data_payload( shred ), fd_shred_payload_sz( shred ) ) ); + + payload_sz = fd_shred_payload_sz( shred ); + flags = shred->data.flags; + + err = fd_buf_shred_map_query_test( query ); + if( FD_UNLIKELY( err == FD_MAP_ERR_KEY ) ) FD_LOG_ERR(( "[%s] map missing shred %lu %u while deshredding", __func__, slot, idx )); + if( FD_UNLIKELY( err == FD_MAP_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] map corrupt. shred %lu %u", __func__, slot, idx )); } - shreds_laddr[i].off = off; - - FD_TEST( !memcmp( &shreds_laddr[i].hdr, shred, sizeof( fd_shred_t ) ) ); - FD_TEST( !memcmp( data_laddr + shreds_laddr[i].off, - fd_shred_data_payload( shred ), - fd_shred_payload_sz( shred ) ) ); - - off += fd_shred_payload_sz( shred ); - - if( FD_LIKELY( (query->hdr.data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE) || query->hdr.data.flags & FD_SHRED_DATA_FLAG_DATA_COMPLETE ) ) { + FD_TEST( !err ); + off += payload_sz; + if( FD_LIKELY( (flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE) || flags & FD_SHRED_DATA_FLAG_DATA_COMPLETE ) ) { batch_laddr[ batch_i++ ].end_off = off; } - - fd_buf_shred_t * ele = fd_buf_shred_map_ele_remove( shred_map, &key, NULL, shred_pool ); - FD_TEST( ele ); - fd_buf_shred_pool_ele_release( shred_pool, ele ); + // fd_blockstore_shred_remove( blockstore, slot, idx ); } if( FD_UNLIKELY( batch_cnt != batch_i ) ) { FD_LOG_ERR(( "batch_cnt(%lu)!=batch_i(%lu) potential memory corruption", batch_cnt, batch_i )); } - /* deshredder error handling */ - int err; - switch( rc ) { - case -FD_SHRED_EINVAL: - err = FD_BLOCKSTORE_ERR_SHRED_INVALID; - goto fail_deshred; - case -FD_SHRED_ENOMEM: - err = FD_BLOCKSTORE_ERR_NO_MEM; - FD_LOG_ERR(( "should have alloc'd enough memory above. likely indicates memory corruption." )); - } - - switch( deshredder.result ) { - case FD_SHRED_ESLOT: - fd_blockstore_scan_block( blockstore, slot, block ); - - /* Do this last when it's safe */ - FD_COMPILER_MFENCE(); - block_map_entry->block_gaddr = fd_wksp_gaddr_fast( wksp, block ); - fd_block_micro_t * micros = fd_wksp_laddr_fast( wksp, block->micros_gaddr ); - uchar * data = fd_wksp_laddr_fast( wksp, block->data_gaddr ); - fd_microblock_hdr_t * last_micro = (fd_microblock_hdr_t *)( data + - micros[block->micros_cnt - 1].off ); - memcpy( &block_map_entry->block_hash, last_micro->hash, sizeof( fd_hash_t ) ); - - block_map_entry->flags = fd_uchar_clear_bit( block_map_entry->flags, FD_BLOCK_FLAG_RECEIVING ); - block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, FD_BLOCK_FLAG_COMPLETED ); - - return FD_BLOCKSTORE_OK; - case FD_SHRED_EBATCH: - case FD_SHRED_EPIPE: - FD_LOG_WARNING(( "deshredding slot %lu produced invalid block", slot )); - err = FD_BLOCKSTORE_ERR_DESHRED_INVALID; - goto fail_deshred; - default: - err = FD_BLOCKSTORE_ERR_UNKNOWN; - } - -fail_deshred: - /* We failed to deshred the block. Throw it away, and try again from scratch. */ - FD_LOG_WARNING(( "[%s] failed to deshred slot %lu. err: %d", __func__, slot, err )); - fd_alloc_free( alloc, block ); - fd_blockstore_slot_remove( blockstore, slot ); - for( uint i = 0; i < shred_cnt; i++ ) { - fd_shred_key_t key = { .slot = slot, .idx = i }; - fd_buf_shred_map_ele_remove( shred_map, &key, NULL, shred_pool ); - } - return err; -} - - -/* Check if we're seeing a different payload for the same shred key, - which indicates equivocation. */ - -static int -is_eqvoc_fec( fd_shred_t * old, fd_shred_t const * new ) { - if( FD_UNLIKELY( fd_shred_type( old->variant ) != fd_shred_type( new->variant ) ) ) { - FD_LOG_WARNING(( "[%s] shred %lu %u not both resigned", __func__, old->slot, old->idx )); - return 1; - } + fd_blockstore_scan_block( blockstore, slot, block ); - if( FD_UNLIKELY( fd_shred_payload_sz( old ) != fd_shred_payload_sz( new ) ) ) { - FD_LOG_WARNING(( "[%s] shred %lu %u payload_sz not eq", __func__, old->slot, old->idx )); - return 1; - } + /* Do this last when it's safe */ + FD_COMPILER_MFENCE(); + block_map_entry->block_gaddr = fd_wksp_gaddr_fast( wksp, block ); + fd_block_micro_t * micros = fd_wksp_laddr_fast( wksp, block->micros_gaddr ); + uchar * data = fd_wksp_laddr_fast( wksp, block->data_gaddr ); + fd_microblock_hdr_t * last_micro = (fd_microblock_hdr_t *)( data + micros[block->micros_cnt - 1].off ); + memcpy( &block_map_entry->block_hash, last_micro->hash, sizeof( fd_hash_t ) ); - ulong memcmp_sz = fd_ulong_if( fd_shred_payload_sz( old ) > FD_SHRED_SIGNATURE_SZ && - fd_shred_is_resigned( fd_shred_type( old->variant ) ), - fd_shred_payload_sz( old ) - FD_SHRED_SIGNATURE_SZ, - fd_shred_payload_sz( old ) ); - if( FD_UNLIKELY( 0 != memcmp( fd_shred_data_payload( old ), fd_shred_data_payload( new ), memcmp_sz ) ) ) { - FD_LOG_WARNING(( "[%s] shred %lu %u payload not eq", __func__, old->slot, old->idx )); - return 1; - } + block_map_entry->flags = fd_uchar_clear_bit( block_map_entry->flags, FD_BLOCK_FLAG_RECEIVING ); + block_map_entry->flags = fd_uchar_set_bit( block_map_entry->flags, FD_BLOCK_FLAG_COMPLETED ); - return 0; + return FD_BLOCKSTORE_SUCCESS; } -int +void fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shred ) { - FD_LOG_DEBUG(( "[%s] slot %lu idx %u", __func__, shred->slot, shred->idx )); + // FD_LOG_NOTICE(( "[%s] slot %lu idx %u", __func__, shred->slot, shred->idx )); + + ulong slot = shred->slot; + fd_shred_key_t key = { slot, .idx = shred->idx }; /* Check this shred > SMR. We ignore shreds before the SMR because by it is invariant that we must have a connected, linear chain for the SMR and its ancestors. */ - if( FD_UNLIKELY( shred->slot <= blockstore->smr ) ) { - return FD_BLOCKSTORE_OK; + if( FD_UNLIKELY( slot <= blockstore->shmem->smr ) ) { + return; } - /* Check if we already have this shred */ + /* Test if the blockstore already contains this shred key. */ - fd_buf_shred_t * shred_pool = fd_blockstore_shred_pool( blockstore ); - fd_buf_shred_map_t * shred_map = fd_blockstore_shred_map( blockstore ); - fd_shred_key_t shred_key = { .slot = shred->slot, .idx = shred->idx }; - fd_buf_shred_t * shred_ = fd_buf_shred_map_ele_query( shred_map, &shred_key, NULL, shred_pool ); - if( FD_UNLIKELY( shred_ ) ) { + if( FD_UNLIKELY( fd_blockstore_shred_test( blockstore, slot, shred->idx ) ) ) { - /* FIXME we currently cannot handle equivocating shreds. */ + /* If we receive a shred with the same key (slot and shred idx) but + different payload as one we already have, we'll only keep the + first. Once we receive the full block, we'll use merkle chaining + from the last FEC set to determine whether we have the correct + shred at every index. - if( FD_UNLIKELY( is_eqvoc_fec( &shred_->hdr, shred ) ) ) { - FD_LOG_WARNING(( "equivocating shred detected %lu %u. halting.", shred->slot, shred->idx )); - return FD_BLOCKSTORE_OK; + Later, if the block fails to replay (dead block) or the block + hash doesn't match the one we observe from votes, we'll dump the + entire block and use repair to recover the one a majority (52%) + of the cluster has voted on. */ + + for(;;) { + fd_buf_shred_map_query_t query[1] = { 0 }; + int err = fd_buf_shred_map_query_try( blockstore->shred_map, &key, NULL, query ); + if( FD_UNLIKELY( err == FD_MAP_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] %s. shred: (%lu, %u)", __func__, fd_buf_shred_map_strerror( err ), slot, shred->idx )); + fd_buf_shred_t * buf_shred = fd_buf_shred_map_query_ele( query ); + buf_shred->eqvoc = ( fd_shred_payload_sz( &buf_shred->hdr ) == fd_shred_payload_sz( shred ) && + memcmp( buf_shred, shred, fd_shred_payload_sz( shred ) ) ); + err = fd_buf_shred_map_query_test( query ); + if( FD_LIKELY( err == FD_MAP_SUCCESS) ) break; } + return; + } - /* Short-circuit if we already have the shred. */ + /* Insert the new shred. */ - return FD_BLOCKSTORE_OK; - } + int err; + fd_buf_shred_t * ele = fd_buf_shred_pool_acquire( blockstore->shred_pool, NULL, 1, &err ); + if( FD_UNLIKELY( err == FD_POOL_ERR_EMPTY ) ) FD_LOG_ERR(( "[%s] %s. increase blockstore shred_max.", __func__, fd_buf_shred_pool_strerror( err ) )); + if( FD_UNLIKELY( err == FD_POOL_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] %s.", __func__, fd_buf_shred_pool_strerror( err ) )); - if( FD_UNLIKELY( !fd_buf_shred_pool_free( shred_pool ) ) ) { - FD_LOG_ERR(( "[%s] OOM: failed to buffer shred. blockstore needs to buffer shreds for slots >= SMR for block assembly, so either increase memory or check for issues with publishing new SMRs.", __func__ )); - } - fd_buf_shred_t * ele = fd_buf_shred_pool_ele_acquire( shred_pool ); /* always non-NULL */ - ele->key = shred_key; - ele->hdr = *shred; + ele->key = key; + ele->hdr = *shred; fd_memcpy( &ele->buf, shred, fd_shred_sz( shred ) ); - fd_buf_shred_map_ele_insert( shred_map, ele, shred_pool ); /* always non-NULL */ + err = fd_buf_shred_map_insert( blockstore->shred_map, ele, FD_MAP_FLAG_BLOCKING ); + if( FD_UNLIKELY( err == FD_MAP_ERR_INVAL ) ) FD_LOG_ERR(( "[%s] map error. ele not in pool.", __func__ )); /* Update shred's associated slot meta */ - ulong slot = shred->slot; fd_block_map_t * block_map = fd_blockstore_block_map( blockstore ); fd_block_map_t * block_map_entry = fd_block_map_query( block_map, &slot, NULL ); if( FD_UNLIKELY( !block_map_entry ) ) { @@ -1219,7 +1190,7 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr /* Try to insert slot into block_map */ block_map_entry = fd_block_map_insert( block_map, &slot ); - if( FD_UNLIKELY( !block_map_entry ) ) return FD_BLOCKSTORE_ERR_SLOT_FULL; + if( FD_UNLIKELY( !block_map_entry ) ) return; /* Initialize the block_map_entry. Note some fields are initialized to dummy values because we do not have all the necessary metadata @@ -1227,7 +1198,7 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr block_map_entry->slot = block_map_entry->slot; - block_map_entry->parent_slot = shred->slot - shred->data.parent_off; + block_map_entry->parent_slot = slot - shred->data.parent_off; memset( block_map_entry->child_slots, UCHAR_MAX, FD_BLOCKSTORE_CHILD_SLOT_MAX * sizeof(ulong) ); block_map_entry->child_slot_cnt = 0; @@ -1238,9 +1209,9 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr block_map_entry->ts = 0; block_map_entry->reference_tick = (uchar)( (int)shred->data.flags & (int)FD_SHRED_DATA_REF_TICK_MASK ); - block_map_entry->consumed_idx = UINT_MAX; + block_map_entry->buffered_idx = UINT_MAX; block_map_entry->received_idx = 0; - block_map_entry->replayed_idx = UINT_MAX; + block_map_entry->consumed_idx = UINT_MAX; block_map_entry->data_complete_idx = UINT_MAX; block_map_entry->slot_complete_idx = UINT_MAX; @@ -1254,17 +1225,17 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr } FD_LOG_DEBUG(( "shred: (%lu, %u). consumed: %u, received: %u, complete: %u", - shred->slot, + slot, shred->idx, - block_map_entry->consumed_idx, + block_map_entry->buffered_idx, block_map_entry->received_idx, block_map_entry->slot_complete_idx )); - /* Advance the consumed_idx watermark. */ + /* Advance the buffered_idx watermark. */ - uint prev_consumed_idx = block_map_entry->consumed_idx; - while( FD_LIKELY( fd_buf_shred_query( blockstore, shred->slot, (uint)( block_map_entry->consumed_idx + 1U ) ) ) ) { - block_map_entry->consumed_idx++; + uint prev_buffered_idx = block_map_entry->buffered_idx; + while( FD_LIKELY( fd_blockstore_shred_test( blockstore, slot, block_map_entry->buffered_idx + 1 ) ) ) { + block_map_entry->buffered_idx++; } /* Mark the ending shred idxs of entry batches. */ @@ -1274,8 +1245,8 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr /* Advance the data_complete_idx watermark using the shreds in between the previous consumed_idx and current consumed_idx. */ - for (uint idx = prev_consumed_idx + 1; block_map_entry->consumed_idx != FD_SHRED_IDX_NULL && idx <= block_map_entry->consumed_idx; idx++) { - if ( FD_UNLIKELY( fd_block_set_test( block_map_entry->data_complete_idxs, idx ) ) ) { + for (uint idx = prev_buffered_idx + 1; block_map_entry->buffered_idx != FD_SHRED_IDX_NULL && idx <= block_map_entry->buffered_idx; idx++) { + if( FD_UNLIKELY( fd_block_set_test( block_map_entry->data_complete_idxs, idx ) ) ) { block_map_entry->data_complete_idx = idx; } } @@ -1283,14 +1254,14 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr /* Update received_idx and slot_complete_idx. */ block_map_entry->received_idx = fd_uint_max( block_map_entry->received_idx, shred->idx + 1 ); - if( FD_UNLIKELY( shred->data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE ) ) block_map_entry->slot_complete_idx = shred->idx; + if( FD_UNLIKELY( shred->data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE ) ) { + // FD_LOG_NOTICE(( "slot %lu %u complete", slot, shred->idx )); + block_map_entry->slot_complete_idx = shred->idx; + } /* Update ancestry metadata: parent_slot, is_connected, next_slot. */ fd_block_map_t * parent_block_map_entry = fd_blockstore_block_map_query( blockstore, block_map_entry->parent_slot ); - - /* Add this slot to its parent's child slots if not already there. */ - if( FD_LIKELY( parent_block_map_entry ) ) { int found = 0; for( ulong i = 0; i < parent_block_map_entry->child_slot_cnt; i++ ) { @@ -1298,8 +1269,8 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr found = 1; } } - if( FD_UNLIKELY( !found ) ) { - if( FD_UNLIKELY( parent_block_map_entry->child_slot_cnt == FD_BLOCKSTORE_CHILD_SLOT_MAX )) { + if( FD_UNLIKELY( !found ) ) { /* add to parent's child slots if not already there */ + if( FD_UNLIKELY( parent_block_map_entry->child_slot_cnt == FD_BLOCKSTORE_CHILD_SLOT_MAX ) ) { FD_LOG_ERR(( "failed to add slot %lu to parent %lu's children. exceeding child slot max", slot, parent_block_map_entry->slot )); @@ -1308,79 +1279,44 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr } } - if( FD_LIKELY( block_map_entry->consumed_idx == UINT_MAX || - block_map_entry->consumed_idx != block_map_entry->slot_complete_idx ) ) { - return FD_BLOCKSTORE_OK; - } - /* Received all shreds, so try to assemble a block. */ - FD_LOG_DEBUG(( "received all shreds for slot %lu - now building a block", shred->slot )); - - int rc = deshred( blockstore, shred->slot ); - switch( rc ) { - case FD_BLOCKSTORE_OK: - return FD_BLOCKSTORE_OK_SLOT_COMPLETE; - case FD_BLOCKSTORE_ERR_SLOT_FULL: - FD_LOG_DEBUG(( "already deshredded slot %lu. ignoring.", shred->slot )); - return FD_BLOCKSTORE_OK; - case FD_BLOCKSTORE_ERR_DESHRED_INVALID: - FD_LOG_DEBUG(( "failed to deshred slot %lu. ignoring.", shred->slot )); - return FD_BLOCKSTORE_OK; - default: - /* FIXME */ - FD_LOG_ERR(( "deshred err %d", rc )); - } -} -fd_shred_t * -fd_buf_shred_query( fd_blockstore_t * blockstore, ulong slot, uint shred_idx ) { - fd_buf_shred_t * shred_pool = fd_blockstore_shred_pool( blockstore ); - fd_buf_shred_map_t * shred_map = fd_blockstore_shred_map( blockstore ); - fd_shred_key_t key = { .slot = slot, .idx = shred_idx }; - fd_buf_shred_t * query = - fd_buf_shred_map_ele_query( shred_map, &key, NULL, shred_pool ); - if( FD_UNLIKELY( !query ) ) return NULL; - return &query->hdr; + if( FD_UNLIKELY( block_map_entry->slot_complete_idx != FD_SHRED_IDX_NULL && + block_map_entry->buffered_idx == block_map_entry->slot_complete_idx ) ) { + deshred( blockstore, shred->slot ); + } } -long -fd_buf_shred_query_copy_data( fd_blockstore_t * blockstore, ulong slot, uint shred_idx, void * buf, ulong buf_max ) { - if( buf_max < FD_SHRED_MAX_SZ ) return -1; +int +fd_blockstore_shred_test( fd_blockstore_t * blockstore, ulong slot, uint idx ) { + fd_shred_key_t key = { slot, idx }; + fd_buf_shred_map_query_t query[1] = { 0 }; - fd_buf_shred_t * shred_pool = fd_blockstore_shred_pool( blockstore ); - fd_buf_shred_map_t * shred_map = fd_blockstore_shred_map( blockstore ); - fd_shred_key_t key = { .slot = slot, .idx = shred_idx }; - fd_buf_shred_t * shred = - fd_buf_shred_map_ele_query( shred_map, &key, NULL, shred_pool ); - if( shred ) { - ulong sz = fd_shred_sz( &shred->hdr ); - if( sz > buf_max ) return -1; - fd_memcpy( buf, shred->buf, sz); - return (long)sz; + for(;;) { + int err = fd_buf_shred_map_query_try( blockstore->shred_map, &key, NULL, query ); + if( FD_UNLIKELY( err == FD_MAP_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] slot: %lu idx: %u. %s", __func__, slot, idx, fd_buf_shred_map_strerror( err ) )); + if( FD_LIKELY( !fd_buf_shred_map_query_test( query ) ) ) return err != FD_MAP_ERR_KEY; } +} - fd_block_map_t * query = - fd_block_map_query( fd_blockstore_block_map( blockstore ), &slot, NULL ); - if( FD_UNLIKELY( !query || query->block_gaddr == 0 ) ) return -1; - if( shred_idx > query->slot_complete_idx ) return -1; - fd_wksp_t * wksp = fd_blockstore_wksp( blockstore ); - fd_block_t * blk = fd_wksp_laddr_fast( wksp, query->block_gaddr ); - fd_block_shred_t * shreds = fd_wksp_laddr_fast( wksp, blk->shreds_gaddr ); - ulong sz = fd_shred_payload_sz( &shreds[shred_idx].hdr ); - if( FD_SHRED_DATA_HEADER_SZ + sz > buf_max ) return -1L; - fd_memcpy( buf, &shreds[shred_idx].hdr, FD_SHRED_DATA_HEADER_SZ ); - fd_memcpy( (uchar*)buf + FD_SHRED_DATA_HEADER_SZ, (uchar*)fd_wksp_laddr_fast( wksp, blk->data_gaddr ) + shreds[shred_idx].off, sz ); - ulong tot_sz = FD_SHRED_DATA_HEADER_SZ + sz; - ulong merkle_sz = shreds[shred_idx].merkle_sz; - if( merkle_sz ) { - if( tot_sz + merkle_sz > buf_max ) return -1; - fd_memcpy( (uchar*)buf + tot_sz, shreds[shred_idx].merkle, merkle_sz ); - tot_sz += merkle_sz; - } - if( tot_sz >= FD_SHRED_MIN_SZ ) return (long)tot_sz; - /* Zero pad */ - fd_memset( (uchar*)buf + tot_sz, 0, FD_SHRED_MIN_SZ - tot_sz ); - return (long)FD_SHRED_MIN_SZ; +long +fd_buf_shred_query_copy_data( fd_blockstore_t * blockstore, ulong slot, uint idx, void * buf, ulong buf_sz ) { + if( buf_sz < FD_SHRED_MAX_SZ ) return -1; + fd_shred_key_t key = { slot, idx }; + ulong sz = 0; + int err = FD_MAP_ERR_AGAIN; + while( err == FD_MAP_ERR_AGAIN ) { + fd_buf_shred_map_query_t query[1] = { 0 }; + err = fd_buf_shred_map_query_try( blockstore->shred_map, &key, NULL, query ); + fd_buf_shred_t const * shred = fd_buf_shred_map_query_ele_const( query ); + sz = fd_shred_sz( &shred->hdr ); + memcpy( buf, shred->buf, sz ); + err = fd_buf_shred_map_query_test( query ); + if( FD_UNLIKELY( err == FD_MAP_ERR_KEY ) ) return -1; + if( FD_UNLIKELY( err == FD_MAP_ERR_CORRUPT ) ) FD_LOG_ERR(( "[%s] map corrupt. shred %lu %u", __func__, slot, idx )); + } + FD_TEST( !err ); + return (long)sz; } fd_block_t * @@ -1424,92 +1360,92 @@ fd_blockstore_child_slots_query( fd_blockstore_t * blockstore, ulong slot, ulong if( FD_UNLIKELY( !query ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; *slots_out = query->child_slots; *slot_cnt_out = query->child_slot_cnt; - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } int -fd_blockstore_batch_assemble( fd_blockstore_t * blockstore, - ulong slot, - uint batch_idx, - ulong batch_data_max, - uchar * batch_data_out, - ulong * batch_data_sz ) { - fd_buf_shred_t * shred_pool = fd_blockstore_shred_pool( blockstore ); - fd_buf_shred_map_t * shred_map = fd_blockstore_shred_map( blockstore ); - +fd_blockstore_slice_query( fd_blockstore_t * blockstore, + ulong slot, + uint idx, + ulong max, + uchar * buf, + ulong * buf_sz ) { fd_block_map_t * query = fd_blockstore_block_map_query( blockstore, slot ); if( FD_UNLIKELY( !query ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; - if( batch_idx > 0 ) { /* verify that the batch_idx provided is actually the start of a batch */ + if( idx > 0 ) { /* verify that the batch_idx provided is actually the start of a batch */ fd_block_set_t * data_complete_idxs = query->data_complete_idxs; - if ( !fd_block_set_test( data_complete_idxs, batch_idx - 1 ) || batch_idx > query->slot_complete_idx ) { + if ( !fd_block_set_test( data_complete_idxs, idx - 1 ) || idx > query->slot_complete_idx ) { return FD_BLOCKSTORE_ERR_SHRED_INVALID; } } - ulong mbatch_sz = 0; - for (uint idx = batch_idx; ; idx++) { - fd_shred_key_t key = { slot, idx }; - - fd_buf_shred_t * shred = fd_buf_shred_map_ele_query( shred_map, &key, NULL, shred_pool ); - uchar const * payload = NULL; - ulong payload_sz = 0; - bool is_batch_end = false; - if( FD_UNLIKELY( shred ) ) { /* FIXME change to likely */ - payload = fd_shred_data_payload( &shred->hdr ); - payload_sz = fd_shred_payload_sz( &shred->hdr ); - is_batch_end = (shred->hdr.data.flags & FD_SHRED_DATA_FLAG_DATA_COMPLETE) || (shred->hdr.data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE); - } else { /* FIXME remove after blockstore refactor */ - fd_block_t * block = fd_blockstore_block_query( blockstore, slot ); - if( FD_UNLIKELY( !block ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; - - fd_wksp_t * wksp = fd_blockstore_wksp( blockstore ); - fd_block_shred_t * shreds = fd_wksp_laddr_fast( wksp, block->shreds_gaddr ); - uchar * data = fd_wksp_laddr_fast( wksp, block->data_gaddr ); - - payload = data + shreds[idx].off; - payload_sz = ( idx + 1 != block->shreds_cnt ) ? ( shreds[idx + 1].off - shreds[idx].off ) - : ( block->data_sz - shreds[idx].off ); - is_batch_end = (shreds[idx].hdr.data.flags & FD_SHRED_DATA_FLAG_DATA_COMPLETE) - || (shreds[idx].hdr.data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE); - } + ulong off = 0; + for(;;) { + ulong payload_sz = 0; + int complete = 0; + + for(;;) { /* speculative copy */ + fd_shred_key_t key = { slot, idx }; + fd_buf_shred_map_query_t query[1] = { 0 }; + int err = fd_buf_shred_map_query_try( blockstore->shred_map, &key, NULL, query ); + if( FD_UNLIKELY( err == FD_MAP_ERR_CORRUPT ) ) { + FD_LOG_WARNING(( "[%s] key: (%lu, %u) %s", __func__, slot, idx, fd_buf_shred_map_strerror( err ) )); + return FD_BLOCKSTORE_ERR_CORRUPT; + } + if( FD_UNLIKELY( err == FD_MAP_ERR_KEY ) ) { + FD_LOG_WARNING(( "[%s] key: (%lu, %u) %s", __func__, slot, idx, fd_buf_shred_map_strerror( err ) )); + return FD_BLOCKSTORE_ERR_KEY; + } - if( FD_UNLIKELY( payload_sz > FD_SHRED_DATA_PAYLOAD_MAX ) ) return FD_BLOCKSTORE_ERR_SHRED_INVALID; - if( FD_UNLIKELY( mbatch_sz + payload_sz > batch_data_max ) ) return FD_BLOCKSTORE_ERR_NO_MEM; - fd_memcpy( batch_data_out + mbatch_sz, payload, payload_sz ); + fd_buf_shred_t const * shred = fd_buf_shred_map_query_ele_const( query ); + uchar const * payload = fd_shred_data_payload( &shred->hdr ); + payload_sz = fd_shred_payload_sz( &shred->hdr ); + if( FD_UNLIKELY( off + payload_sz > max ) ) { + FD_LOG_WARNING(( "[%s] increase `max`", __func__ )); /* caller needs to increase max */ + return FD_BLOCKSTORE_ERR_INVAL; + } - mbatch_sz += payload_sz; - if( FD_UNLIKELY( is_batch_end ) ){ - /* likely has trailing 0s */ - break; - } - } - *batch_data_sz = mbatch_sz; - return FD_BLOCKSTORE_OK; + if( FD_UNLIKELY( payload_sz > FD_SHRED_DATA_PAYLOAD_MAX ) ) return FD_BLOCKSTORE_ERR_SHRED_INVALID; + if( FD_UNLIKELY( off + payload_sz > max ) ) return FD_BLOCKSTORE_ERR_NO_MEM; + fd_memcpy( buf + off, payload, payload_sz ); + complete = ( shred->hdr.data.flags & FD_SHRED_DATA_FLAG_DATA_COMPLETE ) || + ( shred->hdr.data.flags & FD_SHRED_DATA_FLAG_SLOT_COMPLETE ); + err = fd_buf_shred_map_query_test( query ); + if( FD_LIKELY( err == FD_MAP_SUCCESS ) ) break; + }; /* successful speculative copy */ + + off += payload_sz; + if( FD_UNLIKELY( complete ) ) break; + idx++; + } + *buf_sz = off; + return FD_BLOCKSTORE_SUCCESS; } -bool +int fd_blockstore_shreds_complete( fd_blockstore_t * blockstore, ulong slot ){ - bool block_exists = fd_blockstore_block_query( blockstore, slot ); + fd_block_t * block_exists = fd_blockstore_block_query( blockstore, slot ); fd_block_map_t * query = fd_blockstore_block_map_query( blockstore, slot ); - if( FD_UNLIKELY( !query || query->consumed_idx == FD_SHRED_IDX_NULL ) ) { + if( FD_UNLIKELY( !query || query->buffered_idx == FD_SHRED_IDX_NULL ) ) { FD_TEST( !block_exists ); /* FIXME remove after blockstore refactor */ return false; } + /* When replacing block_query( slot ) != NULL with this function: - There are other things verified in a successful deshred & scan block that are not verified here. - scan_block does a round of well-formedness checks like parsing txns, and no premature end of batch + There are other things verified in a successful deshred & scan block that are not verified here. + scan_block does a round of well-formedness checks like parsing txns, and no premature end of batch like needing cnt, microblock, microblock format. - This maybe should be fine in places where we check both + This maybe should be fine in places where we check both shreds_complete and flag PROCESSED/REPLAYING is set, because validation has been for sure done - if the block has been replayed - + if the block has been replayed + Should be careful in places that call this now that happen before the block is replayed, if we want to assume the shreds are well-formed we can't. */ - return query->slot_complete_idx == query->consumed_idx; + return query->slot_complete_idx != UINT_MAX && query->slot_complete_idx == query->buffered_idx; } @@ -1538,22 +1474,22 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, ulong off = ULONG_MAX; for(;;) { uint seqnum; - if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->lock, &seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->shmem->lock, &seqnum ) ) ) continue; idx_entry = fd_block_idx_query( block_idx, slot, NULL ); if( FD_LIKELY( idx_entry ) ) off = idx_entry->off; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; else break; } if ( FD_UNLIKELY( off < ULONG_MAX ) ) { /* optimize for non-archival queries */ FD_LOG_DEBUG( ( "Querying archive for block %lu", slot ) ); fd_block_t block_out; - int err = fd_blockstore_block_meta_restore( &blockstore->archiver, fd, idx_entry, block_map_entry_out, &block_out ); + int err = fd_blockstore_block_meta_restore( &blockstore->shmem->archiver, fd, idx_entry, block_map_entry_out, &block_out ); if( FD_UNLIKELY( err ) ) { return FD_BLOCKSTORE_ERR_SLOT_MISSING; } uchar * block_data = fd_valloc_malloc( alloc, 128UL, block_out.data_sz ); - err = fd_blockstore_block_data_restore( &blockstore->archiver, + err = fd_blockstore_block_data_restore( &blockstore->shmem->archiver, fd, idx_entry, block_data, @@ -1571,7 +1507,7 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, *block_rewards_out = block_out.rewards; *block_data_out = block_data; *block_data_sz_out = block_out.data_sz; - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } fd_block_map_t const * block_map = fd_blockstore_block_map( blockstore ); @@ -1579,7 +1515,7 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, ulong prev_sz = 0; for(;;) { uint seqnum; - if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->lock, &seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->shmem->lock, &seqnum ) ) ) continue; fd_block_map_t const * query = fd_block_map_query_safe( block_map, &slot, NULL ); if( FD_UNLIKELY( !query ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; @@ -1588,7 +1524,7 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, ulong blk_gaddr = query->block_gaddr; if( FD_UNLIKELY( !blk_gaddr ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; fd_block_t * blk = fd_wksp_laddr_fast( wksp, blk_gaddr ); if( block_rewards_out ) memcpy( block_rewards_out, &blk->rewards, sizeof(fd_block_rewards_t) ); @@ -1597,7 +1533,7 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, ulong sz = *block_data_sz_out = blk->data_sz; if( sz >= FD_SHRED_MAX_PER_SLOT * FD_SHRED_MAX_SZ ) continue; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; uchar * data_out; if( prev_sz >= sz ) { @@ -1615,12 +1551,7 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, ulong batch_sz = 0; ulong total_blk_sz = 0; while( batch_idx <= query->slot_complete_idx ){ - int err = fd_blockstore_batch_assemble( blockstore, - slot, - (uint)batch_idx, - sz - total_blk_sz, - data_out + total_blk_sz, - &batch_sz ); + int err = fd_blockstore_slice_query( blockstore, slot, (uint)batch_idx, sz - total_blk_sz, data_out + total_blk_sz, &batch_sz ); if( FD_UNLIKELY( err ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; total_blk_sz += batch_sz; @@ -1631,7 +1562,7 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, } } - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) { + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) { fd_valloc_free( alloc, data_out ); continue; } @@ -1645,14 +1576,14 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, } else { fd_memcpy( parent_block_hash_out, query->block_hash.uc, sizeof(fd_hash_t) ); - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) { + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) { fd_valloc_free( alloc, data_out ); continue; } } } - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } } @@ -1675,10 +1606,10 @@ fd_blockstore_block_map_query_volatile( fd_blockstore_t * blockstore, ulong off = ULONG_MAX; for( ;; ) { uint seqnum; - if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->lock, &seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->shmem->lock, &seqnum ) ) ) continue; fd_block_idx_t * idx_entry = fd_block_idx_query( block_idx, slot, NULL ); if( FD_LIKELY( idx_entry ) ) off = idx_entry->off; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; else break; } @@ -1693,22 +1624,22 @@ fd_blockstore_block_map_query_volatile( fd_blockstore_t * blockstore, FD_LOG_WARNING(( "failed to read block map entry" )); return FD_BLOCKSTORE_ERR_SLOT_MISSING; } - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } fd_block_map_t const * block_map = fd_blockstore_block_map( blockstore ); for(;;) { uint seqnum; - if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->lock, &seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->shmem->lock, &seqnum ) ) ) continue; fd_block_map_t const * query = fd_block_map_query_safe( block_map, &slot, NULL ); if( FD_UNLIKELY( !query ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; memcpy( block_map_entry_out, query, sizeof( fd_block_map_t ) ); ulong blk_gaddr = query->block_gaddr; if( FD_UNLIKELY( !blk_gaddr ) ) return FD_BLOCKSTORE_ERR_SLOT_MISSING; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } } @@ -1740,14 +1671,14 @@ fd_blockstore_txn_query_volatile( fd_blockstore_t * blockstore, for(;;) { uint seqnum; - if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->lock, &seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->shmem->lock, &seqnum ) ) ) continue; fd_txn_key_t key; memcpy( &key, sig, sizeof(key) ); fd_txn_map_t const * txn_map_entry = fd_txn_map_query_safe( txn_map, &key, NULL ); if( FD_UNLIKELY( txn_map_entry == NULL ) ) return FD_BLOCKSTORE_ERR_TXN_MISSING; memcpy( txn_out, txn_map_entry, sizeof(fd_txn_map_t) ); - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; else break; } @@ -1756,10 +1687,10 @@ fd_blockstore_txn_query_volatile( fd_blockstore_t * blockstore, ulong off = ULONG_MAX; for(;;) { uint seqnum; - if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->lock, &seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->shmem->lock, &seqnum ) ) ) continue; fd_block_idx_t * idx_entry = fd_block_idx_query( block_idx, txn_out->slot, NULL ); if( FD_LIKELY( idx_entry ) ) off = idx_entry->off; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; else break; } @@ -1778,19 +1709,19 @@ fd_blockstore_txn_query_volatile( fd_blockstore_t * blockstore, check_read_write_err( err ); err = fd_io_read( fd, txn_data_out, txn_out->sz, txn_out->sz, &rsz ); check_read_write_err( err); - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } for(;;) { uint seqnum; - if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->lock, &seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_start_concur_read( &blockstore->shmem->lock, &seqnum ) ) ) continue; fd_block_map_t const * query = fd_block_map_query_safe( block_map, &txn_out->slot, NULL ); if( FD_UNLIKELY( !query ) ) return FD_BLOCKSTORE_ERR_TXN_MISSING; ulong blk_gaddr = query->block_gaddr; if( FD_UNLIKELY( !blk_gaddr ) ) return FD_BLOCKSTORE_ERR_TXN_MISSING; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; fd_block_t * blk = fd_wksp_laddr_fast( wksp, blk_gaddr ); if( blk_ts ) *blk_ts = query->ts; @@ -1799,15 +1730,15 @@ fd_blockstore_txn_query_volatile( fd_blockstore_t * blockstore, ulong sz = blk->data_sz; if( txn_out->offset + txn_out->sz > sz || txn_out->sz > FD_TXN_MTU ) continue; - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; - if( txn_data_out == NULL ) return FD_BLOCKSTORE_OK; + if( txn_data_out == NULL ) return FD_BLOCKSTORE_SUCCESS; uchar const * data = fd_wksp_laddr_fast( wksp, ptr ); fd_memcpy( txn_data_out, data + txn_out->offset, txn_out->sz ); - if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->lock, seqnum ) ) ) continue; + if( FD_UNLIKELY( fd_rwseq_check_concur_read( &blockstore->shmem->lock, seqnum ) ) ) continue; - return FD_BLOCKSTORE_OK; + return FD_BLOCKSTORE_SUCCESS; } } @@ -1827,7 +1758,7 @@ fd_blockstore_log_block_status( fd_blockstore_t * blockstore, ulong around_slot ( i == around_slot ? "*" : " " ), i, slot_entry->received_idx, - slot_entry->consumed_idx, + slot_entry->buffered_idx, slot_entry->slot_complete_idx )); } } @@ -1853,20 +1784,15 @@ fd_blockstore_log_mem_usage( fd_blockstore_t * blockstore ) { FD_LOG_NOTICE(( "blockstore base footprint: %s", fd_smart_size( sizeof(fd_blockstore_t), tmp1, sizeof(tmp1) ) )); - fd_buf_shred_t * shred_pool = fd_blockstore_shred_pool( blockstore ); - ulong shred_used = fd_buf_shred_pool_used( shred_pool ); - ulong shred_max = fd_buf_shred_pool_max( shred_pool ); - FD_LOG_NOTICE(( "shred pool footprint: %s (%lu entries used out of %lu, %lu%%)", - fd_smart_size( fd_buf_shred_pool_footprint( shred_max ), tmp1, sizeof(tmp1) ), - shred_used, - shred_max, - (100U*shred_used) / shred_max )); - fd_buf_shred_map_t * shred_map = fd_blockstore_shred_map( blockstore ); - ulong shred_map_cnt = fd_buf_shred_map_chain_cnt( shred_map ); + ulong shred_max = fd_buf_shred_pool_ele_max( blockstore->shred_pool ); + FD_LOG_NOTICE(( "shred pool footprint: %s %lu entries)", + fd_smart_size( fd_buf_shred_pool_footprint(), tmp1, sizeof(tmp1) ), + shred_max )); + ulong shred_map_cnt = fd_buf_shred_map_chain_cnt( blockstore->shred_map ); FD_LOG_NOTICE(( "shred map footprint: %s (%lu chains, load is %.3f)", fd_smart_size( fd_buf_shred_map_footprint( shred_map_cnt ), tmp1, sizeof(tmp1) ), shred_map_cnt, - ((double)shred_used)/((double)shred_map_cnt) )); + (double)shred_map_cnt) ); fd_block_map_t * slot_map = fd_blockstore_block_map( blockstore ); ulong slot_map_cnt = fd_block_map_key_cnt( slot_map ); ulong slot_map_max = fd_block_map_key_max( slot_map ); @@ -1891,7 +1817,7 @@ fd_blockstore_log_mem_usage( fd_blockstore_t * blockstore ) { ulong * q = fd_blockstore_slot_deque( blockstore ); fd_slot_deque_remove_all( q ); - fd_slot_deque_push_tail( q, blockstore->smr ); + fd_slot_deque_push_tail( q, blockstore->shmem->smr ); while( !fd_slot_deque_empty( q ) ) { ulong curr = fd_slot_deque_pop_head( q ); @@ -1909,7 +1835,7 @@ fd_blockstore_log_mem_usage( fd_blockstore_t * blockstore ) { ulong * child_slots = NULL; ulong child_slot_cnt = 0; int rc = fd_blockstore_child_slots_query( blockstore, curr, &child_slots, &child_slot_cnt ); - if( FD_UNLIKELY( rc != FD_BLOCKSTORE_OK ) ) { + if( FD_UNLIKELY( rc != FD_BLOCKSTORE_SUCCESS ) ) { continue; } diff --git a/src/flamenco/runtime/fd_blockstore.h b/src/flamenco/runtime/fd_blockstore.h index f8e9982881..49d042f31c 100644 --- a/src/flamenco/runtime/fd_blockstore.h +++ b/src/flamenco/runtime/fd_blockstore.h @@ -1,8 +1,8 @@ #ifndef HEADER_fd_src_flamenco_runtime_fd_blockstore_h #define HEADER_fd_src_flamenco_runtime_fd_blockstore_h -/* Blockstore is a high-performance database for storing, building, and - tracking blocks. +/* Blockstore is a high-performance database for in-memory indexing and + durably storing blocks. `fd_blockstore` defines a number of useful types e.g. `fd_block_t`, `fd_block_shred`, etc. @@ -21,15 +21,16 @@ #include "stdbool.h" #include -/* FD_BLOCKSTORE_{ALIGN,FOOTPRINT} describe the alignment and footprint needed - for a blockstore. ALIGN should be a positive integer power of 2. - FOOTPRINT is multiple of ALIGN. These are provided to facilitate - compile time declarations. */ +/* FD_BLOCKSTORE_ALIGN specifies the alignment needed for blockstore. + ALIGN is double x86 cache line to mitigate various kinds of false + sharing (eg. ACLPF adjacent cache line prefetch). */ -/* clang-format off */ -#define FD_BLOCKSTORE_ALIGN (128UL) -#define FD_BLOCKSTORE_FOOTPRINT (256UL) -#define FD_BLOCKSTORE_MAGIC (0xf17eda2ce7b10c00UL) /* firedancer bloc version 0 */ +#define FD_BLOCKSTORE_ALIGN (128UL) + +/* FD_BLOCKSTORE_MAGIC defines a magic number for verifying the memory + of blockstore is not corrupted. */ + +#define FD_BLOCKSTORE_MAGIC (0xf17eda2ce7b10c00UL) /* firedancer bloc version 0 */ /* DO NOT MODIFY. */ // #define FD_BUF_SHRED_MAP_MAX (1UL << 24UL) /* 16 million shreds can be buffered */ @@ -41,7 +42,7 @@ #define FD_BLOCKSTORE_ARCHIVE_MIN_SIZE (1UL << 26UL) /* 64MB := ceil(MAX_DATA_SHREDS_PER_SLOT*1228) */ /* Maximum size of an entry batch is the entire block */ -#define FD_MBATCH_MAX (FD_SHRED_DATA_PAYLOAD_MAX_PER_SLOT) +#define FD_SLICE_MAX (FD_SHRED_DATA_PAYLOAD_MAX_PER_SLOT) /* 64 ticks per slot, and then one min size transaction per microblock for all the remaining microblocks. This bound should be used along with the transaction parser and tick @@ -61,8 +62,14 @@ // https://github.com/firedancer-io/solana/blob/v1.17.5/core/src/repair/repair_service.rs#L55 #define FD_REPAIR_TIMEOUT (200 / FD_MS_PER_TICK) -#define FD_BLOCKSTORE_OK 0 -#define FD_BLOCKSTORE_OK_SLOT_COMPLETE 1 +#define FD_BLOCKSTORE_SUCCESS 0 +#define FD_BLOCKSTORE_SUCCESS_SLOT_COMPLETE 1 +#define FD_BLOCKSTORE_ERR_INVAL (-1) +#define FD_BLOCKSTORE_ERR_AGAIN (-2) +#define FD_BLOCKSTORE_ERR_CORRUPT (-3) +#define FD_BLOCKSTORE_ERR_EMPTY (-4) +#define FD_BLOCKSTORE_ERR_FULL (-5) +#define FD_BLOCKSTORE_ERR_KEY (-6) #define FD_BLOCKSTORE_ERR_SHRED_FULL -1 /* no space left for shreds */ #define FD_BLOCKSTORE_ERR_SLOT_FULL -2 /* no space left for slots */ #define FD_BLOCKSTORE_ERR_TXN_FULL -3 /* no space left for txns */ @@ -73,7 +80,20 @@ #define FD_BLOCKSTORE_ERR_DESHRED_INVALID -8 /* deshredded block was invalid */ #define FD_BLOCKSTORE_ERR_NO_MEM -9 /* no mem */ #define FD_BLOCKSTORE_ERR_UNKNOWN -99 -/* clang-format on */ + +static inline char const * fd_blockstore_strerror( int err ) { + switch( err ) { + case FD_BLOCKSTORE_SUCCESS: return "success"; + case FD_BLOCKSTORE_ERR_INVAL: return "bad input"; + case FD_BLOCKSTORE_ERR_AGAIN: return "try again"; + case FD_BLOCKSTORE_ERR_CORRUPT: return "corruption detected"; + case FD_BLOCKSTORE_ERR_EMPTY: return "empty"; + case FD_BLOCKSTORE_ERR_FULL: return "full"; + case FD_BLOCKSTORE_ERR_KEY: return "key not found"; + default: break; + } + return "unknown"; +} struct fd_shred_key { ulong slot; @@ -81,13 +101,13 @@ struct fd_shred_key { }; typedef struct fd_shred_key fd_shred_key_t; -/* clang-format off */ static const fd_shred_key_t fd_shred_key_null = { 0 }; #define FD_SHRED_KEY_NULL fd_shred_key_null #define FD_SHRED_KEY_INVAL(key) (!((key).slot) & !((key).idx)) #define FD_SHRED_KEY_EQ(k0,k1) (!(((k0).slot) ^ ((k1).slot))) & !(((k0).idx) ^ (((k1).idx))) #define FD_SHRED_KEY_HASH(key) ((uint)(((key).slot)<<15UL) | (((key).idx))) /* current max shred idx is 32KB = 2 << 15*/ -/* clang-format on */ + + /* fd_buf_shred is a thin wrapper around fd_shred_t that facilitates buffering data shreds before all the shreds for a slot have been @@ -111,28 +131,31 @@ static const fd_shred_key_t fd_shred_key_null = { 0 }; | shred hdr | shred payload | */ -struct fd_buf_shred { +struct __attribute__((aligned(128UL))) fd_buf_shred { fd_shred_key_t key; + ulong prev; ulong next; + ulong memo; + int eqvoc; /* we've seen an equivocating version of this + shred (same key but different payload). */ union { fd_shred_t hdr; /* shred header */ - uchar buf[FD_SHRED_MAX_SZ]; /* the entire shred buffer, both header and payload. */ + uchar buf[FD_SHRED_MIN_SZ]; /* the entire shred buffer, both header and payload. */ }; }; typedef struct fd_buf_shred fd_buf_shred_t; -#define POOL_NAME fd_buf_shred_pool -#define POOL_T fd_buf_shred_t -#include "../../util/tmpl/fd_pool.c" +#define POOL_NAME fd_buf_shred_pool +#define POOL_ELE_T fd_buf_shred_t +#include "../../util/tmpl/fd_pool_para.c" -/* clang-format off */ #define MAP_NAME fd_buf_shred_map #define MAP_ELE_T fd_buf_shred_t #define MAP_KEY_T fd_shred_key_t #define MAP_KEY_EQ(k0,k1) (FD_SHRED_KEY_EQ(*k0,*k1)) +#define MAP_KEY_EQ_IS_SLOW 1 #define MAP_KEY_HASH(key,seed) (FD_SHRED_KEY_HASH(*key)^seed) -#include "../../util/tmpl/fd_map_chain.c" -/* clang-format on */ +#include "../../util/tmpl/fd_map_para.c" #define DEQUE_NAME fd_slot_deque #define DEQUE_T ulong @@ -143,8 +166,6 @@ typedef struct fd_buf_shred fd_buf_shred_t; region. */ struct fd_block_shred { fd_shred_t hdr; /* ptr to the data shred header */ - uchar merkle[FD_SHRED_MERKLE_ROOT_SZ + FD_SHRED_MERKLE_NODE_SZ*9U /* FD_FEC_SET_MAX_BMTREE_DEPTH */]; - ulong merkle_sz; ulong off; /* offset to the payload relative to the start of the block's data region */ }; typedef struct fd_block_shred fd_block_shred_t; @@ -280,11 +301,27 @@ struct fd_block_map { uchar reference_tick; /* the tick when the leader prepared the block. */ long ts; /* the wallclock time when we finished receiving the block. */ - /* Windowing */ + /* Windowing - uint consumed_idx; /* the highest shred idx we've contiguously received from idx 0 (inclusive). */ - uint received_idx; /* the highest shred idx we've received + 1 (exclusive). */ - uint replayed_idx; /* the highest shred idx we've replayed (inclusive). */ + Shreds are buffered into a map as they are received: + + | 0 | 1 | 2 | x | x | 5 | x | + ^ ^ ^ + c b r + + c = "consumed" = contiguous shred idxs that have been consumed. + the "consumer" is replay and the idx is + incremented after replaying each block slice. + b = "buffered" = contiguous shred idxs that have been buffered. + when buffered == block_slice_end the next slice of + a block is ready for replay. + r = "received" = highest shred idx received so far. used to detect + when repair is needed. + */ + + uint consumed_idx; /* the highest shred idx we've contiguously consumed (consecutive from 0). */ + uint buffered_idx; /* the highest shred idx we've contiguously buffered (consecutive from 0). */ + uint received_idx; /* the highest shred idx we've received (can be out-of-order). */ uint data_complete_idx; /* the highest shred idx wrt contiguous entry batches (inclusive). */ uint slot_complete_idx; /* the highest shred idx for the entire slot (inclusive). */ @@ -308,12 +345,10 @@ struct fd_block_map { }; typedef struct fd_block_map fd_block_map_t; -/* clang-format off */ -#define MAP_NAME fd_block_map -#define MAP_T fd_block_map_t -#define MAP_KEY slot +#define MAP_NAME fd_block_map +#define MAP_T fd_block_map_t +#define MAP_KEY slot #include "../../util/tmpl/fd_map_giant.c" -/* clang-format on */ /* fd_block_idx is an in-memory index of finalized blocks that have been archived to disk. It records the slot together with the byte offset @@ -351,7 +386,6 @@ struct fd_txn_map { }; typedef struct fd_txn_map fd_txn_map_t; -/* clang-format off */ int fd_txn_key_equal(fd_txn_key_t const * k0, fd_txn_key_t const * k1); ulong fd_txn_key_hash(fd_txn_key_t const * k, ulong seed); @@ -368,7 +402,6 @@ ulong fd_txn_key_hash(fd_txn_key_t const * k, ulong seed); files can be read back on initialization. */ struct fd_blockstore_archiver { - ulong magic; ulong fd_size_max; /* maximum size of the archival file */ ulong num_blocks; /* number of blocks in the archival file. needed for reading back */ ulong head; /* location of least recently written block */ @@ -377,8 +410,7 @@ struct fd_blockstore_archiver { typedef struct fd_blockstore_archiver fd_blockstore_archiver_t; #define FD_BLOCKSTORE_ARCHIVE_START sizeof(fd_blockstore_archiver_t) -struct __attribute__((aligned(FD_BLOCKSTORE_ALIGN))) fd_blockstore { -/* clang-format on */ +struct __attribute__((aligned(FD_BLOCKSTORE_ALIGN))) fd_blockstore_shmem { /* Metadata */ @@ -411,29 +443,43 @@ struct __attribute__((aligned(FD_BLOCKSTORE_ALIGN))) fd_blockstore { ulong txn_max; /* maximum # of transactions that can be indexed from blocks */ ulong alloc_max; /* maximum bytes that can be allocated */ - /* Owned */ - - ulong shred_pool_gaddr; /* memory pool for buffering shreds before block assembly */ - ulong shred_map_gaddr; /* map of (slot, shred_idx)->shred */ ulong block_map_gaddr; /* map of slot->(slot_meta, block) */ ulong block_idx_gaddr; /* map of slot->byte offset in archival file */ ulong slot_deque_gaddr; /* deque of slot numbers */ ulong txn_map_gaddr; ulong alloc_gaddr; }; +typedef struct fd_blockstore_shmem fd_blockstore_shmem_t; + +/* fd_blockstore_t is a local join to the blockstore. This is specific + to the local address space should not be shared across tiles. */ + +struct fd_blockstore { + + /* shared memory region */ + + fd_blockstore_shmem_t * shmem; + + /* local join handles */ + + fd_buf_shred_pool_t shred_pool[1]; + fd_buf_shred_map_t shred_map[1]; +}; typedef struct fd_blockstore fd_blockstore_t; FD_PROTOTYPES_BEGIN /* Construction API */ -/* TODO document lifecycle methods */ - FD_FN_CONST static inline ulong fd_blockstore_align( void ) { - return alignof(fd_blockstore_t); + return FD_BLOCKSTORE_ALIGN; } +/* fd_blockstore_footprint returns the footprint of the entire + blockstore shared memory region occupied by `fd_blockstore_shmem_t` + including data structures. */ + FD_FN_CONST static inline ulong fd_blockstore_footprint( ulong shred_max, ulong block_max, ulong idx_max, ulong txn_max ) { int lg_idx_max = fd_ulong_find_msb( fd_ulong_pow2_up( idx_max ) ); @@ -446,18 +492,29 @@ fd_blockstore_footprint( ulong shred_max, ulong block_max, ulong idx_max, ulong FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( + FD_LAYOUT_APPEND( + FD_LAYOUT_APPEND( FD_LAYOUT_INIT, - alignof(fd_blockstore_t), sizeof(fd_blockstore_t) ), - fd_buf_shred_pool_align(), fd_buf_shred_pool_footprint( shred_max ) ), - fd_buf_shred_map_align(), fd_buf_shred_map_footprint( shred_max ) ), - fd_block_map_align(), fd_block_map_footprint( block_max ) ), - fd_block_idx_align(), fd_block_idx_footprint( lg_idx_max ) ), - fd_slot_deque_align(), fd_slot_deque_footprint( block_max ) ), - fd_txn_map_align(), fd_txn_map_footprint( txn_max ) ), - fd_alloc_align(), fd_alloc_footprint() ), + alignof(fd_blockstore_shmem_t), sizeof(fd_blockstore_shmem_t) ), + alignof(fd_buf_shred_t), sizeof(fd_buf_shred_t) * shred_max ), + fd_buf_shred_pool_align(), fd_buf_shred_pool_footprint() ), + fd_buf_shred_map_align(), fd_buf_shred_map_footprint( shred_max ) ), + alignof(fd_block_map_t), sizeof(fd_block_map_t) * block_max ), + fd_block_map_align(), fd_block_map_footprint( block_max ) ), + fd_block_idx_align(), fd_block_idx_footprint( lg_idx_max ) ), + fd_slot_deque_align(), fd_slot_deque_footprint( block_max ) ), + fd_txn_map_align(), fd_txn_map_footprint( txn_max ) ), + fd_alloc_align(), fd_alloc_footprint() ), fd_blockstore_align() ); } +/* fd_blockstore_new formats a memory region with the appropriate + alignment and footprint into a blockstore. shmem points in the the + caller's address space of the memory region to format. Returns shmem + on success (blockstore has ownership of the memory region) and NULL + on failure (no changes, logs details). Caller is not joined on + return. The blockstore will be empty and unlocked. */ + void * fd_blockstore_new( void * shmem, ulong wksp_tag, @@ -467,8 +524,16 @@ fd_blockstore_new( void * shmem, ulong idx_max, ulong txn_max ); +/* fd_blockstore_join joins a blockstore. ljoin points to a + fd_blockstore_t compatible memory region in the caller's address + space used to hold info about the local join, shblockstore points in + the caller's address space to the memory region containing the + blockstore. Returns a handle to the caller's local join on success + (join has ownership of the ljoin region) and NULL on failure (no + changes, logs details). */ + fd_blockstore_t * -fd_blockstore_join( void * shblockstore ); +fd_blockstore_join( void * ljoin, void * shblockstore ); void * fd_blockstore_leave( fd_blockstore_t * blockstore ); @@ -509,7 +574,7 @@ fd_blockstore_fini( fd_blockstore_t * blockstore ); FD_FN_PURE static inline fd_wksp_t * fd_blockstore_wksp( fd_blockstore_t * blockstore ) { - return (fd_wksp_t *)( ( (ulong)blockstore ) - blockstore->blockstore_gaddr ); + return (fd_wksp_t *)( ( (ulong)blockstore->shmem ) - blockstore->shmem->blockstore_gaddr ); } /* fd_blockstore_wksp_tag returns the workspace allocation tag used by @@ -518,7 +583,7 @@ fd_blockstore_wksp( fd_blockstore_t * blockstore ) { FD_FN_PURE static inline ulong fd_blockstore_wksp_tag( fd_blockstore_t const * blockstore ) { - return blockstore->wksp_tag; + return blockstore->shmem->wksp_tag; } /* fd_blockstore_seed returns the hash seed used by the blockstore for various hash @@ -526,27 +591,7 @@ fd_blockstore_wksp_tag( fd_blockstore_t const * blockstore ) { TODO: consider renaming hash_seed? */ FD_FN_PURE static inline ulong fd_blockstore_seed( fd_blockstore_t const * blockstore ) { - return blockstore->seed; -} - -/* fd_blockstore_buf_shred_pool returns a pointer in the caller's - address space to the pool pointer fd_buf_shred_t * in the blockstore - wksp. Assumes blockstore is local join. Lifetime of the returned - pointer is that of the local join. */ - -FD_FN_PURE static inline fd_buf_shred_t * -fd_blockstore_shred_pool( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), blockstore->shred_pool_gaddr ); -} - -/* fd_blockstore_buf_shred_map returns a pointer in the caller's address - space to the fd_buf_shred_map_t * in the blockstore wksp. Assumes - blockstore is local join. Lifetime of the returned pointer is that - of the local join. */ - -FD_FN_PURE static inline fd_buf_shred_map_t * -fd_blockstore_shred_map( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), blockstore->shred_map_gaddr ); + return blockstore->shmem->seed; } /* fd_block_map returns a pointer in the caller's address space to the @@ -555,7 +600,7 @@ fd_blockstore_shred_map( fd_blockstore_t * blockstore ) { FD_FN_PURE static inline fd_block_map_t * fd_blockstore_block_map( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), blockstore->block_map_gaddr ); + return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), blockstore->shmem->block_map_gaddr ); } /* fd_block_idx returns a pointer in the caller's address space to the @@ -564,7 +609,7 @@ fd_blockstore_block_map( fd_blockstore_t * blockstore ) { FD_FN_PURE static inline fd_block_idx_t * fd_blockstore_block_idx( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), blockstore->block_idx_gaddr ); + return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), blockstore->shmem->block_idx_gaddr ); } /* fd_slot_deque returns a pointer in the caller's address space to the @@ -573,7 +618,7 @@ fd_blockstore_block_idx( fd_blockstore_t * blockstore ) { FD_FN_PURE static inline ulong * fd_blockstore_slot_deque( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->slot_deque_gaddr ); + return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->shmem->slot_deque_gaddr ); } /* fd_txn_map returns a pointer in the caller's address space to the blockstore's @@ -582,7 +627,7 @@ fd_blockstore_slot_deque( fd_blockstore_t * blockstore ) { FD_FN_PURE static inline fd_txn_map_t * fd_blockstore_txn_map( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->txn_map_gaddr ); + return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->shmem->txn_map_gaddr ); } /* fd_blockstore_alloc returns a pointer in the caller's address space to @@ -590,7 +635,7 @@ fd_blockstore_txn_map( fd_blockstore_t * blockstore ) { FD_FN_PURE static inline fd_alloc_t * /* Lifetime is that of the local join */ fd_blockstore_alloc( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->alloc_gaddr ); + return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->shmem->alloc_gaddr ); } /* fd_blockstore_block_data_laddr returns a local pointer to the block's @@ -611,16 +656,11 @@ fd_blockstore_block_micro_laddr( fd_blockstore_t * blockstore, fd_block_t * bloc return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), block->micros_gaddr ); } -/* fd_buf_shred_query queries the blockstore for shred at slot, - shred_idx. Returns a pointer to the shred or NULL if not in - blockstore. The returned pointer lifetime is until the shred is - removed. Check return value for error info. This API only works for - shreds from incomplete blocks. +/* fd_blockstore_shred_test returns 1 if a shred keyed by (slot, idx) is + already in the blockstore and 0 otherwise. */ - Callers should hold the read lock during the entirety of its read to - ensure the pointer remains valid. */ -fd_shred_t * -fd_buf_shred_query( fd_blockstore_t * blockstore, ulong slot, uint shred_idx ); +int +fd_blockstore_shred_test( fd_blockstore_t * blockstore, ulong slot, uint idx ); /* fd_buf_shred_query_copy_data queries the blockstore for shred at slot, shred_idx. Copies the shred data to the given buffer and @@ -628,6 +668,7 @@ fd_buf_shred_query( fd_blockstore_t * blockstore, ulong slot, uint shred_idx ); IMPORTANT! Caller MUST hold the read lock when calling this function. */ + long fd_buf_shred_query_copy_data( fd_blockstore_t * blockstore, ulong slot, @@ -679,7 +720,7 @@ ulong fd_blockstore_parent_slot_query( fd_blockstore_t * blockstore, ulong slot ); /* fd_blockstore_child_slots_query queries slot's child slots. Return - values are saved in slots_out and slot_cnt. Returns FD_BLOCKSTORE_OK + values are saved in slots_out and slot_cnt. Returns FD_BLOCKSTORE_SUCCESS on success, FD_BLOCKSTORE_ERR_SLOT_MISSING if slot is not in the blockstore. The returned slot array is always <= the max size FD_BLOCKSTORE_CHILD_SLOT_MAX and contiguous. Empty slots in the @@ -711,7 +752,7 @@ fd_blockstore_block_frontier_query( fd_blockstore_t * blockstore, block data (an allocator is needed because the block data sz is not known apriori). Returns FD_BLOCKSTORE_SLOT_MISSING if slot is missing: caller MUST ignore out pointers in this case. Otherwise this - call cannot fail and returns FD_BLOCKSTORE_OK. */ + call cannot fail and returns FD_BLOCKSTORE_SUCCESS. */ int fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, @@ -727,7 +768,7 @@ fd_blockstore_block_data_query_volatile( fd_blockstore_t * blockstore, /* fd_blockstore_block_map_query_volatile is the same as above except it only copies out the metadata (fd_block_map_t). Returns FD_BLOCKSTORE_SLOT_MISSING if slot is missing, otherwise - FD_BLOCKSTORE_OK. */ + FD_BLOCKSTORE_SUCCESS. */ int fd_blockstore_block_map_query_volatile( fd_blockstore_t * blockstore, @@ -766,12 +807,12 @@ fd_blockstore_slot_remove( fd_blockstore_t * blockstore, ulong slot ); /* Operations */ /* fd_blockstore_shred_insert inserts shred into the blockstore, fast - O(1). Returns the current `consumed_idx` for the shred's slot if - insert is successful, otherwise returns FD_SHRED_IDX_NULL on error. - Reasons for error include this shred is already in the blockstore or - the blockstore is full. */ + O(1). Returns the current `consumed_idx` for the shred's slot if + insert is successful, otherwise returns FD_SHRED_IDX_NULL on error. + Reasons for error include this shred is already in the blockstore or + the blockstore is full. */ -int +void fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shred ); /* fd_blockstore_buffered_shreds_remove removes all the unassembled shreds @@ -779,32 +820,35 @@ fd_blockstore_shred_insert( fd_blockstore_t * blockstore, fd_shred_t const * shr IMPORTANT! Caller MUST hold the write lock when calling this function. */ -int -fd_blockstore_buffered_shreds_remove( fd_blockstore_t * blockstore, ulong slot ); +void +fd_blockstore_shred_remove( fd_blockstore_t * blockstore, ulong slot, uint idx ); -/* fd_blockstore_batch_assemble assembles shreds for a given batch starting at shred_idx - Shred payloads are copied contiguously into block_data_out, and the total size - of the concatenated shred data is returned in block_data_sz. The caller provides the - max buffer size. Function will check if the provided shred_idx is the start of a batch - Returns an error code on success or failure. +/* fd_blockstore_slice_query queries for the block slice beginning from + shred `idx`. Copies at most `max` bytes of the shred payloads + consecutively from `idx` until the first {DATA, SLOT}_COMPLETES. + + Returns FD_BLOCKSTORE_SUCCESS (0) on success and a FD_MAP_ERR + (negative) on failure. On success, `buf` will be populated with the + copied slice and `buf_sz` will contain the number of bytes copied. + Caller must ignore the values of `buf` and `buf_sz` on failure. + + Implementation is lockfree and safe with concurrent operations on + blockstore. */ - IMPORTANT! Caller MUST hold the read lock when calling this - function. - */ int -fd_blockstore_batch_assemble( fd_blockstore_t * blockstore, - ulong slot, - uint batch_idx, - ulong block_data_max, - uchar * block_data_out, - ulong * block_data_sz ); - -/* fd_blockstore_shreds_complete should be a replacement for anywhere that is - querying for an fd_block_t * for existence but not actually using the block data. +fd_blockstore_slice_query( fd_blockstore_t * blockstore, + ulong slot, + uint idx, + ulong max, + uchar * buf, + ulong * buf_sz ); + +/* fd_blockstore_shreds_complete should be a replacement for anywhere that is + querying for an fd_block_t * for existence but not actually using the block data. Semantically equivalent to query_block( slot ) != NULL. IMPORTANT! Caller MUST hold the read lock when calling this function */ -bool +int fd_blockstore_shreds_complete( fd_blockstore_t * blockstore, ulong slot ); /* fd_blockstore_block_height_update sets the block height. @@ -835,25 +879,25 @@ fd_blockstore_publish( fd_blockstore_t * blockstore, int fd, ulong wmk ); /* fd_blockstore_start_read acquires the read lock */ static inline void fd_blockstore_start_read( fd_blockstore_t * blockstore ) { - fd_rwseq_start_read( &blockstore->lock ); + fd_rwseq_start_read( &blockstore->shmem->lock ); } /* fd_blockstore_end_read releases the read lock */ static inline void fd_blockstore_end_read( fd_blockstore_t * blockstore ) { - fd_rwseq_end_read( &blockstore->lock ); + fd_rwseq_end_read( &blockstore->shmem->lock ); } /* fd_blockstore_start_write acquire the write lock */ static inline void fd_blockstore_start_write( fd_blockstore_t * blockstore ) { - fd_rwseq_start_write( &blockstore->lock ); + fd_rwseq_start_write( &blockstore->shmem->lock ); } /* fd_blockstore_end_write releases the write lock */ static inline void fd_blockstore_end_write( fd_blockstore_t * blockstore ) { - fd_rwseq_end_write( &blockstore->lock ); + fd_rwseq_end_write( &blockstore->shmem->lock ); } void @@ -881,9 +925,9 @@ typedef struct fd_blockstore_ser fd_blockstore_ser_t; any necessary bookkeeping. If fd is -1, no write is attempted. Returns written size */ ulong -fd_blockstore_block_checkpt( fd_blockstore_t * blockstore, - fd_blockstore_ser_t * ser, - int fd, +fd_blockstore_block_checkpt( fd_blockstore_t * blockstore, + fd_blockstore_ser_t * ser, + int fd, ulong slot ); /* Restores a block and block map entry from fd at given offset. As this used by @@ -897,7 +941,7 @@ fd_blockstore_block_meta_restore( fd_blockstore_archiver_t * archvr, /* Reads block data from fd into a given buf. Modifies data_off similarly to meta_restore */ -int +int fd_blockstore_block_data_restore( fd_blockstore_archiver_t * archvr, int fd, fd_block_idx_t * block_idx_entry, diff --git a/src/flamenco/runtime/fd_rocksdb.c b/src/flamenco/runtime/fd_rocksdb.c index 59e42c160d..fae4fc6496 100644 --- a/src/flamenco/runtime/fd_rocksdb.c +++ b/src/flamenco/runtime/fd_rocksdb.c @@ -574,13 +574,13 @@ fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db, fd_blockstore_end_write(blockstore); return -1; } - int rc = fd_blockstore_shred_insert( blockstore, shred ); - if (rc != FD_BLOCKSTORE_OK_SLOT_COMPLETE && rc != FD_BLOCKSTORE_OK) { - FD_LOG_WARNING(("failed to store shred %lu/%lu", slot, i)); - rocksdb_iter_destroy(iter); - fd_blockstore_end_write(blockstore); - return -1; - } + fd_blockstore_shred_insert( blockstore, shred ); + fd_blockstore_end_write(blockstore); + // if (rc != FD_BLOCKSTORE_SUCCESS_SLOT_COMPLETE && rc != FD_BLOCKSTORE_SUCCESS) { + // FD_LOG_WARNING(("failed to store shred %lu/%lu", slot, i)); + // rocksdb_iter_destroy(iter); + // return -1; + // } rocksdb_iter_next(iter); } @@ -726,12 +726,12 @@ fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db, FD_TEST( blk->txns_meta_gaddr + blk->txns_meta_sz == fd_wksp_gaddr_fast( wksp, cur_laddr ) ); } - blockstore->lps = slot; - blockstore->hcs = slot; - blockstore->smr = slot; + blockstore->shmem->lps = slot; + blockstore->shmem->hcs = slot; + blockstore->shmem->smr = slot; if( FD_LIKELY( block_map_entry ) ) { - block_map_entry->flags = + block_map_entry->flags = fd_uchar_set_bit( fd_uchar_set_bit( fd_uchar_set_bit( diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index f3522124b5..3345bb4eea 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -1209,12 +1209,7 @@ fd_runtime_block_verify_ticks( fd_blockstore_t * blockstore, while ( batch_idx <= query->slot_complete_idx ) { batch_cnt++; ulong batch_sz = 0; - FD_TEST( fd_blockstore_batch_assemble( blockstore, - slot, - (uint) batch_idx, - block_data_sz, - block_data, - &batch_sz ) == FD_BLOCKSTORE_OK ); + FD_TEST( fd_blockstore_slice_query( blockstore, slot, (uint) batch_idx, block_data_sz, block_data, &batch_sz ) == FD_BLOCKSTORE_SUCCESS ); ulong micro_cnt = FD_LOAD( ulong, block_data ); ulong off = sizeof(ulong); for( ulong i = 0UL; i < micro_cnt; i++ ){ diff --git a/src/flamenco/runtime/test_archive_block.c b/src/flamenco/runtime/test_archive_block.c index a2ccdf6d1e..2152d15383 100644 --- a/src/flamenco/runtime/test_archive_block.c +++ b/src/flamenco/runtime/test_archive_block.c @@ -37,7 +37,8 @@ ulong txn_max = 128; txn_max ), \ 1UL ); \ FD_TEST( mem ); \ - fd_blockstore_t * blockstore = fd_blockstore_join( fd_blockstore_new( mem, \ + fd_blockstore_t * blockstore = fd_blockstore_join( &blockstore_ljoin, \ + fd_blockstore_new( mem, \ 1, \ 0, \ shred_max, \ @@ -63,14 +64,14 @@ blocks_equal(fd_block_t* block1, fd_block_t* block2) { return block1->data_sz == block2->data_sz && block1->rewards.collected_fees == block2->rewards.collected_fees; } -fd_block_map_t +fd_block_map_t query_block(bool expect, fd_blockstore_t * blockstore, int fd, ulong slotn){ ulong blk_sz; fd_block_map_t meta[1]; fd_block_rewards_t rewards[1]; fd_hash_t parent_hash; uchar * blk_data = NULL; - fd_valloc_t valloc = fd_alloc_virtual( fd_blockstore_alloc(blockstore) ); + fd_valloc_t valloc = fd_alloc_virtual( fd_blockstore_alloc(blockstore) ); bool success = fd_blockstore_block_data_query_volatile( blockstore, fd, slotn, valloc, &parent_hash, meta, rewards, &blk_data, &blk_sz ) == 0; if ( blk_data ) { fd_alloc_free( fd_blockstore_alloc(blockstore), blk_data ); @@ -96,15 +97,16 @@ test_archive_many_blocks( fd_wksp_t * wksp, int fd, ulong fd_size_max, ulong idx FD_TEST( ftruncate(fd, 0) == 0 ); FD_LOG_NOTICE(("fd is %d", fd)); + fd_blockstore_t blockstore_ljoin; \ CREATE_BLOCKSTORE(blockstore, slot_bank, mem, fake_hash); FD_TEST(fd_blockstore_init(blockstore, fd, fd_size_max,&slot_bank)); /* Store blocks that have been written to compare them later */ - fd_block_map_t * block_map_record = fd_alloc_malloc( fd_blockstore_alloc(blockstore), + fd_block_map_t * block_map_record = fd_alloc_malloc( fd_blockstore_alloc(blockstore), fd_block_map_align(), sizeof(fd_block_map_t) * (blocks + 1) ); int max_data_sz_pow = 20; - uchar buf_out[ (1 << max_data_sz_pow) ]; + uchar buf_out[ (1 << max_data_sz_pow) ]; fd_block_map_t block_map_entry; fd_block_t block; @@ -127,9 +129,9 @@ test_archive_many_blocks( fd_wksp_t * wksp, int fd, ulong fd_size_max, ulong idx fd_block_map_t block_map_entry_out; fd_block_t block_out; //ulong read_off = block_idx_entry->off; - fd_blockstore_block_meta_restore(&blockstore->archiver, fd, block_idx_entry, &block_map_entry_out, &block_out); - //read_off = wrap_offset(&blockstore->archiver, read_off + sizeof(fd_block_map_t) + sizeof(fd_block_t)); - fd_blockstore_block_data_restore(&blockstore->archiver, fd, block_idx_entry, buf_out, block_out.data_sz, block_out.data_sz); + fd_blockstore_block_meta_restore(&blockstore->shmem->archiver, fd, block_idx_entry, &block_map_entry_out, &block_out); + //read_off = wrap_offset(&blockstore->shmem->archiver, read_off + sizeof(fd_block_map_t) + sizeof(fd_block_t)); + fd_blockstore_block_data_restore(&blockstore->shmem->archiver, fd, block_idx_entry, buf_out, block_out.data_sz, block_out.data_sz); /* Check data read back matches data written */ @@ -149,7 +151,7 @@ test_archive_many_blocks( fd_wksp_t * wksp, int fd, ulong fd_size_max, ulong idx if ( slot % 10 == 0 ) { // periodically check all blocks in the block_idx match the blocks in the archive // and blocks in archive match what we store in memory - for( ulong s = lrw_slot; s != blockstore->mrw_slot; s++ ){ + for( ulong s = lrw_slot; s != blockstore->shmem->mrw_slot; s++ ){ fd_block_map_t blk_map = query_block(true, blockstore, fd, s); FD_TEST( memcmp( &blk_map, &block_map_record[s], sizeof(fd_block_map_t)) == 0 ); } @@ -167,6 +169,7 @@ void test_blockstore_archive_big( fd_wksp_t * wksp, int fd, ulong first_idx_max, FD_TEST( ftruncate(fd, 0) == 0 ); ulong idx_max = first_idx_max; + fd_blockstore_t blockstore_ljoin; \ CREATE_BLOCKSTORE(blockstore, slot_bank, mem, fake_hash); FD_TEST(fd_blockstore_init(blockstore, fd, FD_BLOCKSTORE_ARCHIVE_MIN_SIZE, &slot_bank)); @@ -215,7 +218,8 @@ void test_blockstore_archive_small( fd_wksp_t * wksp, int fd, ulong first_idx_ma test_archive_many_blocks( wksp, fd, FD_BLOCKSTORE_ARCHIVE_MIN_SIZE, first_idx_max, first_idx_max ); ulong last_archived = first_idx_max; - ulong idx_max = replay_idx_max; + ulong idx_max = replay_idx_max; + fd_blockstore_t blockstore_ljoin; \ CREATE_BLOCKSTORE( blockstore, slot_bank, mem, fake_hash ); // initialize from fd that was created from the test_archive_many_blocks @@ -233,9 +237,9 @@ void test_blockstore_archive_small( fd_wksp_t * wksp, int fd, ulong first_idx_ma fd_block_map_t lrw_block_map; fd_block_t lrw_block; ulong lrw_slot = fd_blockstore_archiver_lrw_slot( blockstore, fd, &lrw_block_map, &lrw_block ); - FD_LOG_NOTICE(("lrw_slot: %lu, mrw_slot: %lu", lrw_slot, blockstore->mrw_slot)); + FD_LOG_NOTICE(("lrw_slot: %lu, mrw_slot: %lu", lrw_slot, blockstore->shmem->mrw_slot)); FD_TEST( lrw_slot == last_archived - (idx_max - 1) + 1); - FD_TEST( blockstore->mrw_slot == last_archived); + FD_TEST( blockstore->shmem->mrw_slot == last_archived); /* Insert slot idx_max + 1 into the blockstore, should succeed */ @@ -245,12 +249,12 @@ void test_blockstore_archive_small( fd_wksp_t * wksp, int fd, ulong first_idx_ma fd_blockstore_block_checkpt( blockstore, &ser, fd, slot); /* Check that LRW was evicted, and MRW is updated */ - + lrw_slot = fd_blockstore_archiver_lrw_slot( blockstore, fd, &lrw_block_map, &lrw_block); FD_TEST( lrw_slot == slot - fd_block_idx_key_max( block_idx ) + 1); - FD_TEST( blockstore->mrw_slot == slot); + FD_TEST( blockstore->shmem->mrw_slot == slot); - for(ulong i = lrw_slot; i != blockstore->mrw_slot; i++){ + for(ulong i = lrw_slot; i != blockstore->shmem->mrw_slot; i++){ // can be reasonably sure that the blocks are read from file properly, as // the slot key is derived from block_map_out read from the file. FD_TEST( fd_block_idx_query(block_idx, i, NULL) ); @@ -262,18 +266,17 @@ void test_blockstore_metadata_invalid( int fd ){ FD_TEST( ftruncate(fd, 0) == 0 ); fd_blockstore_t blockstore; - blockstore.archiver.fd_size_max = 0x6000; + blockstore.shmem->archiver.fd_size_max = 0x6000; - fd_blockstore_archiver_t metadata = { .magic = FD_BLOCKSTORE_MAGIC, - .fd_size_max = 0x6000, - .head = 2, + fd_blockstore_archiver_t metadata = { .fd_size_max = 0x6000, + .head = 2, .tail = 3 }; FD_TEST( fd_blockstore_archiver_verify(&blockstore, &metadata) ); metadata.fd_size_max = 0x5000; FD_TEST( fd_blockstore_archiver_verify(&blockstore, &metadata) ); } -int +int main( int argc, char ** argv ) { fd_boot( &argc, &argv ); @@ -300,10 +303,10 @@ main( int argc, char ** argv ) { test_blockstore_archive_small(wksp, fd, 128, 64); test_blockstore_metadata_invalid(fd); - // tested archive with smaller fd size ( on order of 20KB ), by setting FD_BLOCKSTORE_ARCHIVE_MIN_SIZE + // tested archive with smaller fd size ( on order of 20KB ), by setting FD_BLOCKSTORE_ARCHIVE_MIN_SIZE ulong small_fd_size_max = FD_BLOCKSTORE_ARCHIVE_MIN_SIZE; test_archive_many_blocks(wksp, fd, small_fd_size_max, 4, 128); // small idx_mas - test_archive_many_blocks(wksp, fd, small_fd_size_max, 256, 512); + test_archive_many_blocks(wksp, fd, small_fd_size_max, 256, 512); test_archive_many_blocks(wksp, fd, small_fd_size_max, 1 << 12, 1025); // idx_max > blocks test_archive_many_blocks(wksp, fd, small_fd_size_max, 1 << 13, 1<<15); // large blocks @@ -313,7 +316,7 @@ main( int argc, char ** argv ) { return 0; } -#else +#else int main( int argc, diff --git a/src/flamenco/runtime/test_blockstore.c b/src/flamenco/runtime/test_blockstore.c new file mode 100644 index 0000000000..45d65d29d4 --- /dev/null +++ b/src/flamenco/runtime/test_blockstore.c @@ -0,0 +1,183 @@ +#include "fd_blockstore.h" + +#include +#include + +struct __attribute__((packed)) fd_shred_cap_file_hdr { + ushort magic; + ushort shred_cap_hdr_sz; +}; +typedef struct fd_shred_cap_file_hdr fd_shred_cap_file_hdr_t; + +struct __attribute__((packed)) fd_shred_cap_hdr { + ulong sz; + uchar flags; +}; +typedef struct fd_shred_cap_hdr fd_shred_cap_hdr_t; + +// static const uchar shred_bytes[FD_SHRED_MIN_SZ] = { 12, 20, 88, 140, 221, 68, 111, 148, 187, 119, 30, 22, 42, 221, 65, 43, 93, 170, 201, 121, 37, 87, 253, 68, 228, 161, 159, 159, 149, 93, 96, 134, 155, 92, 2, 73, 33, 46, 100, 22, 245, 94, 0, 144, 43, 171, 120, 101, 93, 222, 110, 116, 17, 96, 149, 145, 33, 119, 0, 163, 70, 166, 206, 6, 149, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 1, 0, 42, 47, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 247, 131, 90, 55, 245, 41, 73, 211, 141, 173, 29, 87, 159, 58, 136, 18, 205, 115, 200, 64, 195, 242, 252, 120, 220, 58, 254, 31, 67, 199, 42, 81, 109, 14, 250, 128, 50, 24, 176, 41, 132, 8, 60, 164, 149, 81, 6, 236, 49, 238, 200, 131, 75, 27, 146, 57, 2, 85, 228, 37, 131, 223, 245, 89, 100, 51, 148, 245, 134, 194, 194, 110, 240, 25, 201, 234, 239, 3, 62, 134, 94, 74, 139, 131, 28, 116, 160, 239, 153, 61, 58, 57, 122, 55, 56, 220, 88, 16, 105, 185 }; +// static void +// replay_slice( fd_blockstore_t * blockstore, uchar * slice, ulong slot, uint start_shred_idx, uint end_shred_idx ) { +// FD_LOG_NOTICE(( "replay_batch: slot: %lu, start_shred_idx: %u, end_shred_idx: %u", slot, start_shred_idx, end_shred_idx )); + +// ulong slice_sz; +// int err = fd_blockstore_slice_query( blockstore, slot, start_shred_idx, FD_SLICE_MAX, slice, &slice_sz ); +// FD_TEST( err == FD_BLOCKSTORE_SUCCESS ); + +// /* Loop through microblocks, parse out txns, and round-robin publish +// txns to the executor tiles. */ + +// uchar txn[FD_TXN_MAX_SZ] = { 0 }; +// ulong micro_cnt = FD_LOAD( ulong, slice ); +// ulong off = sizeof( ulong ); +// for( ulong i = 0; i < micro_cnt; ++i ) { +// fd_microblock_hdr_t * hdr = (fd_microblock_hdr_t *)fd_type_pun( slice+off ); +// off += sizeof(fd_microblock_hdr_t); +// for( ulong j = 0; j < hdr->txn_cnt; j++ ) { +// ulong pay_sz = 0; +// ulong txn_sz = fd_txn_parse_core( slice + off, fd_ulong_min( slice_sz - off, FD_TXN_MTU ), txn, NULL, &pay_sz ); +// if( FD_UNLIKELY( !pay_sz ) ) FD_LOG_ERR(( "failed to parse transaction %lu in microblock %lu in slot %lu", j, i, slot ) ); +// if( FD_UNLIKELY( !txn_sz || txn_sz > FD_TXN_MTU )) FD_LOG_ERR(( "failed to parse transaction %lu in microblock %lu in slot %lu. txn size: %lu", j, i, slot, txn_sz )); + +// /* TODO: PUBLISH TO MCACHE / DCACHE EXECUTOR TILE HERE */ +// // FD_LOG_HEXDUMP_NOTICE(( "txn", txn, txn_sz )); + +// off += pay_sz; +// } +// } +// } + +static void +replay_slice( fd_blockstore_t * blockstore, uchar * slice, ulong slot, uint idx ) { + FD_LOG_NOTICE(( "replay_batch: slot: %lu, idx: %u", slot, idx )); + + ulong slice_sz; + int err = fd_blockstore_slice_query( blockstore, slot, idx, FD_SLICE_MAX, slice, &slice_sz ); + FD_TEST( slice_sz < FD_SLICE_MAX ); + FD_TEST( err == FD_BLOCKSTORE_SUCCESS ); + + /* Loop through microblocks, parse out txns, and round-robin publish + txns to the executor tiles. */ + + uchar txn[FD_TXN_MAX_SZ] = { 0 }; + ulong micro_cnt = FD_LOAD( ulong, slice ); + ulong off = sizeof( ulong ); + for( ulong i = 0; i < micro_cnt; ++i ) { + fd_microblock_hdr_t * hdr = (fd_microblock_hdr_t *)fd_type_pun( slice+off ); + off += sizeof(fd_microblock_hdr_t); + for( ulong j = 0; j < hdr->txn_cnt; j++ ) { + ulong pay_sz = 0; + ulong txn_sz = fd_txn_parse_core( slice + off, fd_ulong_min( slice_sz - off, FD_TXN_MTU ), txn, NULL, &pay_sz ); + if( FD_UNLIKELY( !pay_sz ) ) FD_LOG_ERR(( "failed to parse transaction %lu in microblock %lu in slot %lu", j, i, slot ) ); + if( FD_UNLIKELY( !txn_sz || txn_sz > FD_TXN_MTU )) FD_LOG_ERR(( "failed to parse transaction %lu in microblock %lu in slot %lu. txn size: %lu", j, i, slot, txn_sz )); + + /* TODO: PUBLISH TO MCACHE / DCACHE EXECUTOR TILE HERE */ + // FD_LOG_HEXDUMP_NOTICE(( "txn", txn, txn_sz )); + + off += pay_sz; + } + } +} + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + fd_wksp_t * wksp = fd_wksp_new_anonymous( fd_cstr_to_shmem_page_sz( "gigantic" ), 50, fd_shmem_cpu_idx( fd_shmem_numa_idx( 0 ) ), "wksp", 0UL ); + FD_TEST( wksp ); + FD_PARAM_UNUSED uchar * slice = fd_wksp_alloc_laddr( wksp, 128UL, FD_SLICE_MAX, 1UL ); + + ulong shred_max = 1 << 24; + + void * mem = fd_wksp_alloc_laddr( wksp, fd_blockstore_align(), fd_blockstore_footprint( shred_max, 4096, 4096, shred_max ), 1UL ); + FD_TEST( mem ); + void * shblockstore = fd_blockstore_new( mem, 1UL, 42UL, shred_max, 4096, 4096, shred_max ); + FD_TEST( shblockstore ); + fd_blockstore_t blockstore_ljoin; + fd_blockstore_t * blockstore = fd_blockstore_join( &blockstore_ljoin, shblockstore ); + fd_buf_shred_pool_reset( blockstore->shred_pool, 0 ); + + blockstore->shmem->smr = 0; + + FILE * shred_cap = fopen( "/data/chali/testnet2.shredcap", "rb" ); + FD_TEST( shred_cap ); + + ulong cnt = 0; + ulong dup_cnt = 0; + ulong filter_cnt = 0; + for( ;; ) { + fd_shred_cap_hdr_t header; + ulong nshredcap_hdr = fread( &header, sizeof( fd_shred_cap_hdr_t ), 1, shred_cap ); + // FD_LOG_NOTICE(( "nshredcap_hdr: %lu", header.sz )); + if ( nshredcap_hdr != 1 ) break; + + uchar buffer[FD_SHRED_MAX_SZ]; + ulong shred_len = header.sz; + ulong bytes_read = fread( buffer, sizeof( uchar ), shred_len, shred_cap ); + if ( bytes_read != shred_len ) break; + + fd_shred_t const * shred = fd_shred_parse( buffer, shred_len ); + uchar shred_type = fd_shred_type( shred->variant ); + if( shred_type == FD_SHRED_TYPE_LEGACY_DATA || + shred_type == FD_SHRED_TYPE_MERKLE_DATA || + shred_type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED || + shred_type == FD_SHRED_TYPE_MERKLE_DATA_CHAINED_RESIGNED ) { + if( FD_UNLIKELY( !fd_blockstore_shreds_complete( blockstore, shred->slot ) ) ) { + fd_blockstore_shred_insert( blockstore, shred ); + + if ( fd_blockstore_shreds_complete( blockstore, shred->slot ) ) { + // fd_blockstore_start_read( blockstore ); + fd_block_map_t * block_map_entry = fd_blockstore_block_map_query( blockstore, shred->slot ); + // fd_blockstore_end_read( blockstore ); + FD_LOG_NOTICE(( "slot %lu block_map_entry->consumed_idx: %u, block_map_entry->buffered_idx: %u", shred->slot, block_map_entry->consumed_idx, block_map_entry->buffered_idx )); + + uint idx = block_map_entry->consumed_idx + 1; + while( idx < block_map_entry->buffered_idx ) { + if( FD_UNLIKELY( fd_block_set_test( block_map_entry->data_complete_idxs, idx ) ) ) { + FD_LOG_NOTICE(( "replay_batch: slot: %lu, start_shred_idx: %u, end_shred_idx: %u", shred->slot, block_map_entry->consumed_idx + 1, idx )); + + /* FIXME: backpressure? consumer will need to make sure they aren't overrun */ + + replay_slice( blockstore, slice, shred->slot, block_map_entry->consumed_idx + 1 ); + block_map_entry->consumed_idx = idx; + } + idx++; + } + fd_blockstore_end_read( blockstore ); + } + + + + cnt++; + } else { + dup_cnt++; + } + } else { + // FD_LOG_HEXDUMP_NOTICE(( "filtering", &shred_type, 1 )); + filter_cnt++; + } + /* + if ( FD_SHRED_CAP_FLAG_IS_TURBINE(header.flags) ) { + fd_replay_turbine_rx( replay, shred, fd_shred_sz( shred )); + } else { + fd_replay_repair_rx( replay, shred ); + } + */ + } + + FD_LOG_NOTICE(("inserted %lu %lu %lu shreds", cnt, dup_cnt, filter_cnt)); + + // fd_shred_t * shred = (fd_shred_t *)fd_type_pun_const( shred_bytes ); + // ulong slot = shred->slot; + // uint idx = shred->idx; + // fd_blockstore_shred_insert( blockstore, shred ); + + // fd_shred_key_t key = { slot, idx }; + // fd_buf_shred_map_query_t query; + // int err = fd_buf_shred_map_query_try( blockstore->shred_map, &key, NULL, &query ); + // FD_TEST( err != FD_MAP_ERR_INVAL && err != FD_MAP_ERR_CORRUPT && err != FD_MAP_ERR_KEY ); + // err = fd_buf_shred_map_query_test( &query ); + // FD_TEST( err == FD_MAP_SUCCESS ); + + fd_halt(); + return 0; +} diff --git a/src/flamenco/shredcap/fd_shredcap.c b/src/flamenco/shredcap/fd_shredcap.c index 227ff1da23..a85be6c753 100644 --- a/src/flamenco/shredcap/fd_shredcap.c +++ b/src/flamenco/shredcap/fd_shredcap.c @@ -356,9 +356,7 @@ fd_shredcap_verify_slot( fd_shredcap_slot_hdr_t * slot_hdr, } fd_shred_t * shred = (fd_shred_t*)rbuf; - fd_blockstore_start_write( blockstore ); fd_blockstore_shred_insert( blockstore, shred ); - fd_blockstore_end_write( blockstore ); if ( FD_UNLIKELY( slot != shred->slot ) ) { FD_LOG_ERR(( "slot header's slot=%lu doesn't match shred's slot=%lu", slot, shred->slot )); } @@ -1049,9 +1047,7 @@ fd_shredcap_populate_blockstore( const char * capture_dir, } fd_shred_t * shred = (fd_shred_t*)capture_buf; - fd_blockstore_start_write( blockstore ); fd_blockstore_shred_insert( blockstore, shred ); - fd_blockstore_end_write( blockstore ); } offset = lseek( capture_fd, (long)FD_SHREDCAP_SLOT_FTR_FOOTPRINT, SEEK_CUR );