From 4beaffe37630e409439b6976b761c6b9ff7abc07 Mon Sep 17 00:00:00 2001 From: SW van Heerden Date: Thu, 23 Jan 2025 09:10:31 +0200 Subject: [PATCH 1/4] fixes the smt cache height --- applications/minotari_node/src/builder.rs | 1 + applications/minotari_node/src/recovery.rs | 2 ++ .../core/src/chain_storage/lmdb_db/lmdb_db.rs | 20 +++++++++++++++---- .../core/src/test_helpers/blockchain.rs | 4 ++-- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/applications/minotari_node/src/builder.rs b/applications/minotari_node/src/builder.rs index cf7c70f39a..c908a07456 100644 --- a/applications/minotari_node/src/builder.rs +++ b/applications/minotari_node/src/builder.rs @@ -188,6 +188,7 @@ pub async fn configure_and_initialize_node( app_config.base_node.lmdb_path.as_path(), app_config.base_node.lmdb.clone(), app_config.base_node.storage.pruning_interval, + app_config.base_node.storage.pruning_horizon, rules, ) .map_err(|e| ExitError::new(ExitCode::DatabaseError, e))?; diff --git a/applications/minotari_node/src/recovery.rs b/applications/minotari_node/src/recovery.rs index 5020de3526..ffc79d3cc1 100644 --- a/applications/minotari_node/src/recovery.rs +++ b/applications/minotari_node/src/recovery.rs @@ -85,6 +85,7 @@ pub async fn run_recovery(node_config: &BaseNodeConfig) -> Result<(), anyhow::Er &node_config.lmdb_path, node_config.lmdb.clone(), node_config.storage.pruning_interval, + node_config.storage.pruning_horizon, rules.clone(), ) .map_err(|e| { @@ -97,6 +98,7 @@ pub async fn run_recovery(node_config: &BaseNodeConfig) -> Result<(), anyhow::Er &temp_path, node_config.lmdb.clone(), node_config.storage.pruning_interval, + node_config.storage.pruning_horizon, rules.clone(), ) .map_err(|e| { diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 53550dd3b2..186c0e30e4 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -163,7 +163,7 @@ const LMDB_DB_VALIDATOR_NODES: &str = "validator_nodes"; const LMDB_DB_VALIDATOR_NODES_MAPPING: &str = "validator_nodes_mapping"; const LMDB_DB_TEMPLATE_REGISTRATIONS: &str = "template_registrations"; const LMDB_DB_UTXO_SMT: &str = "utxo_smt"; -const SMT_CACHE_PERIOD: u64 = 1000; +const SMT_CACHE_PERIOD: u64 = 500; /// HeaderHash(32), mmr_pos(8), hash(32) type KernelKey = CompositeKey<72>; @@ -174,6 +174,7 @@ pub fn create_lmdb_database>( path: P, config: LMDBConfig, prune_interval: u64, + pruning_horizon: u64, consensus_manager: ConsensusManager, ) -> Result { let flags = db::CREATE; @@ -218,7 +219,13 @@ pub fn create_lmdb_database>( .build() .map_err(|err| ChainStorageError::CriticalError(format!("Could not create LMDB store:{}", err)))?; debug!(target: LOG_TARGET, "LMDB database creation successful"); - LMDBDatabase::new(&lmdb_store, file_lock, consensus_manager, prune_interval) + LMDBDatabase::new( + &lmdb_store, + file_lock, + consensus_manager, + prune_interval, + pruning_horizon, + ) } /// This is a lmdb-based blockchain database for persistent storage of the chain state. @@ -291,12 +298,17 @@ impl LMDBDatabase { file_lock: File, consensus_manager: ConsensusManager, prune_interval: u64, + pruning_horizon: u64, ) -> Result { let env = store.env(); - let smt_cache_period = if prune_interval == 0 { + let smt_cache_period = if pruning_horizon == 0 { SMT_CACHE_PERIOD } else { - prune_interval / 2 + if prune_interval == 0 { + SMT_CACHE_PERIOD + } else { + prune_interval / 2 + } }; let db = Self { diff --git a/base_layer/core/src/test_helpers/blockchain.rs b/base_layer/core/src/test_helpers/blockchain.rs index e800c2a1b0..735220fbb5 100644 --- a/base_layer/core/src/test_helpers/blockchain.rs +++ b/base_layer/core/src/test_helpers/blockchain.rs @@ -168,7 +168,7 @@ impl TempDatabase { let rules = create_consensus_rules(); Self { - db: Some(create_lmdb_database(&temp_path, LMDBConfig::default(), 0, rules).unwrap()), + db: Some(create_lmdb_database(&temp_path, LMDBConfig::default(), 0, 0, rules).unwrap()), path: temp_path, delete_on_drop: true, } @@ -177,7 +177,7 @@ impl TempDatabase { pub fn from_path>(temp_path: P) -> Self { let rules = create_consensus_rules(); Self { - db: Some(create_lmdb_database(&temp_path, LMDBConfig::default(), 0, rules).unwrap()), + db: Some(create_lmdb_database(&temp_path, LMDBConfig::default(), 0, 0, rules).unwrap()), path: temp_path.as_ref().to_path_buf(), delete_on_drop: true, } From 0109468e03bc7153c9d15ad6eb4bf050cb5a5f4f Mon Sep 17 00:00:00 2001 From: SW van Heerden Date: Thu, 23 Jan 2025 09:14:57 +0200 Subject: [PATCH 2/4] clippy --- base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 186c0e30e4..94c8d1d365 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -301,14 +301,10 @@ impl LMDBDatabase { pruning_horizon: u64, ) -> Result { let env = store.env(); - let smt_cache_period = if pruning_horizon == 0 { + let smt_cache_period = if pruning_horizon == 0 || prune_interval == 0 { SMT_CACHE_PERIOD } else { - if prune_interval == 0 { - SMT_CACHE_PERIOD - } else { - prune_interval / 2 - } + prune_interval / 2 }; let db = Self { From e46aa8592165322db523aa908023853b060ac57b Mon Sep 17 00:00:00 2001 From: SW van Heerden Date: Thu, 23 Jan 2025 11:36:15 +0200 Subject: [PATCH 3/4] add more logs --- base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 94c8d1d365..087462b375 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -469,6 +469,10 @@ impl LMDBDatabase { // for security we check that the best block does exist, and we check the previous value // we dont want to check this if the prev block has never been set, this means a empty hash of 32 // bytes. + debug!(target: LOG_TARGET, + "Setting new best block as height: {}", + height + ); if *height > 0 { let prev = fetch_best_block(&write_txn, &self.metadata_db)?; if *expected_prev_best_block != prev { @@ -1779,7 +1783,7 @@ impl LMDBDatabase { match lmdb_replace(txn, &self.utxo_smt, &k.as_u32(), smt, Some(estimated_bytes)) { Ok(_) => { - trace!( + debug!( target: LOG_TARGET, "Inserted ~{} MB with key '{}' into '{}' (size {}) in {:.2?}", estimated_bytes / BYTES_PER_MB, From 497dc82f47207b9b51b0abfa30790db2b3049925 Mon Sep 17 00:00:00 2001 From: SW van Heerden Date: Thu, 23 Jan 2025 15:11:28 +0200 Subject: [PATCH 4/4] add comment --- base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 087462b375..aadb611db8 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -304,6 +304,7 @@ impl LMDBDatabase { let smt_cache_period = if pruning_horizon == 0 || prune_interval == 0 { SMT_CACHE_PERIOD } else { + // we make sure we run this in the pruning interval, 2 is just a same number here. prune_interval / 2 };