Fix horizon sync

- Added integration-level horizon sync unit tests.
- Fixed horizon sync: initial, re-sync, re-sync after being offline.
- Added logic to detect genesys block outputs being spend.
- Fixed an issue where a tip block body could not be inserted due to the input being
  a compact input.
- Added integration-level block sync unit tests to verufy ^^.

applications/minotari_app_grpc/src/conversions/chain_metadata.rs

@@ -29,8 +29,8 @@ impl From<ChainMetadata> for grpc::MetaData {
         let mut diff = [0u8; 32];
         meta.accumulated_difficulty().to_big_endian(&mut diff);
         Self {
-            height_of_longest_chain: meta.height_of_longest_chain(),
-            best_block: meta.best_block().to_vec(),
+            height_of_longest_chain: meta.best_block_height(),
+            best_block: meta.best_block_hash().to_vec(),
             pruned_height: meta.pruned_height(),
             accumulated_difficulty: diff.to_vec(),
         }

applications/minotari_console_wallet/src/grpc/wallet_grpc_server.rs

@@ -165,7 +165,7 @@ impl WalletGrpcServer {
             .wallet
             .db
             .get_chain_metadata()?
-            .map(|m| m.height_of_longest_chain())
+            .map(|m| m.best_block_height())
             .unwrap_or_default();
         Ok(self.rules.consensus_constants(height))
     }

applications/minotari_console_wallet/src/ui/components/base_node.rs

@@ -68,7 +68,7 @@ impl<B: Backend> Component<B> for BaseNode {
             OnlineStatus::Online => {
                 let base_node_state = app_state.get_base_node_state();
                 if let Some(ref metadata) = base_node_state.chain_metadata {
-                    let tip = metadata.height_of_longest_chain();
+                    let tip = metadata.best_block_height();
 
                     let synced = base_node_state.is_synced.unwrap_or_default();
                     let (tip_color, sync_text) = if synced {

applications/minotari_console_wallet/src/ui/components/transactions_tab.rs

@@ -198,10 +198,7 @@ impl TransactionsTab {
             .collect();
 
         let base_node_state = app_state.get_base_node_state();
-        let chain_height = base_node_state
-            .chain_metadata
-            .as_ref()
-            .map(|cm| cm.height_of_longest_chain());
+        let chain_height = base_node_state.chain_metadata.as_ref().map(|cm| cm.best_block_height());
 
         let mut column0_items = Vec::new();
         let mut column1_items = Vec::new();

applications/minotari_node/src/commands/command/check_db.rs

@@ -43,12 +43,12 @@ impl CommandContext {
     /// Function to process the check-db command
     pub async fn check_db(&mut self) -> Result<(), Error> {
         let meta = self.node_service.get_metadata().await?;
-        let mut height = meta.height_of_longest_chain();
+        let mut height = meta.best_block_height();
         let mut missing_blocks = Vec::new();
         let mut missing_headers = Vec::new();
         print!("Searching for height: ");
         // We need to check every header, but not every block.
-        let horizon_height = meta.horizon_block_height(height);
+        let horizon_height = meta.pruned_height_at_given_chain_tip(height);
         while height > 0 {
             print!("{}", height);
             io::stdout().flush().await?;

applications/minotari_node/src/commands/command/list_connections.rs

@@ -64,7 +64,7 @@ impl CommandContext {
             let chain_height = peer
                 .get_metadata(1)
                 .and_then(|v| bincode::deserialize::<PeerMetadata>(v).ok())
-                .map(|metadata| format!("height: {}", metadata.metadata.height_of_longest_chain()));
+                .map(|metadata| format!("height: {}", metadata.metadata.best_block_height()));
 
             let ua = peer.user_agent;
             let rpc_sessions = self

applications/minotari_node/src/commands/command/list_peers.rs

@@ -86,7 +86,7 @@ impl CommandContext {
                     .get_metadata(1)
                     .and_then(|v| bincode::deserialize::<PeerMetadata>(v).ok())
                 {
-                    s.push(format!("chain height: {}", metadata.metadata.height_of_longest_chain()));
+                    s.push(format!("chain height: {}", metadata.metadata.best_block_height()));
                 }
 
                 if let Some(last_seen) = peer.addresses.last_seen() {

applications/minotari_node/src/commands/command/list_validator_nodes.rs

@@ -62,13 +62,11 @@ impl CommandContext {
     /// Function to process the list-connections command
     pub async fn list_validator_nodes(&mut self, args: Args) -> Result<(), Error> {
         let metadata = self.blockchain_db.get_chain_metadata().await?;
-        let constants = self
-            .consensus_rules
-            .consensus_constants(metadata.height_of_longest_chain());
+        let constants = self.consensus_rules.consensus_constants(metadata.best_block_height());
         let height = args
             .epoch
             .map(|epoch| constants.epoch_to_block_height(epoch))
-            .unwrap_or_else(|| metadata.height_of_longest_chain());
+            .unwrap_or_else(|| metadata.best_block_height());
         let current_epoch = constants.block_height_to_epoch(height);
         let next_epoch = VnEpoch(current_epoch.as_u64() + 1);
         let next_epoch_height = constants.epoch_to_block_height(next_epoch);

applications/minotari_node/src/commands/command/period_stats.rs

@@ -58,7 +58,7 @@ impl CommandContext {
     ) -> Result<(), Error> {
         let meta = self.node_service.get_metadata().await?;
 
-        let mut height = meta.height_of_longest_chain();
+        let mut height = meta.best_block_height();
         // Currently gets the stats for: tx count, hash rate estimation, target difficulty, solvetime.
         let mut results: Vec<(usize, f64, u64, u64, usize)> = Vec::new();
 

applications/minotari_node/src/commands/command/status.rs

@@ -64,7 +64,7 @@ impl CommandContext {
         status_line.add_field("State", self.state_machine_info.borrow().state_info.short_desc());
 
         let metadata = self.node_service.get_metadata().await?;
-        let height = metadata.height_of_longest_chain();
+        let height = metadata.best_block_height();
         let last_header = self
             .node_service
             .get_header(height)
@@ -76,16 +76,10 @@ impl CommandContext {
         );
         status_line.add_field(
             "Tip",
-            format!(
-                "{} ({})",
-                metadata.height_of_longest_chain(),
-                last_block_time.to_rfc2822()
-            ),
+            format!("{} ({})", metadata.best_block_height(), last_block_time.to_rfc2822()),
         );
 
-        let constants = self
-            .consensus_rules
-            .consensus_constants(metadata.height_of_longest_chain());
+        let constants = self.consensus_rules.consensus_constants(metadata.best_block_height());
         let fut = self.mempool_service.get_mempool_stats();
         if let Ok(mempool_stats) = time::timeout(Duration::from_secs(5), fut).await? {
             status_line.add_field(

applications/minotari_node/src/grpc/base_node_grpc_server.rs

@@ -379,7 +379,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
                     Status::internal(err.to_string()),
                 ));
             },
-            Ok(data) => data.height_of_longest_chain(),
+            Ok(data) => data.best_block_height(),
         };
 
         let sorting: Sorting = request.sorting();

applications/minotari_node/src/grpc/blocks.rs

@@ -55,7 +55,7 @@ pub async fn block_heights(
             .get_metadata()
             .await
             .map_err(|e| Status::internal(e.to_string()))?;
-        let tip = metadata.height_of_longest_chain();
+        let tip = metadata.best_block_height();
         // Avoid overflow
         let height_from_tip = cmp::min(tip, from_tip);
         let start = cmp::max(tip - height_from_tip, 0);

applications/minotari_node/src/recovery.rs

@@ -152,7 +152,7 @@ async fn do_recovery<D: BlockchainBackend + 'static>(
     let max_height = source_database
         .get_chain_metadata()
         .map_err(|e| anyhow!("Could not get max chain height: {}", e))?
-        .height_of_longest_chain();
+        .best_block_height();
     // we start at height 1
     let mut counter = 1;
     print!("Starting recovery at height: ");

base_layer/common_types/src/chain_metadata.rs

@@ -25,22 +25,22 @@ use std::fmt::{Display, Error, Formatter};
 use primitive_types::U256;
 use serde::{Deserialize, Serialize};
 
-use crate::types::{BlockHash, FixedHash};
+use crate::types::BlockHash;
 
 #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, Hash)]
 pub struct ChainMetadata {
-    /// The current chain height, or the block number of the longest valid chain, or `None` if there is no chain
-    height_of_longest_chain: u64,
+    /// The current chain height, or the block number of the longest valid chain
+    best_block_height: u64,
     /// The block hash of the current tip of the longest valid chain
-    best_block: BlockHash,
+    best_block_hash: BlockHash,
     /// The configured number of blocks back from the tip that this database tracks. A value of 0 indicates that
     /// pruning mode is disabled and the node will keep full blocks from the time it was set. If pruning horizon
     /// was previously enabled, previously pruned blocks will remain pruned. If set from initial sync, full blocks
     /// are preserved from genesis (i.e. the database is in full archival mode).
     pruning_horizon: u64,
     /// The height of the pruning horizon. This indicates from what height a full block can be provided
     /// (exclusive). If `pruned_height` is equal to the `height_of_longest_chain` no blocks can be
-    /// provided. Archival nodes wil always have an `pruned_height` of zero.
+    /// provided. Archival nodes wil always have a `pruned_height` of zero.
     pruned_height: u64,
     /// The total accumulated proof of work of the longest chain
     accumulated_difficulty: U256,
@@ -50,54 +50,33 @@ pub struct ChainMetadata {
 
 impl ChainMetadata {
     pub fn new(
-        height: u64,
-        hash: BlockHash,
+        best_block_height: u64,
+        best_block_hash: BlockHash,
         pruning_horizon: u64,
         pruned_height: u64,
         accumulated_difficulty: U256,
         timestamp: u64,
     ) -> ChainMetadata {
         ChainMetadata {
-            height_of_longest_chain: height,
-            best_block: hash,
+            best_block_height,
+            best_block_hash,
             pruning_horizon,
             pruned_height,
             accumulated_difficulty,
             timestamp,
         }
     }
 
-    pub fn empty() -> ChainMetadata {
-        ChainMetadata {
-            height_of_longest_chain: 0,
-            best_block: FixedHash::zero(),
-            pruning_horizon: 0,
-            pruned_height: 0,
-            accumulated_difficulty: 0.into(),
-            timestamp: 0,
-        }
-    }
-
     /// The block height at the pruning horizon, given the chain height of the network. Typically database backends
     /// cannot provide any block data earlier than this point.
     /// Zero is returned if the blockchain still hasn't reached the pruning horizon.
-    pub fn horizon_block_height(&self, chain_tip: u64) -> u64 {
+    pub fn pruned_height_at_given_chain_tip(&self, chain_tip: u64) -> u64 {
         match self.pruning_horizon {
             0 => 0,
             horizon => chain_tip.saturating_sub(horizon),
         }
     }
 
-    /// Set the pruning horizon to indicate that the chain is in archival mode (i.e. a pruning horizon of zero)
-    pub fn archival_mode(&mut self) {
-        self.pruning_horizon = 0;
-    }
-
-    /// Set the pruning horizon
-    pub fn set_pruning_horizon(&mut self, pruning_horizon: u64) {
-        self.pruning_horizon = pruning_horizon;
-    }
-
     /// The configured number of blocks back from the tip that this database tracks. A value of 0 indicates that
     /// pruning mode is disabled and the node will keep full blocks from the time it was set. If pruning horizon
     /// was previously enabled, previously pruned blocks will remain pruned. If set from initial sync, full blocks
@@ -117,13 +96,13 @@ impl ChainMetadata {
     }
 
     /// Returns the height of longest chain.
-    pub fn height_of_longest_chain(&self) -> u64 {
-        self.height_of_longest_chain
+    pub fn best_block_height(&self) -> u64 {
+        self.best_block_height
     }
 
     /// The height of the pruning horizon. This indicates from what height a full block can be provided
     /// (exclusive). If `pruned_height` is equal to the `height_of_longest_chain` no blocks can be
-    /// provided. Archival nodes wil always have an `pruned_height` of zero.
+    /// provided. Archival nodes wil always have a `pruned_height` of zero.
     pub fn pruned_height(&self) -> u64 {
         self.pruned_height
     }
@@ -132,8 +111,8 @@ impl ChainMetadata {
         self.accumulated_difficulty
     }
 
-    pub fn best_block(&self) -> &BlockHash {
-        &self.best_block
+    pub fn best_block_hash(&self) -> &BlockHash {
+        &self.best_block_hash
     }
 
     pub fn timestamp(&self) -> u64 {
@@ -143,14 +122,11 @@ impl ChainMetadata {
 
 impl Display for ChainMetadata {
     fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> {
-        let height = self.height_of_longest_chain;
-        let best_block = self.best_block;
-        let accumulated_difficulty = self.accumulated_difficulty;
-        writeln!(f, "Height of longest chain: {}", height)?;
-        writeln!(f, "Total accumulated difficulty: {}", accumulated_difficulty)?;
-        writeln!(f, "Best block: {}", best_block)?;
+        writeln!(f, "Best block height: {}", self.best_block_height)?;
+        writeln!(f, "Total accumulated difficulty: {}", self.accumulated_difficulty)?;
+        writeln!(f, "Best block hash: {}", self.best_block_hash)?;
         writeln!(f, "Pruning horizon: {}", self.pruning_horizon)?;
-        writeln!(f, "Effective pruned height: {}", self.pruned_height)?;
+        writeln!(f, "Pruned height: {}", self.pruned_height)?;
         Ok(())
     }
 }
@@ -161,33 +137,53 @@ mod test {
 
     #[test]
     fn horizon_block_on_default() {
-        let metadata = ChainMetadata::empty();
-        assert_eq!(metadata.horizon_block_height(0), 0);
+        let metadata = ChainMetadata {
+            best_block_height: 0,
+            best_block_hash: Default::default(),
+            pruning_horizon: 0,
+            pruned_height: 0,
+            accumulated_difficulty: Default::default(),
+            timestamp: 0,
+        };
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(0), 0);
     }
 
     #[test]
     fn pruned_mode() {
-        let mut metadata = ChainMetadata::empty();
+        let mut metadata = ChainMetadata {
+            best_block_height: 0,
+            best_block_hash: Default::default(),
+            pruning_horizon: 0,
+            pruned_height: 0,
+            accumulated_difficulty: Default::default(),
+            timestamp: 0,
+        };
         assert!(!metadata.is_pruned_node());
         assert!(metadata.is_archival_node());
-        metadata.set_pruning_horizon(2880);
+        metadata.pruning_horizon = 2880;
         assert!(metadata.is_pruned_node());
         assert!(!metadata.is_archival_node());
-        assert_eq!(metadata.horizon_block_height(0), 0);
-        assert_eq!(metadata.horizon_block_height(100), 0);
-        assert_eq!(metadata.horizon_block_height(2880), 0);
-        assert_eq!(metadata.horizon_block_height(2881), 1);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(0), 0);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(100), 0);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(2880), 0);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(2881), 1);
     }
 
     #[test]
     fn archival_node() {
-        let mut metadata = ChainMetadata::empty();
-        metadata.archival_mode();
+        let metadata = ChainMetadata {
+            best_block_height: 0,
+            best_block_hash: Default::default(),
+            pruning_horizon: 0,
+            pruned_height: 0,
+            accumulated_difficulty: Default::default(),
+            timestamp: 0,
+        };
         // Chain is still empty
-        assert_eq!(metadata.horizon_block_height(0), 0);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(0), 0);
         // When pruning horizon is zero, the horizon block is always 0, the genesis block
-        assert_eq!(metadata.horizon_block_height(0), 0);
-        assert_eq!(metadata.horizon_block_height(100), 0);
-        assert_eq!(metadata.horizon_block_height(2881), 0);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(0), 0);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(100), 0);
+        assert_eq!(metadata.pruned_height_at_given_chain_tip(2881), 0);
     }
 }

base_layer/core/src/base_node/chain_metadata_service/service.rs

@@ -202,7 +202,7 @@ impl ChainMetadataService {
             target: LOG_TARGET,
             "Received chain metadata from NodeId '{}' #{}, Acc_diff {}",
             event.node_id,
-            chain_metadata.height_of_longest_chain(),
+            chain_metadata.best_block_height(),
             chain_metadata.accumulated_difficulty(),
         );
 
@@ -333,7 +333,7 @@ mod test {
         let metadata = events_rx.recv().await.unwrap().peer_metadata().unwrap();
         assert_eq!(*metadata.node_id(), node_id);
         assert_eq!(
-            metadata.claimed_chain_metadata().height_of_longest_chain(),
+            metadata.claimed_chain_metadata().best_block_height(),
             proto_chain_metadata.height_of_longest_chain
         );
     }

base_layer/core/src/base_node/comms_interface/inbound_handlers.rs

@@ -602,15 +602,15 @@ where B: BlockchainBackend + 'static
         // We check the current tip and orphan status of the block because we cannot guarantee that mempool state is
         // correct and the mmr root calculation is only valid if the block is building on the tip.
         let current_meta = self.blockchain_db.get_chain_metadata().await?;
-        if header.prev_hash != *current_meta.best_block() {
+        if header.prev_hash != *current_meta.best_block_hash() {
             debug!(
                 target: LOG_TARGET,
                 "Orphaned block #{}: ({}), current tip is: #{} ({}). We need to fetch the complete block from peer: \
                  ({})",
                 header.height,
                 block_hash.to_hex(),
-                current_meta.height_of_longest_chain(),
-                current_meta.best_block().to_hex(),
+                current_meta.best_block_height(),
+                current_meta.best_block_hash().to_hex(),
                 source_peer,
             );
             #[cfg(feature = "metrics")]